From 42efb0942ae187c199b7c0da69807b78ea7f7c03 Mon Sep 17 00:00:00 2001 From: zhouzhou Date: Sat, 15 Feb 2025 13:55:16 +0800 Subject: [PATCH 1/7] fix http request timeout --- .../main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java index f66dce3..2f29f19 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java @@ -235,11 +235,11 @@ public CompletableFuture download(Path path, RequestID rid) throws URI result.completeExceptionally(ex); } // } - }); + }).orTimeout(20, TimeUnit.MINUTES); // fut.whenComplete(onComplete); BH bh = new BH(dlId, u, fut, perUrl); futures.add(fut); - client.sendAsync(req, bh); + client.sendAsync(req, bh).orTimeout(15, TimeUnit.MINUTES); } return result; } From 6ebb4f1cafef4d27715a6c1255d49c7bf5bc20a1 Mon Sep 17 00:00:00 2001 From: zhouzhou Date: Sat, 15 Feb 2025 13:59:11 +0800 Subject: [PATCH 2/7] fix if network error --- .../java/com/mastfrog/tinymavenproxy/DownloaderV2A.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java index 2f29f19..1fa96b7 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java @@ -239,7 +239,12 @@ public CompletableFuture download(Path path, RequestID rid) throws URI // fut.whenComplete(onComplete); BH bh = new BH(dlId, u, fut, perUrl); futures.add(fut); - client.sendAsync(req, bh).orTimeout(15, TimeUnit.MINUTES); + client.sendAsync(req, bh).orTimeout(15, TimeUnit.MINUTES).exceptionally((e) -> { + if(!fut.isCancelled()){ + fut.cancel(true); + } + return null; + }); } return result; } From 4526a25cae2ef8c50c41d60de83f831aac66ef39 Mon Sep 17 00:00:00 2001 From: zhouzhou Date: Mon, 17 Feb 2025 11:02:12 +0800 Subject: [PATCH 3/7] modify request user-agent --- .../com/mastfrog/tinymavenproxy/DownloaderV2A.java | 10 +++++----- .../java/com/mastfrog/tinymavenproxy/GetActeur.java | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java index 1fa96b7..d4041ef 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java @@ -87,7 +87,7 @@ public class DownloaderV2A { static final AtomicLong counter = new AtomicLong(); private final TempFiles tempFiles; private final String runId; - private final String userAgent; + // private final String userAgent; private final ExecutorService pool; @Inject @@ -104,7 +104,7 @@ public DownloaderV2A(HttpClient client, Config config, FileFinder finder, this.control = control; this.runId = runId; this.tempFiles = tempFiles; - userAgent = "tmpx-" + ver.version; + // userAgent = "tmpx-" + ver.version; } String nextDownloadId() { @@ -115,8 +115,8 @@ public boolean isFailedPath(Path path) { return failedURLs.getIfPresent(path) != null; } - CompletableFuture download(Path path, RequestID rid, DownloadReceiver recv) throws URISyntaxException { - CompletableFuture tf = download(path, rid); + CompletableFuture download(Path path, RequestID rid, String userAgent, DownloadReceiver recv) throws URISyntaxException { + CompletableFuture tf = download(path, rid, userAgent); Logs requestLog = logger.child("download", rid); tf.whenComplete((file, thrown) -> { if (thrown != null) { @@ -137,7 +137,7 @@ CompletableFuture download(Path path, RequestID rid, DownloadReceiver return tf; } - public CompletableFuture download(Path path, RequestID rid) throws URISyntaxException { + public CompletableFuture download(Path path, RequestID rid, String userAgent) throws URISyntaxException { Collection urls = config.withPath(path); List> futures = new ArrayList<>(urls.size()); Int remainder = Int.createAtomic(); diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java index 3aa2155..6d564a1 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java @@ -179,7 +179,8 @@ public class GetActeur extends Acteur { Path pth = path.elideEmptyElements(); def.defer((Resumer res) -> { config.debugLog(" defer and download ", pth); - CompletableFuture l = dl.download(pth, id, new DownloadReceiverImpl(res, config)); + CompletableFuture l = dl.download(pth, id, req.header(Headers.USER_AGENT).toString(), + new DownloadReceiverImpl(res, config)); req.channel().closeFuture().addListener(cl -> { l.cancel(false); }); From 809c8cf7d7ca81e0e1441da382cb17e35f8388c9 Mon Sep 17 00:00:00 2001 From: zhouzhou Date: Mon, 17 Feb 2025 16:32:42 +0800 Subject: [PATCH 4/7] modify Downloader to Single class --- .../main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java | 2 ++ .../src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java index d4041ef..26d1f53 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java @@ -26,6 +26,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.inject.Inject; +import com.google.inject.Singleton; import com.google.inject.name.Named; import com.mastfrog.acteur.errors.ResponseException; import com.mastfrog.acteur.headers.Headers; @@ -75,6 +76,7 @@ * * @author Tim Boudreau */ +@Singleton public class DownloaderV2A { private final HttpClient client; diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java index bd1452a..00887c9 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java @@ -24,6 +24,7 @@ package com.mastfrog.tinymavenproxy; import com.google.inject.Inject; +import com.google.inject.Singleton; import com.google.inject.name.Named; import com.mastfrog.acteur.server.ServerModule; import static com.mastfrog.tinymavenproxy.GetActeur.isGzipCacheFile; @@ -48,6 +49,7 @@ * * @author Tim Boudreau */ +@Singleton public class FileFinder { private final Config config; From 51ac0d2ed8df7bfa9b76aaf30810dd35010f2390 Mon Sep 17 00:00:00 2001 From: zhouzhou Date: Tue, 18 Feb 2025 09:33:48 +0800 Subject: [PATCH 5/7] add background setting --- .../java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java index 6214481..f49a183 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java @@ -104,11 +104,12 @@ static SettingsBuilder defaultSettings() { .add("cors.enabled", false) .add("download-tmp", System.getProperty("java.io.tmpdir")) .add(HTTP_COMPRESSION, "false") - .add(SETTINGS_KEY_DOWNLOAD_THREADS, "24") + .add(SETTINGS_KEY_DOWNLOAD_THREADS, 24) .add(SETTINGS_KEY_ASYNC_LOGGING, false) .add(LoggingModule.SETTINGS_KEY_LOG_TO_CONSOLE, true) - .add(WORKER_THREADS, "6") - .add(EVENT_THREADS, "3") + .add(WORKER_THREADS, 6) + .add(EVENT_THREADS, 3) + .add(ServerModule.BACKGROUND_THREADS, 4) .add(ServerModule.SETTINGS_KEY_SOCKET_WRITE_SPIN_COUNT, 32) .add(SETTINGS_KEY_LOG_LEVEL, "trace") // .add(MAX_CONTENT_LENGTH, "128") // we don't accept PUTs, no need for a big buffer From e470e9851a3d2860eec700129eeea467a16d100b Mon Sep 17 00:00:00 2001 From: zhouzhou Date: Tue, 18 Feb 2025 12:20:08 +0800 Subject: [PATCH 6/7] modify cancel request cache --- .../mastfrog/tinymavenproxy/DownloaderV2A.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java index 26d1f53..24d4e1b 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java @@ -122,8 +122,10 @@ CompletableFuture download(Path path, RequestID rid, String userAgent, Logs requestLog = logger.child("download", rid); tf.whenComplete((file, thrown) -> { if (thrown != null) { - failedURLs.put(path, path); // System.out.println("DO FAIL FOR " + thrown + " " + path); + if (!(thrown instanceof CancellationException)) { + failedURLs.put(path, path); + } recv.failed(GONE, thrown.getMessage()); } else if (file != null) { HttpResponseStatus status = file.info().map(info -> { @@ -218,7 +220,12 @@ public CompletableFuture download(Path path, RequestID rid, String use lr.add("cancelled"); if (remaining == 0) { // System.out.println("REMAINING 0 COMPLETE 1 " + path); - result.completeExceptionally(new ResponseException(GONE, "No result " + path)); + result.completeExceptionally(new CancellationException("No result " + path)); + } + } else if (thrown instanceof ResponseException) { + lr.add("faild"); + if (remaining == 0) { + result.completeExceptionally(thrown); } } else if (thrown != null) { if (remaining == 0) { @@ -283,7 +290,9 @@ public BodySubscriber apply(HttpResponse.ResponseInfo info) { logs.warn("request-failed") .add("status", info.statusCode()).close(); // .add("headers", info.headers().map()).close(); - result.cancel(true); + result.completeExceptionally(new ResponseException(HttpResponseStatus.valueOf(info.statusCode()), + HttpResponseStatus.valueOf(info.statusCode()).reasonPhrase())); + // result.cancel(true); return NO_OP; } else { logs.info("potential-success") From a657ff827fc05f95796da5c938bff063bd7a5a6e Mon Sep 17 00:00:00 2001 From: zhouzhou Date: Tue, 18 Feb 2025 12:47:44 +0800 Subject: [PATCH 7/7] cache timeout result --- .../main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java index 24d4e1b..f1e075b 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java @@ -220,7 +220,9 @@ public CompletableFuture download(Path path, RequestID rid, String use lr.add("cancelled"); if (remaining == 0) { // System.out.println("REMAINING 0 COMPLETE 1 " + path); - result.completeExceptionally(new CancellationException("No result " + path)); + // result.completeExceptionally(new CancellationException("No result " + path)); + // if timeout，cache No result url + result.completeExceptionally(new ResponseException(GONE, "No result " + path)); } } else if (thrown instanceof ResponseException) { lr.add("faild");