kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2025-04-27 23:40:36 +05:30
Code Issues Projects Releases Wiki Activity

Improve rate limiting

Browse Source 
* Bandcamp also has rate-limiting (but a lot more req are allowed than with YT)
* Shorten default warm up time
* Fix typo
This commit is contained in:
litetex 2025-02-11 21:50:38 +01:00
parent 287123b0fd
commit cf5df3fcff
No known key found for this signature in database
GPG Key ID: 525B43E6039B3689
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
extractor/src/test/java/org/schabi/newpipe/downloader/ratelimiting/RateLimitedClientWrapper.java
View File

@ -21,7 +21,9 @@ public class RateLimitedClientWrapper {
private static final int REQUEST_RATE_LIMITED_WAIT_MS = 5_000; private static final int REQUEST_RATE_LIMITED_WAIT_MS = 5_000;
private static final Map<Predicate<String>, RateLimiter> FORCED_RATE_LIMITERS = Map.ofEntries( private static final Map<Predicate<String>, RateLimiter> FORCED_RATE_LIMITERS = Map.ofEntries(
Map.entry(host -> host.endsWith("youtube.com"), Map.entry(host -> host.endsWith("youtube.com"),
RateLimiter.create(1.6, Duration.ofSeconds(1), 3.0)) RateLimiter.create(1.6, Duration.ofSeconds(1))),
Map.entry(host -> host.endsWith("bandcamp.com"),
RateLimiter.create(2.5, Duration.ofSeconds(1)))
); );
private final OkHttpClient client; private final OkHttpClient client;
@ -40,7 +42,7 @@ public class RateLimitedClientWrapper {
.map(Map.Entry::getValue) .map(Map.Entry::getValue)
.orElseGet(() -> .orElseGet(() ->
// Default rate limiter per domain // Default rate limiter per domain
RateLimiter.create(5, Duration.ofSeconds(5), 3.0))); RateLimiter.create(5, Duration.ofSeconds(3))));
} }
public Response executeRequestWithLimit(final Request request) throws IOException { public Response executeRequestWithLimit(final Request request) throws IOException {
@ -76,6 +78,7 @@ public class RateLimitedClientWrapper {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} }
} }
throw new IllegalStateException("Retrying/Rate-limiting for " + request.url() + "failed", cause); throw new IllegalStateException(
"Retrying/Rate-limiting for " + request.url() + " failed", cause);
} }
} }