Implement pooling PoTokens support.

build.gradle

@@ -16,7 +16,7 @@ dependencies {
     implementation 'it.unimi.dsi:fastutil-core:8.5.13'
     implementation 'commons-codec:commons-codec:1.17.0'
     implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1'
-    implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:a64e202bb498032e817a702145263590829f3c1d'
+    implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:e45fa4d37f809a07dd5bdf6f920adabe0077704f'
     implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7'
     implementation 'com.fasterxml.jackson.core:jackson-core:2.17.2'
     implementation 'com.fasterxml.jackson.core:jackson-annotations:2.17.2'

config.properties

@@ -73,6 +73,9 @@ MATRIX_SERVER:https://matrix-client.matrix.org
 # Geo Restriction Checker for federated bypassing of Geo Restrictions
 #GEO_RESTRICTION_CHECKER_URL:INSERT_HERE
 
+# BG Helper URL for supplying PoTokens
+#BG_HELPER_URL:INSERT_HERE
+
 # S3 Configuration Data (compatible with any provider that offers an S3 compatible API)
 #S3_ENDPOINT:INSERT_HERE
 #S3_ACCESS_KEY:INSERT_HERE

src/main/java/me/kavin/piped/Main.java

@@ -13,6 +13,7 @@ import me.kavin.piped.utils.obj.db.PlaylistVideo;
 import me.kavin.piped.utils.obj.db.PubSub;
 import me.kavin.piped.utils.obj.db.Video;
 import okhttp3.OkHttpClient;
+import org.apache.commons.lang3.StringUtils;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.hibernate.Session;
 import org.hibernate.StatelessSession;
@@ -21,6 +22,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfo;
 import rocks.kavin.reqwest4j.ReqwestUtils;
 
@@ -44,6 +46,8 @@ public class Main {
         ReqwestUtils.init(REQWEST_PROXY, REQWEST_PROXY_USER, REQWEST_PROXY_PASS);
 
         NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT);
+        if (!StringUtils.isEmpty(Constants.BG_HELPER_URL))
+            YoutubeStreamExtractor.setPoTokenProvider(new BgPoTokenProvider(Constants.BG_HELPER_URL));
         YoutubeParsingHelper.setConsentAccepted(CONSENT_COOKIE);
 
         // Warm up the extractor
@@ -82,7 +86,7 @@ public class Main {
             System.exit(1);
         }
 
-        Multithreading.runAsync(() ->  Thread.ofVirtual().start(new SyncRunner(
+        Multithreading.runAsync(() -> Thread.ofVirtual().start(new SyncRunner(
                 new OkHttpClient.Builder().readTimeout(60, TimeUnit.SECONDS).build(),
                 MATRIX_SERVER,
                 MatrixHelper.MATRIX_TOKEN)

src/main/java/me/kavin/piped/consts/Constants.java

@@ -28,7 +28,7 @@ import java.util.Properties;
 
 public class Constants {
 
-    public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0";
+    public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:128.0) Gecko/20100101 Firefox/128.0";
 
     public static final int PORT;
     public static final String HTTP_WORKERS;
@@ -100,6 +100,8 @@ public class Constants {
 
     public static final String GEO_RESTRICTION_CHECKER_URL;
 
+    public static final String BG_HELPER_URL;
+
     public static String YOUTUBE_COUNTRY;
 
     public static final String VERSION;
@@ -170,6 +172,7 @@ public class Constants {
             MATRIX_SERVER = getProperty(prop, "MATRIX_SERVER", "https://matrix-client.matrix.org");
             MATRIX_TOKEN = getProperty(prop, "MATRIX_TOKEN");
             GEO_RESTRICTION_CHECKER_URL = getProperty(prop, "GEO_RESTRICTION_CHECKER_URL");
+            BG_HELPER_URL = getProperty(prop, "BG_HELPER_URL");
             prop.forEach((_key, _value) -> {
                 String key = String.valueOf(_key), value = String.valueOf(_value);
                 if (key.startsWith("hibernate"))

src/main/java/me/kavin/piped/utils/BgPoTokenProvider.java

@@ -0,0 +1,93 @@
+package me.kavin.piped.utils;
+
+import lombok.RequiredArgsConstructor;
+import org.jetbrains.annotations.Nullable;
+import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider;
+import org.schabi.newpipe.extractor.services.youtube.PoTokenResult;
+import rocks.kavin.reqwest4j.ReqwestUtils;
+
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.*;
+import java.util.regex.Pattern;
+
+import static me.kavin.piped.consts.Constants.mapper;
+
+@RequiredArgsConstructor
+public class BgPoTokenProvider implements PoTokenProvider {
+
+    private final String bgHelperUrl;
+
+    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+
+    private String getWebVisitorData() throws Exception {
+        var html = RequestUtils.sendGet("https://www.youtube.com").get();
+        var matcher = Pattern.compile("visitorData\":\"([\\w%-]+)\"").matcher(html);
+
+        if (matcher.find()) {
+            return matcher.group(1);
+        }
+
+        throw new RuntimeException("Failed to get visitor data");
+    }
+
+    private final Queue<PoTokenResult> validPoTokens = new ConcurrentLinkedQueue<>();
+
+    private PoTokenResult getPoTokenPooled() throws Exception {
+        PoTokenResult poToken = validPoTokens.poll();
+
+        if (poToken == null) {
+            poToken = createWebClientPoToken();
+        }
+
+        // if still null, return null
+        if (poToken == null) {
+            return null;
+        }
+
+        // timer to insert back into queue after 10 + random seconds
+        int delay = 10_000 + ThreadLocalRandom.current().nextInt(5000);
+        PoTokenResult finalPoToken = poToken;
+        scheduler.schedule(() -> validPoTokens.offer(finalPoToken), delay, TimeUnit.MILLISECONDS);
+
+        return poToken;
+    }
+
+    private PoTokenResult createWebClientPoToken() throws Exception {
+        String visitorDate = getWebVisitorData();
+
+        String poToken = ReqwestUtils.fetch(bgHelperUrl + "/generate", "POST", mapper.writeValueAsBytes(mapper.createObjectNode().put(
+                "visitorData", visitorDate
+        )), Map.of(
+                "Content-Type", "application/json"
+        )).thenApply(response -> {
+            try {
+                return mapper.readTree(response.body()).get("poToken").asText();
+            } catch (Exception e) {
+                return null;
+            }
+        }).join();
+
+        if (poToken != null) {
+            return new PoTokenResult(visitorDate, poToken);
+        }
+
+        return null;
+    }
+
+    @Override
+    public @Nullable PoTokenResult getWebClientPoToken() {
+        try {
+            return getPoTokenPooled();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    @Override
+    public @Nullable PoTokenResult getAndroidClientPoToken() {
+        // TODO: allow setting from config maybe?
+        return null;
+    }
+}

src/main/java/me/kavin/piped/utils/RequestUtils.java

@@ -1,6 +1,7 @@
 package me.kavin.piped.utils;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import me.kavin.piped.consts.Constants;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import rocks.kavin.reqwest4j.ReqwestUtils;
@@ -15,11 +16,15 @@ import static me.kavin.piped.consts.Constants.mapper;
 public class RequestUtils {
 
     public static CompletableFuture<Response> sendGetRaw(String url) throws Exception {
-        return ReqwestUtils.fetch(url, "GET", null, Map.of());
+        return ReqwestUtils.fetch(url, "GET", null, Map.of(
+                "User-Agent", Constants.USER_AGENT
+        ));
     }
 
     public static CompletableFuture<String> sendGet(String url) throws Exception {
-        return ReqwestUtils.fetch(url, "GET", null, Map.of())
+        return ReqwestUtils.fetch(url, "GET", null, Map.of(
+                        "User-Agent", Constants.USER_AGENT
+                ))
                 .thenApply(Response::body)
                 .thenApplyAsync(String::new);
     }