Extract YouTube's key

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -64,6 +64,8 @@ public class YoutubeParsingHelper {
     private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
     private static String clientVersion;
 
+    private static String key;
+
     private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
     private static String[] youtubeMusicKeys;
 
@@ -214,39 +216,31 @@ public class YoutubeParsingHelper {
         return response.length() > 50; // ensure to have a valid response
     }
 
-    /**
+    private static void getClientVersionAndKey() throws IOException, ExtractionException {
-     * Get the client version from a page
-     * @return
-     * @throws ParsingException
-     */
-    public static String getClientVersion() throws IOException, ExtractionException {
-        if (!isNullOrEmpty(clientVersion)) return clientVersion;
-        if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
-
         final String url = "https://www.youtube.com/results?search_query=test";
         final String html = getDownloader().get(url).responseBody();
-        JsonObject initialData = getInitialData(html);
+        final JsonObject initialData = getInitialData(html);
-        JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
+        final JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
         String shortClientVersion = null;
 
         // try to get version from initial data first
-        for (Object service : serviceTrackingParams) {
+        for (final Object service : serviceTrackingParams) {
-            JsonObject s = (JsonObject) service;
+            final JsonObject s = (JsonObject) service;
             if (s.getString("service").equals("CSI")) {
-                JsonArray params = s.getArray("params");
+                final JsonArray params = s.getArray("params");
-                for (Object param : params) {
+                for (final Object param : params) {
-                    JsonObject p = (JsonObject) param;
+                    final JsonObject p = (JsonObject) param;
-                    String key = p.getString("key");
+                    final String key = p.getString("key");
                     if (key != null && key.equals("cver")) {
-                        return clientVersion = p.getString("value");
+                        clientVersion = p.getString("value");
                     }
                 }
             } else if (s.getString("service").equals("ECATCHER")) {
                 // fallback to get a shortened client version which does not contain the last two digits
-                JsonArray params = s.getArray("params");
+                final JsonArray params = s.getArray("params");
-                for (Object param : params) {
+                for (final Object param : params) {
-                    JsonObject p = (JsonObject) param;
+                    final JsonObject p = (JsonObject) param;
-                    String key = p.getString("key");
+                    final String key = p.getString("key");
                     if (key != null && key.equals("client.version")) {
                         shortClientVersion = p.getString("value");
                     }
@@ -255,26 +249,54 @@ public class YoutubeParsingHelper {
         }
 
         String contextClientVersion;
-        String[] patterns = {
+        final String[] patterns = {
                 "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
                 "innertube_context_client_version\":\"([0-9\\.]+?)\"",
                 "client.version=([0-9\\.]+)"
         };
-        for (String pattern : patterns) {
+        for (final String pattern : patterns) {
             try {
                 contextClientVersion = Parser.matchGroup1(pattern, html);
                 if (!isNullOrEmpty(contextClientVersion)) {
-                    return clientVersion = contextClientVersion;
+                    clientVersion = contextClientVersion;
                 }
-            } catch (Exception ignored) {
+            } catch (Parser.RegexException ignored) { }
-            }
         }
 
         if (shortClientVersion != null) {
-            return clientVersion = shortClientVersion;
+            clientVersion = shortClientVersion;
         }
 
-        throw new ParsingException("Could not get client version");
+        try {
+            key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
+        } catch (Parser.RegexException e) {
+            try {
+                key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
+            } catch (Parser.RegexException ignored) { }
+        }
+    }
+
+    /**
+     * Get the client version
+     */
+    public static String getClientVersion() throws IOException, ExtractionException {
+        if (!isNullOrEmpty(clientVersion)) return clientVersion;
+        if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
+
+        getClientVersionAndKey();
+        if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version");
+        return clientVersion;
+    }
+
+    /**
+     * Get the key
+     */
+    public static String getKey() throws IOException, ExtractionException {
+        if (!isNullOrEmpty(key)) return key;
+
+        getClientVersionAndKey();
+        if (isNullOrEmpty(key)) throw new ParsingException("Could not extract key");
+        return key;
     }
 
     public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -28,6 +28,7 @@ import javax.annotation.Nonnull;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -107,7 +108,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
 
     @Nonnull
     @Override
-    public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
+    public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
         final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
 
         final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
@@ -221,7 +222,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
                 + "&itct=" + clickTrackingParams);
     }
 
-    private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) {
+    private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException {
         if (isNullOrEmpty(continuationItemRenderer)) {
             return null;
         }
@@ -229,8 +230,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         final String token = continuationItemRenderer.getObject("continuationEndpoint")
                 .getObject("continuationCommand").getString("token");
 
-        // FIXME: Key needs to be extracted
+        final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
-        final String url = "https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
 
         return new Page(url, token);
     }