Merge pull request #373 from wb9688/fix-yt-continuations

Support YouTube's new continuations

extractor/src/main/java/org/schabi/newpipe/extractor/Page.java

@@ -8,35 +8,45 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class Page implements Serializable {
     private final String url;
+    private final String id;
     private final List<String> ids;
     private final Map<String, String> cookies;
 
-    public Page(final String url, final List<String> ids, final Map<String, String> cookies) {
+    public Page(final String url, final String id, final List<String> ids, final Map<String, String> cookies) {
         this.url = url;
+        this.id = id;
         this.ids = ids;
         this.cookies = cookies;
     }
 
     public Page(final String url) {
-        this(url, null, null);
+        this(url, null, null, null);
+    }
+
+    public Page(final String url, final String id) {
+        this(url, id, null, null);
     }
 
     public Page(final String url, final Map<String, String> cookies) {
-        this(url, null, cookies);
+        this(url, null, null, cookies);
     }
 
     public Page(final List<String> ids) {
-        this(null, ids, null);
+        this(null, null, ids, null);
     }
 
     public Page(final List<String> ids, final Map<String, String> cookies) {
-        this(null, ids, cookies);
+        this(null, null, ids, cookies);
     }
 
     public String getUrl() {
         return url;
     }
 
+    public String getId() {
+        return id;
+    }
+
     public List<String> getIds() {
         return ids;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -64,6 +64,8 @@ public class YoutubeParsingHelper {
     private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
     private static String clientVersion;
 
+    private static String key;
+
     private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
     private static String[] youtubeMusicKeys;
 
@@ -214,39 +216,31 @@ public class YoutubeParsingHelper {
         return response.length() > 50; // ensure to have a valid response
     }
 
-    /**
+    private static void extractClientVersionAndKey() throws IOException, ExtractionException {
-     * Get the client version from a page
-     * @return
-     * @throws ParsingException
-     */
-    public static String getClientVersion() throws IOException, ExtractionException {
-        if (!isNullOrEmpty(clientVersion)) return clientVersion;
-        if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
-
         final String url = "https://www.youtube.com/results?search_query=test";
         final String html = getDownloader().get(url).responseBody();
-        JsonObject initialData = getInitialData(html);
+        final JsonObject initialData = getInitialData(html);
-        JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
+        final JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
         String shortClientVersion = null;
 
         // try to get version from initial data first
-        for (Object service : serviceTrackingParams) {
+        for (final Object service : serviceTrackingParams) {
-            JsonObject s = (JsonObject) service;
+            final JsonObject s = (JsonObject) service;
             if (s.getString("service").equals("CSI")) {
-                JsonArray params = s.getArray("params");
+                final JsonArray params = s.getArray("params");
-                for (Object param : params) {
+                for (final Object param : params) {
-                    JsonObject p = (JsonObject) param;
+                    final JsonObject p = (JsonObject) param;
-                    String key = p.getString("key");
+                    final String key = p.getString("key");
                     if (key != null && key.equals("cver")) {
-                        return clientVersion = p.getString("value");
+                        clientVersion = p.getString("value");
                     }
                 }
             } else if (s.getString("service").equals("ECATCHER")) {
                 // fallback to get a shortened client version which does not contain the last two digits
-                JsonArray params = s.getArray("params");
+                final JsonArray params = s.getArray("params");
-                for (Object param : params) {
+                for (final Object param : params) {
-                    JsonObject p = (JsonObject) param;
+                    final JsonObject p = (JsonObject) param;
-                    String key = p.getString("key");
+                    final String key = p.getString("key");
                     if (key != null && key.equals("client.version")) {
                         shortClientVersion = p.getString("value");
                     }
@@ -255,26 +249,55 @@ public class YoutubeParsingHelper {
         }
 
         String contextClientVersion;
-        String[] patterns = {
+        final String[] patterns = {
                 "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
                 "innertube_context_client_version\":\"([0-9\\.]+?)\"",
                 "client.version=([0-9\\.]+)"
         };
-        for (String pattern : patterns) {
+        for (final String pattern : patterns) {
             try {
                 contextClientVersion = Parser.matchGroup1(pattern, html);
                 if (!isNullOrEmpty(contextClientVersion)) {
-                    return clientVersion = contextClientVersion;
+                    clientVersion = contextClientVersion;
+                    break;
                 }
-            } catch (Exception ignored) {
+            } catch (Parser.RegexException ignored) { }
-            }
         }
 
-        if (shortClientVersion != null) {
+        if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
-            return clientVersion = shortClientVersion;
+            clientVersion = shortClientVersion;
         }
 
-        throw new ParsingException("Could not get client version");
+        try {
+            key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
+        } catch (Parser.RegexException e) {
+            try {
+                key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
+            } catch (Parser.RegexException ignored) { }
+        }
+    }
+
+    /**
+     * Get the client version
+     */
+    public static String getClientVersion() throws IOException, ExtractionException {
+        if (!isNullOrEmpty(clientVersion)) return clientVersion;
+        if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
+
+        extractClientVersionAndKey();
+        if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version");
+        return clientVersion;
+    }
+
+    /**
+     * Get the key
+     */
+    public static String getKey() throws IOException, ExtractionException {
+        if (!isNullOrEmpty(key)) return key;
+
+        extractClientVersionAndKey();
+        if (isNullOrEmpty(key)) throw new ParsingException("Could not extract key");
+        return key;
     }
 
     public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -2,6 +2,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import com.grack.nanojson.JsonWriter;
+
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -14,11 +18,19 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
+import javax.annotation.Nonnull;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /*
@@ -96,7 +108,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
 
     @Nonnull
     @Override
-    public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
+    public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
         final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
 
         final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
@@ -104,12 +116,16 @@ public class YoutubeSearchExtractor extends SearchExtractor {
 
         Page nextPage = null;
 
-        for (Object section : sections) {
+        for (final Object section : sections) {
-            final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
+            if (((JsonObject) section).has("itemSectionRenderer")) {
+                final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
 
-            collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
+                collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
 
-            nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
+                nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
+            } else if (((JsonObject) section).has("continuationItemRenderer")) {
+                nextPage = getNewNextPageFrom(((JsonObject) section).getObject("continuationItemRenderer"));
+            }
         }
 
         return new InfoItemsPage<>(collector, nextPage);
@@ -122,15 +138,58 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         }
 
         final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
-        final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
 
-        final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
+        if (page.getId() == null) {
-                .getObject("continuationContents").getObject("itemSectionContinuation");
+            final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
 
-        collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
+            final JsonObject itemSectionContinuation = ajaxJson.getObject(1).getObject("response")
-        final JsonArray continuations = itemSectionRenderer.getArray("continuations");
+                    .getObject("continuationContents").getObject("itemSectionContinuation");
 
-        return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
+            collectStreamsFrom(collector, itemSectionContinuation.getArray("contents"));
+            final JsonArray continuations = itemSectionContinuation.getArray("continuations");
+
+            return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
+        } else {
+            // @formatter:off
+            final byte[] json = JsonWriter.string()
+                .object()
+                    .object("context")
+                        .object("client")
+                            .value("hl", "en")
+                            .value("gl", getExtractorContentCountry().getCountryCode())
+                            .value("clientName", "WEB")
+                            .value("clientVersion", getClientVersion())
+                            .value("utcOffsetMinutes", 0)
+                        .end()
+                        .object("request").end()
+                        .object("user").end()
+                    .end()
+                    .value("continuation", page.getId())
+                .end().done().getBytes("UTF-8");
+            // @formatter:on
+
+            final Map<String, List<String>> headers = new HashMap<>();
+            headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
+            headers.put("Referer", Collections.singletonList(this.getUrl()));
+            headers.put("Content-Type", Collections.singletonList("application/json"));
+
+            final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json));
+
+            final JsonObject ajaxJson;
+            try {
+                ajaxJson = JsonParser.object().from(responseBody);
+            } catch (JsonParserException e) {
+                throw new ParsingException("Could not parse JSON", e);
+            }
+
+            final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
+                    .getObject(0).getObject("appendContinuationItemsAction").getArray("continuationItems");
+
+            final JsonArray contents = continuationItems.getObject(0).getObject("itemSectionRenderer").getArray("contents");
+            collectStreamsFrom(collector, contents);
+
+            return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1).getObject("continuationItemRenderer")));
+        }
     }
 
     private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException {
@@ -162,4 +221,17 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
                 + "&itct=" + clickTrackingParams);
     }
+
+    private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException {
+        if (isNullOrEmpty(continuationItemRenderer)) {
+            return null;
+        }
+
+        final String token = continuationItemRenderer.getObject("continuationEndpoint")
+                .getObject("continuationCommand").getString("token");
+
+        final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
+
+        return new Page(url, token);
+    }
 }