Merge pull request #599 from XiangRongLin/post_body

Add body field to Page and use it

extractor/src/main/java/org/schabi/newpipe/extractor/Page.java

@@ -4,6 +4,8 @@ import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
 
+import javax.annotation.Nullable;
+
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class Page implements Serializable {
@@ -12,31 +14,40 @@ public class Page implements Serializable {
     private final List<String> ids;
     private final Map<String, String> cookies;
 
-    public Page(final String url, final String id, final List<String> ids, final Map<String, String> cookies) {
+    @Nullable
+    private final byte[] body;
+
+    public Page(final String url, final String id, final List<String> ids,
+                final Map<String, String> cookies, @Nullable final byte[] body) {
         this.url = url;
         this.id = id;
         this.ids = ids;
         this.cookies = cookies;
+        this.body = body;
     }
 
     public Page(final String url) {
-        this(url, null, null, null);
+        this(url, null, null, null, null);
     }
 
     public Page(final String url, final String id) {
-        this(url, id, null, null);
+        this(url, id, null, null, null);
+    }
+
+    public Page(final String url, final byte[] body) {
+        this(url, null, null, null, body);
     }
 
     public Page(final String url, final Map<String, String> cookies) {
-        this(url, null, null, cookies);
+        this(url, null, null, cookies, null);
     }
 
     public Page(final List<String> ids) {
-        this(null, null, ids, null);
+        this(null, null, ids, null, null);
     }
 
     public Page(final List<String> ids, final Map<String, String> cookies) {
-        this(null, null, ids, cookies);
+        this(null, null, ids, cookies, null);
     }
 
     public String getUrl() {
@@ -59,4 +70,9 @@ public class Page implements Serializable {
         return page != null && (!isNullOrEmpty(page.getUrl())
                 || !isNullOrEmpty(page.getIds()));
     }
+
+    @Nullable
+    public byte[] getBody() {
+        return body;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
 import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonBuilder;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
@@ -676,6 +677,19 @@ public class YoutubeParsingHelper {
         return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
     }
 
+    public static JsonBuilder<JsonObject> prepareJsonBuilder()
+            throws IOException, ExtractionException {
+        // @formatter:off
+        return JsonObject.builder()
+                .object("context")
+                    .object("client")
+                        .value("clientName", "1")
+                        .value("clientVersion", getClientVersion())
+                    .end()
+                .end();
+        // @formatter:on
+    }
+
     /**
      * Shared alert detection function, multiple endpoints return the error similarly structured.
      * <p>

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -21,15 +21,16 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 
+import javax.annotation.Nonnull;
+
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -264,23 +265,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         // as they don't deliver enough information on their own (the channel name, for example).
         fetchPage();
 
-        // @formatter:off
-        byte[] json = JsonWriter.string()
-                .object()
-                .object("context")
-                .object("client")
-                .value("clientName", "1")
-                .value("clientVersion", getClientVersion())
-                .end()
-                .end()
-                .value("continuation", page.getId())
-                .end()
-                .done()
-                .getBytes(UTF_8);
-        // @formatter:on
-
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
+        final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
+                getExtractorLocalization());
 
         final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
 
@@ -300,8 +287,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
         final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
         final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
+
+        final byte[] body = JsonWriter.string(prepareJsonBuilder()
+                .value("continuation", continuation)
+                .done())
+                .getBytes(UTF_8);
+
         return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
-                continuation);
+                body);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
@@ -28,13 +29,12 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
-import static org.schabi.newpipe.extractor.utils.JsonUtils.toJsonObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -217,25 +217,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
             throw new IllegalArgumentException("Page doesn't contain an URL");
         }
 
-        // @formatter:off
-        byte[] json = JsonWriter.string()
-                .object()
-                    .object("context")
-                        .object("client")
-                            .value("clientName", "1")
-                            .value("clientVersion", getClientVersion())
-                        .end()
-                    .end()
-                    .value("continuation", page.getId())
-                .end()
-                .done()
-                .getBytes(UTF_8);
-        // @formatter:on
-
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
 
-        final JsonObject ajaxJson = toJsonObject(getValidJsonResponseBody(response));
+        final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
+                getExtractorLocalization());
+        final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
 
         final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
                 .getObject(0)
@@ -259,9 +245,15 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                     .getObject("continuationEndpoint")
                     .getObject("continuationCommand")
                     .getString("token");
+
+            final byte[] body = JsonWriter.string(prepareJsonBuilder()
+                    .value("continuation", continuation)
+                    .done())
+                    .getBytes(UTF_8);
+
             return new Page(
                     "https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
-                    continuation);
+                    body);
         } else {
             return null;
         }