From 6b620914b64891740140a5ca0d3a03f48c568c42 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Thu, 27 Sep 2018 02:04:12 +0530 Subject: [PATCH] moved cookie logic outside --- .../newpipe/extractor/DownloadRequest.java | 44 +++++++++++++++++++ .../newpipe/extractor/DownloadResponse.java | 13 ++++++ .../schabi/newpipe/extractor/Downloader.java | 4 +- .../extractors/YoutubeCommentsExtractor.java | 8 ++-- .../java/org/schabi/newpipe/Downloader.java | 22 +++++----- 5 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/DownloadRequest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadRequest.java b/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadRequest.java new file mode 100644 index 000000000..32c8a67fa --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadRequest.java @@ -0,0 +1,44 @@ +package org.schabi.newpipe.extractor; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class DownloadRequest { + + private final String requestBody; + private final Map> requestHeaders; + public static final DownloadRequest emptyRequest = new DownloadRequest(null, null); + + public DownloadRequest(String requestBody, Map> headers) { + super(); + this.requestBody = requestBody; + if(null != headers) { + this.requestHeaders = headers; + }else { + this.requestHeaders = Collections.emptyMap(); + } + } + + public String getRequestBody() { + return requestBody; + } + + public Map> getRequestHeaders() { + return requestHeaders; + } + + public void setRequestCookies(List cookies){ + requestHeaders.put("Cookie", cookies); + } + + public List getRequestCookies(){ + if(null == requestHeaders) return Collections.emptyList(); + List cookies = requestHeaders.get("Cookie"); + if(null == cookies) + return Collections.emptyList(); + else + return cookies; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java b/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java index 64fc2ade1..2165002a8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java @@ -1,8 +1,11 @@ package org.schabi.newpipe.extractor; +import java.util.Collections; import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; + public class DownloadResponse { private final String responseBody; private final Map> responseHeaders; @@ -20,5 +23,15 @@ public class DownloadResponse { public Map> getResponseHeaders() { return responseHeaders; } + + @Nonnull + public List getResponseCookies(){ + if(null == responseHeaders) return Collections.emptyList(); + List cookies = responseHeaders.get("Set-Cookie"); + if(null == cookies) + return Collections.emptyList(); + else + return cookies; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java index f0b6692d4..7a9db6aed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java @@ -61,11 +61,11 @@ public interface Downloader { */ String download(String siteUrl) throws IOException, ReCaptchaException; - DownloadResponse get(String siteUrl, Map> requestHeaders) + DownloadResponse get(String siteUrl, DownloadRequest request) throws IOException, ReCaptchaException; DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException; - DownloadResponse post(String siteUrl, String requestBody, Map> requestHeaders) + DownloadResponse post(String siteUrl, DownloadRequest request) throws IOException, ReCaptchaException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 672207e36..9aa2fba3c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -8,6 +8,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.schabi.newpipe.extractor.DownloadRequest; import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; @@ -145,7 +146,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { DownloadResponse response = downloader.get(getUrl()); String responseBody = response.getResponseBody(); - cookies = response.getResponseHeaders().get("Set-Cookie"); + cookies = response.getResponseCookies(); sessionToken = findValue(responseBody, "XSRF_TOKEN"); String commentsToken = findValue(responseBody, "COMMENTS_TOKEN"); initPage = getPage(getNextPageUrl(commentsToken)); @@ -168,9 +169,10 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT)); requestHeaders.put("X-YouTube-Client-Version", Arrays.asList("2.20180815")); requestHeaders.put("X-YouTube-Client-Name", Arrays.asList("1")); - requestHeaders.put("Cookie", cookies); + DownloadRequest request = new DownloadRequest(postData, requestHeaders); + request.setRequestCookies(cookies); - return NewPipe.getDownloader().post(siteUrl, postData, requestHeaders).getResponseBody(); + return NewPipe.getDownloader().post(siteUrl, request).getResponseBody(); } private String getDataString(Map params) throws UnsupportedEncodingException { diff --git a/extractor/src/test/java/org/schabi/newpipe/Downloader.java b/extractor/src/test/java/org/schabi/newpipe/Downloader.java index 4745a5d1a..77940ba92 100644 --- a/extractor/src/test/java/org/schabi/newpipe/Downloader.java +++ b/extractor/src/test/java/org/schabi/newpipe/Downloader.java @@ -12,6 +12,7 @@ import java.util.Map; import javax.net.ssl.HttpsURLConnection; +import org.schabi.newpipe.extractor.DownloadRequest; import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -172,11 +173,11 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { } @Override - public DownloadResponse get(String siteUrl, Map> requestHeaders) + public DownloadResponse get(String siteUrl, DownloadRequest request) throws IOException, ReCaptchaException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); - for (Map.Entry> pair : requestHeaders.entrySet()) { + for (Map.Entry> pair : request.getRequestHeaders().entrySet()) { for(String value: pair.getValue()) { con.addRequestProperty(pair.getKey(), value); } @@ -187,16 +188,16 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { @Override public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException { - return get(siteUrl, Collections.EMPTY_MAP); + return get(siteUrl, DownloadRequest.emptyRequest); } @Override - public DownloadResponse post(String siteUrl, String requestBody, Map> requestHeaders) + public DownloadResponse post(String siteUrl, DownloadRequest request) throws IOException, ReCaptchaException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); con.setRequestMethod("POST"); - for (Map.Entry> pair : requestHeaders.entrySet()) { + for (Map.Entry> pair : request.getRequestHeaders().entrySet()) { for(String value: pair.getValue()) { con.addRequestProperty(pair.getKey(), value); } @@ -204,11 +205,12 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { // set fields to default if not set already setDefaults(con); - byte[] postDataBytes = requestBody.toString().getBytes("UTF-8"); - con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length)); - - con.setDoOutput(true); - con.getOutputStream().write(postDataBytes); + if(null != request.getRequestBody()) { + byte[] postDataBytes = request.getRequestBody().getBytes("UTF-8"); + con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length)); + con.setDoOutput(true); + con.getOutputStream().write(postDataBytes); + } StringBuilder sb = new StringBuilder(); try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) {