added http post method in downloader, formatting

extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java

@@ -7,8 +7,6 @@ public class DownloadResponse {
     private final String responseBody;
     private final Map<String, List<String>> responseHeaders;
 
-	
-
     public DownloadResponse(String responseBody, Map<String, List<String>> headers) {
         super();
         this.responseBody = responseBody;

extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java

@@ -61,8 +61,11 @@ public interface Downloader {
      */
     String download(String siteUrl) throws IOException, ReCaptchaException;
 
-	DownloadResponse downloadWithHeaders(String siteUrl, Map<String, List<String>> requestHeaders)
+    DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders)
             throws IOException, ReCaptchaException;
 
-	DownloadResponse downloadWithHeaders(String siteUrl) throws IOException, ReCaptchaException;
+    DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException;
+
+    DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders)
+            throws IOException, ReCaptchaException;
 }

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java

@@ -4,19 +4,11 @@ import org.schabi.newpipe.extractor.InfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 public interface CommentsInfoItemExtractor extends InfoItemExtractor {
-
     String getCommentId() throws ParsingException;
-
     String getCommentText() throws ParsingException;
-
     String getAuthorName() throws ParsingException;
-
     String getAuthorThumbnail() throws ParsingException;
-
     String getAuthorEndpoint() throws ParsingException;
-
     String getPublishedTime() throws ParsingException;
-
     Integer getLikeCount() throws ParsingException;
-
 }

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java

@@ -7,26 +7,6 @@ import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.InfoItemsCollector;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
-/*
- * Created by Christian Schabesberger on 28.02.16.
- *
- * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
- * CommentsInfoItemsCollector.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoItem, CommentsInfoItemExtractor> {
 
 	public CommentsInfoItemsCollector(int serviceId) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -1,20 +1,17 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.UnsupportedEncodingException;
-import java.net.URL;
 import java.net.URLEncoder;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
-import javax.net.ssl.HttpsURLConnection;
-
 import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@@ -22,6 +19,7 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -195,7 +193,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
 
     @Override
     public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
-		DownloadResponse response = downloader.downloadWithHeaders(getUrl());
+        DownloadResponse response = downloader.get(getUrl());
         String responseBody = response.getResponseBody();
         cookies = response.getResponseHeaders().get("Set-Cookie");
         sessionToken = findValue(responseBody, "XSRF_TOKEN");
@@ -208,35 +206,22 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
         return null;
     }
 
-	private String makeAjaxRequest(String siteUrl) throws IOException {
+    private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
 
         StringBuilder postData = new StringBuilder();
         postData.append(URLEncoder.encode("session_token", "UTF-8"));
         postData.append('=');
         postData.append(URLEncoder.encode(sessionToken, "UTF-8"));
-		byte[] postDataBytes = postData.toString().getBytes("UTF-8");
 
-		URL url = new URL(siteUrl);
+        Map<String, List<String>> requestHeaders = new HashMap<>();
-		HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
+        requestHeaders.put("Content-Type", Arrays.asList("application/x-www-form-urlencoded"));
-		con.setRequestMethod("POST");
+        requestHeaders.put("Accept", Arrays.asList("*/*"));
-		con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+        requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
-		con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
+        requestHeaders.put("X-YouTube-Client-Version", Arrays.asList("2.20180815"));
-		con.setRequestProperty("Accept", "*/*");
+        requestHeaders.put("X-YouTube-Client-Name", Arrays.asList("1"));
-		con.setRequestProperty("User-Agent", USER_AGENT);
+        requestHeaders.put("Cookie", cookies);
-		con.setRequestProperty("X-YouTube-Client-Version", "2.20180815");
-		con.setRequestProperty("X-YouTube-Client-Name", "1");
-		// set cookies
-		cookies.stream().forEach(c -> con.addRequestProperty("Cookie", c));
-		con.setDoOutput(true);
-		con.getOutputStream().write(postDataBytes);
 
-		BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), "UTF-8"));
+        return NewPipe.getDownloader().post(siteUrl, postData.toString(), requestHeaders).getResponseBody();
-		StringBuilder sb = new StringBuilder();
-		String inputLine;
-		while ((inputLine = in.readLine()) != null) {
-			sb.append(inputLine);
-		}
-		return sb.toString();
     }
 
     private String getDataString(Map<String, String> params) throws UnsupportedEncodingException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java

@@ -1,16 +1,5 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.schabi.newpipe.extractor.Downloader;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.exceptions.FoundAdException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
-import org.schabi.newpipe.extractor.utils.Parser;
-
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
@@ -18,25 +7,16 @@ import java.net.URISyntaxException;
 import java.net.URLDecoder;
 import java.util.List;
 
-/*
+import org.jsoup.Jsoup;
- * Created by Christian Schabesberger on 25.07.16.
+import org.jsoup.nodes.Document;
- *
+import org.jsoup.nodes.Element;
- * Copyright (C) Christian Schabesberger 2018 <chrźis.schabesberger@mailbox.org>
+import org.schabi.newpipe.extractor.Downloader;
- * YoutubeChannelLinkHandlerFactory.java is part of NewPipe.
+import org.schabi.newpipe.extractor.NewPipe;
- *
+import org.schabi.newpipe.extractor.exceptions.FoundAdException;
- * NewPipe is free software: you can redistribute it and/or modify
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
- * it under the terms of the GNU General Public License as published by
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
- * the Free Software Foundation, either version 3 of the License, or
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
- * (at your option) any later version.
+import org.schabi.newpipe.extractor.utils.Parser;
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
 
 public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
 

extractor/src/test/java/org/schabi/newpipe/Downloader.java

@@ -106,14 +106,9 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
         BufferedReader in = null;
 
         try {
-			con.setConnectTimeout(30 * 1000);// 30s
-			con.setReadTimeout(30 * 1000);// 30s
-			con.setRequestMethod("GET");
-			con.setRequestProperty("User-Agent", USER_AGENT);
 
-			if (getCookies().length() > 0) {
+            con.setRequestMethod("GET");
-				con.addRequestProperty("Cookie", getCookies());
+            setDefaults(con);
-			}
 
             in = new BufferedReader(new InputStreamReader(con.getInputStream()));
             String inputLine;
@@ -121,9 +116,11 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
             while ((inputLine = in.readLine()) != null) {
                 response.append(inputLine);
             }
-		} catch (UnknownHostException uhe) {// thrown when there's no internet connection
+        } catch (UnknownHostException uhe) {// thrown when there's no internet
+                                            // connection
             throw new IOException("unknown host or no network", uhe);
-			// Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
+            // Toast.makeText(getActivity(), uhe.getMessage(),
+            // Toast.LENGTH_LONG).show();
         } catch (Exception e) {
             /*
              * HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
@@ -143,6 +140,22 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
         return response.toString();
     }
 
+    private static void setDefaults(HttpsURLConnection con) {
+
+        con.setConnectTimeout(30 * 1000);// 30s
+        con.setReadTimeout(30 * 1000);// 30s
+
+        // set default user agent
+        if (null == con.getRequestProperty("User-Agent")) {
+            con.setRequestProperty("User-Agent", USER_AGENT);
+        }
+
+        // add default cookies
+        if (getCookies().length() > 0) {
+            con.addRequestProperty("Cookie", getCookies());
+        }
+    }
+
     /**
      * Download (via HTTP) the text file located at the supplied URL, and return its
      * contents. Primarily intended for downloading web pages.
@@ -158,7 +171,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
     }
 
     @Override
-	public DownloadResponse downloadWithHeaders(String siteUrl, Map<String, List<String>> requestHeaders)
+    public DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders)
             throws IOException, ReCaptchaException {
         URL url = new URL(siteUrl);
         HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
@@ -170,10 +183,37 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
     }
 
     @Override
-	public DownloadResponse downloadWithHeaders(String siteUrl) throws IOException, ReCaptchaException {
+    public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
         URL url = new URL(siteUrl);
         HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
         String responseBody = dl(con);
         return new DownloadResponse(responseBody, con.getHeaderFields());
     }
+
+    @Override
+    public DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders)
+            throws IOException, ReCaptchaException {
+        URL url = new URL(siteUrl);
+        HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
+        con.setRequestMethod("POST");
+        for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
+            pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
+        }
+        // set fields to default if not set already
+        setDefaults(con);
+
+        byte[] postDataBytes = requestBody.toString().getBytes("UTF-8");
+        con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
+
+        con.setDoOutput(true);
+        con.getOutputStream().write(postDataBytes);
+
+        BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
+        StringBuilder sb = new StringBuilder();
+        String inputLine;
+        while ((inputLine = in.readLine()) != null) {
+            sb.append(inputLine);
+        }
+        return new DownloadResponse(sb.toString(), con.getHeaderFields());
+    }
 }