kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2024-12-13 22:00:32 +05:30
Code Issues Projects Releases Wiki Activity

added http post method in downloader, formatting

Browse Source
This commit is contained in:
Ritvik Saraf 2018-09-02 05:06:26 +05:30
parent 8b8779b176
commit 95575756ee
8 changed files with 442 additions and 464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java
View File

@ -7,8 +7,6 @@ public class DownloadResponse {
private final String responseBody;
private final Map<String, List<String>> responseHeaders;
public DownloadResponse(String responseBody, Map<String, List<String>> headers) {
super();
this.responseBody = responseBody;

7
extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java
View File

@ -61,8 +61,11 @@ public interface Downloader {
*/
String download(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse downloadWithHeaders(String siteUrl, Map<String, List<String>> requestHeaders)
DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders)
throws IOException, ReCaptchaException;
DownloadResponse downloadWithHeaders(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders)
throws IOException, ReCaptchaException;
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java
View File

@ -4,19 +4,11 @@ import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public interface CommentsInfoItemExtractor extends InfoItemExtractor {
String getCommentId() throws ParsingException;
String getCommentText() throws ParsingException;
String getAuthorName() throws ParsingException;
String getAuthorThumbnail() throws ParsingException;
String getAuthorEndpoint() throws ParsingException;
String getPublishedTime() throws ParsingException;
Integer getLikeCount() throws ParsingException;
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java
View File

@ -7,26 +7,6 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/*
* Created by Christian Schabesberger on 28.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* CommentsInfoItemsCollector.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoItem, CommentsInfoItemExtractor> {
public CommentsInfoItemsCollector(int serviceId) {

41
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -1,20 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.net.ssl.HttpsURLConnection;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@ -22,6 +19,7 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import com.fasterxml.jackson.databind.JsonNode;
@ -195,7 +193,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
DownloadResponse response = downloader.downloadWithHeaders(getUrl());
DownloadResponse response = downloader.get(getUrl());
String responseBody = response.getResponseBody();
cookies = response.getResponseHeaders().get("Set-Cookie");
sessionToken = findValue(responseBody, "XSRF_TOKEN");
@ -208,35 +206,22 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
return null;
}
private String makeAjaxRequest(String siteUrl) throws IOException {
private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
StringBuilder postData = new StringBuilder();
postData.append(URLEncoder.encode("session_token", "UTF-8"));
postData.append('=');
postData.append(URLEncoder.encode(sessionToken, "UTF-8"));
byte[] postDataBytes = postData.toString().getBytes("UTF-8");
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
con.setRequestMethod("POST");
con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
con.setRequestProperty("Accept", "*/*");
con.setRequestProperty("User-Agent", USER_AGENT);
con.setRequestProperty("X-YouTube-Client-Version", "2.20180815");
con.setRequestProperty("X-YouTube-Client-Name", "1");
// set cookies
cookies.stream().forEach(c -> con.addRequestProperty("Cookie", c));
con.setDoOutput(true);
con.getOutputStream().write(postDataBytes);
Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Content-Type", Arrays.asList("application/x-www-form-urlencoded"));
requestHeaders.put("Accept", Arrays.asList("*/*"));
requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", Arrays.asList("2.20180815"));
requestHeaders.put("X-YouTube-Client-Name", Arrays.asList("1"));
requestHeaders.put("Cookie", cookies);
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), "UTF-8"));
StringBuilder sb = new StringBuilder();
String inputLine;
while ((inputLine = in.readLine()) != null) {
sb.append(inputLine);
}
return sb.toString();
return NewPipe.getDownloader().post(siteUrl, postData.toString(), requestHeaders).getResponseBody();
}
private String getDataString(Map<String, String> params) throws UnsupportedEncodingException {

40
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
View File

@ -1,16 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Parser;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
@ -18,25 +7,16 @@ import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.util.List;
/*
* Created by Christian Schabesberger on 25.07.16.
*
* Copyright (C) Christian Schabesberger 2018 <chrźis.schabesberger@mailbox.org>
* YoutubeChannelLinkHandlerFactory.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {

62
extractor/src/test/java/org/schabi/newpipe/Downloader.java
View File

@ -106,14 +106,9 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
BufferedReader in = null;
try {
con.setConnectTimeout(30 * 1000);// 30s
con.setReadTimeout(30 * 1000);// 30s
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
if (getCookies().length() > 0) {
con.addRequestProperty("Cookie", getCookies());
}
con.setRequestMethod("GET");
setDefaults(con);
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
@ -121,9 +116,11 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch (UnknownHostException uhe) {// thrown when there's no internet connection
} catch (UnknownHostException uhe) {// thrown when there's no internet
// connection
throw new IOException("unknown host or no network", uhe);
// Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
// Toast.makeText(getActivity(), uhe.getMessage(),
// Toast.LENGTH_LONG).show();
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
@ -143,6 +140,22 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return response.toString();
}
private static void setDefaults(HttpsURLConnection con) {
con.setConnectTimeout(30 * 1000);// 30s
con.setReadTimeout(30 * 1000);// 30s
// set default user agent
if (null == con.getRequestProperty("User-Agent")) {
con.setRequestProperty("User-Agent", USER_AGENT);
}
// add default cookies
if (getCookies().length() > 0) {
con.addRequestProperty("Cookie", getCookies());
}
}
/**
* Download (via HTTP) the text file located at the supplied URL, and return its
* contents. Primarily intended for downloading web pages.
@ -158,7 +171,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
}
@Override
public DownloadResponse downloadWithHeaders(String siteUrl, Map<String, List<String>> requestHeaders)
public DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
@ -170,10 +183,37 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
}
@Override
public DownloadResponse downloadWithHeaders(String siteUrl) throws IOException, ReCaptchaException {
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
String responseBody = dl(con);
return new DownloadResponse(responseBody, con.getHeaderFields());
}
@Override
public DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
con.setRequestMethod("POST");
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
}
// set fields to default if not set already
setDefaults(con);
byte[] postDataBytes = requestBody.toString().getBytes("UTF-8");
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
con.setDoOutput(true);
con.getOutputStream().write(postDataBytes);
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
StringBuilder sb = new StringBuilder();
String inputLine;
while ((inputLine = in.readLine()) != null) {
sb.append(inputLine);
}
return new DownloadResponse(sb.toString(), con.getHeaderFields());
}
}