kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2025-01-06 01:20:32 +05:30
Code Issues Projects Releases Wiki Activity

Extract some code to getValidResponseBody()

Browse Source
This commit is contained in:
wb9688 2020-04-01 16:01:21 +02:00
parent 2af610e3e7
commit 8a9e137385
2 changed files with 21 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -9,8 +9,6 @@ import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -31,6 +29,7 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
@ -112,23 +111,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json")); headers.put("Content-Type", Collections.singletonList("application/json"));
final Response response = getDownloader().post(url, headers, json); final String responseBody = getValidResponseBody(getDownloader().post(url, headers, json));
if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Not found" +
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
}
final String responseBody = response.responseBody();
if (responseBody.length() < 50) { // ensure to have a valid response
throw new ParsingException("JSON response is too short");
}
final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")");
}
try { try {
initialData = JsonParser.object().from(responseBody); initialData = JsonParser.object().from(responseBody);
@ -232,23 +215,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json")); headers.put("Content-Type", Collections.singletonList("application/json"));
final Response response = getDownloader().post(pageUrl, headers, json); final String responseBody = getValidResponseBody(getDownloader().post(pageUrl, headers, json));
if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Not found" +
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
}
final String responseBody = response.responseBody();
if (responseBody.length() < 50) { // ensure to have a valid response
throw new ParsingException("JSON response is too short");
}
final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")");
}
final JsonObject ajaxJson; final JsonObject ajaxJson;
try { try {

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.text.ParseException; import java.text.ParseException;
@ -427,12 +428,8 @@ public class YoutubeParsingHelper {
return thumbnailUrl; return thumbnailUrl;
} }
public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException { public static String getValidResponseBody(final Response response)
Map<String, List<String>> headers = new HashMap<>(); throws ParsingException, MalformedURLException {
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
if (response.responseCode() == 404) { if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Not found" + throw new ContentNotAvailableException("Not found" +
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); " (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
@ -453,11 +450,24 @@ public class YoutubeParsingHelper {
} }
final String responseContentType = response.getHeader("Content-Type"); final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { if (responseContentType != null
&& responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" + throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")"); " (latest url was: \"" + response.latestUrl() + "\")");
} }
return responseBody;
}
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
final String responseBody = getValidResponseBody(response);
try { try {
return JsonParser.array().from(responseBody); return JsonParser.array().from(responseBody);
} catch (JsonParserException e) { } catch (JsonParserException e) {
@ -469,6 +479,7 @@ public class YoutubeParsingHelper {
* Shared alert detection function, multiple endpoints return the error similarly structured. * Shared alert detection function, multiple endpoints return the error similarly structured.
* <p> * <p>
* Will check if the object has an alert of the type "ERROR". * Will check if the object has an alert of the type "ERROR".
* </p>
* *
* @param initialData the object which will be checked if an alert is present * @param initialData the object which will be checked if an alert is present
* @throws ContentNotAvailableException if an alert is detected * @throws ContentNotAvailableException if an alert is detected