kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2025-04-28 07:50:34 +05:30
Code Issues Projects Releases Wiki Activity

Use the youtubei API for YouTube playlists

Browse Source
This commit is contained in:
TiA4f8R 2021-04-08 16:17:59 +02:00
parent c97a19d719
commit 5794eb2350
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
3 changed files with 56 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -95,7 +95,7 @@ public class YoutubeParsingHelper {
final String host = u.getHost(); final String host = u.getHost();
return host.startsWith("google.") || host.startsWith("m.google.") return host.startsWith("google.") || host.startsWith("m.google.")
|| host.startsWith("www.google."); || host.startsWith("www.google.");
} catch (MalformedURLException e) { } catch (final MalformedURLException e) {
return false; return false;
} }
} }
@ -207,10 +207,10 @@ public class YoutubeParsingHelper {
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException { public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try { try {
return OffsetDateTime.parse(textualUploadDate); return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) { } catch (final DateTimeParseException e) {
try { try {
return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC); return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC);
} catch (DateTimeParseException e1) { } catch (final DateTimeParseException e1) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1); throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1);
} }
} }
@ -277,11 +277,11 @@ public class YoutubeParsingHelper {
try { try {
final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html); final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData); return JsonParser.object().from(initialData);
} catch (Parser.RegexException e) { } catch (final Parser.RegexException e) {
final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html); final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData); return JsonParser.object().from(initialData);
} }
} catch (JsonParserException | Parser.RegexException e) { } catch (final JsonParserException | Parser.RegexException e) {
throw new ParsingException("Could not get ytInitialData", e); throw new ParsingException("Could not get ytInitialData", e);
} }
} }
@ -342,7 +342,7 @@ public class YoutubeParsingHelper {
clientVersion = contextClientVersion; clientVersion = contextClientVersion;
break; break;
} }
} catch (Parser.RegexException ignored) { } catch (final Parser.RegexException ignored) {
} }
} }
@ -352,10 +352,10 @@ public class YoutubeParsingHelper {
try { try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException e) { } catch (final Parser.RegexException e) {
try { try {
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html); key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException ignored) { } catch (final Parser.RegexException ignored) {
} }
} }
} }
@ -468,7 +468,7 @@ public class YoutubeParsingHelper {
String key; String key;
try { try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException e) { } catch (final Parser.RegexException e) {
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html); key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
} }
@ -477,10 +477,10 @@ public class YoutubeParsingHelper {
String clientVersion; String clientVersion;
try { try {
clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (Parser.RegexException e) { } catch (final Parser.RegexException e) {
try { try {
clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (Parser.RegexException ee) { } catch (final Parser.RegexException ee) {
clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html); clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
} }
} }
@ -491,7 +491,7 @@ public class YoutubeParsingHelper {
@Nullable @Nullable
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { public static String getUrlFromNavigationEndpoint(final JsonObject navigationEndpoint) throws ParsingException {
if (navigationEndpoint.has("urlEndpoint")) { if (navigationEndpoint.has("urlEndpoint")) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("https://www.youtube.com/redirect?")) { if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
@ -508,7 +508,7 @@ public class YoutubeParsingHelper {
String url; String url;
try { try {
url = URLDecoder.decode(param.split("=")[1], UTF_8); url = URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (UnsupportedEncodingException e) { } catch (final UnsupportedEncodingException e) {
return null; return null;
} }
return url; return url;
@ -662,6 +662,31 @@ public class YoutubeParsingHelper {
return response; return response;
} }
public static String extractCookieValue(final String cookieName, final Response response) {
final List<String> cookies = response.responseHeaders().get("set-cookie");
int startIndex;
String result = "";
for (final String cookie : cookies) {
startIndex = cookie.indexOf(cookieName);
if (startIndex != -1) {
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
cookie.indexOf(";", startIndex));
}
}
return result;
}
public static JsonObject getJsonPostResponse(final String endpoint,
final byte[] body,
final Localization localization)
throws IOException, ExtractionException {
final Response response = getDownloader().post("https://youtubei.googleapis.com/youtubei/v1/"
+ endpoint + "?key=" + getKey(), new HashMap<>(), body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final String url, final Localization localization) public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException { throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
@ -885,7 +910,7 @@ public class YoutubeParsingHelper {
metaInfo.addUrl(new URL(url)); metaInfo.addUrl(new URL(url));
final String description = getTextFromObject(clarificationRenderer.getObject("secondarySource")); final String description = getTextFromObject(clarificationRenderer.getObject("secondarySource"));
metaInfo.addUrlText(description == null ? url : description); metaInfo.addUrlText(description == null ? url : description);
} catch (MalformedURLException e) { } catch (final MalformedURLException e) {
throw new ParsingException("Could not get metadata info secondary URL", e); throw new ParsingException("Could not get metadata info secondary URL", e);
} }
} }

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -147,7 +147,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title"); return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title");
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get channel name", e); throw new ParsingException("Could not get channel name", e);
} }
} }
@ -159,7 +159,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getArray("thumbnails").getObject(0).getString("url"); .getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get avatar", e); throw new ParsingException("Could not get avatar", e);
} }
} }
@ -175,7 +175,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get banner", e); throw new ParsingException("Could not get banner", e);
} }
} }
@ -184,7 +184,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getFeedUrl() throws ParsingException { public String getFeedUrl() throws ParsingException {
try { try {
return YoutubeParsingHelper.getFeedUrlFrom(getId()); return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get feed url", e); throw new ParsingException("Could not get feed url", e);
} }
} }
@ -207,7 +207,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getDescription() throws ParsingException { public String getDescription() throws ParsingException {
try { try {
return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description"); return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description");
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get channel description", e); throw new ParsingException("Could not get channel description", e);
} }
} }
@ -293,7 +293,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.done()) .done())
.getBytes(UTF_8); .getBytes(UTF_8);
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), return new Page("https://youtubei.googleapis.com/youtubei/v1/browse?key=" + getKey(),
body); body);
} }
@ -314,7 +314,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
JsonObject continuation = null; JsonObject continuation = null;
for (Object object : videos) { for (final Object object : videos) {
final JsonObject video = (JsonObject) object; final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) { if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor( collector.commit(new YoutubeStreamInfoItemExtractor(
@ -344,7 +344,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getArray("tabs"); .getArray("tabs");
JsonObject videoTab = null; JsonObject videoTab = null;
for (Object tab : tabs) { for (final Object tab : tabs) {
if (((JsonObject) tab).has("tabRenderer")) { if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) { if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) {
videoTab = ((JsonObject) tab).getObject("tabRenderer"); videoTab = ((JsonObject) tab).getObject("tabRenderer");

17
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -29,7 +29,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@ -50,11 +50,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1"; final byte[] body = JsonWriter.string(prepareJsonBuilder()
.value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // show unavailable videos
.done())
.getBytes(UTF_8);
initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = getJsonPostResponse("browse", body, getExtractorLocalization());
initialData = initialAjaxJson.getObject(1).getObject("response");
YoutubeParsingHelper.defaultAlertsCheck(initialData); YoutubeParsingHelper.defaultAlertsCheck(initialData);
playlistInfo = getPlaylistInfo(); playlistInfo = getPlaylistInfo();
@ -251,9 +253,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.done()) .done())
.getBytes(UTF_8); .getBytes(UTF_8);
return new Page( return new Page("https://youtubei.googleapis.com/youtubei/v1/browse?key="
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), + getKey(), body);
body);
} else { } else {
return null; return null;
} }