From 0efb854d27deeee0f9f85d6f5537aa137aeb07c0 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sun, 2 Feb 2020 14:19:48 +0100 Subject: [PATCH 01/14] [Youtube] Implement mix extractor for auto-generated playlists. -New YoutubeMixPlaylistExtractor, that extracts from a mix (auto-generated playlist). -The url has the format of "youtube.com/watch?v=videoID&playlistID", where playlistID always starts with "RD" and usually followed by the videoID. -Change YoutubePlaylistLinkHandlerFactory to create a linkhandler with the given url if it is a mix. -Change YoutubeService to return YoutubeMixPlaylistExtractor if the url is a mix. --- .../youtube/YoutubeParsingHelper.java | 4 + .../services/youtube/YoutubeService.java | 8 +- .../YoutubeMixPlaylistExtractor.java | 196 ++++++++++++++++++ .../YoutubePlaylistLinkHandlerFactory.java | 21 ++ 4 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 6ea588341..42420a1a7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -192,6 +192,10 @@ public class YoutubeParsingHelper { } } + public static boolean isYoutubeMixId(String playlistId) { + return playlistId.startsWith("RD"); + } + public static JsonObject getInitialData(String html) throws ParsingException { try { try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 519672141..7d7a83eba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -110,7 +110,11 @@ public class YoutubeService extends StreamingService { @Override public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { - return new YoutubePlaylistExtractor(this, linkHandler); + if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { + return new YoutubeMixPlaylistExtractor(this, linkHandler); + } else { + return new YoutubePlaylistExtractor(this, linkHandler); + } } @Override @@ -140,7 +144,7 @@ public class YoutubeService extends StreamingService { public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) - throws ExtractionException { + throws ExtractionException { return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java new file mode 100644 index 000000000..69b3fb987 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -0,0 +1,196 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import java.io.IOException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + + private Document doc; + + public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String url = getUrl(); + final Response response = downloader.get(url, getExtractorLocalization()); + doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + try { + return doc.select("div[class=\"playlist-info\"] h3[class=\"playlist-title\"]").first().text(); + } catch (Exception e) { + throw new ParsingException("Could not get playlist name", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return doc.select("ol[class*=\"playlist-videos-list\"] li").first().attr("data-thumbnail-url"); + } catch (Exception e) { + throw new ParsingException("Could not get playlist thumbnail", e); + } + } + + @Override + public String getBannerUrl() { + return ""; + } + + @Override + public String getUploaderUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderName() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderAvatarUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public long getStreamCount() { + // Auto-generated playlist always start with 25 videos and are endless + // But the html doesn't have a continuation url + return doc.select("ol[class*=\"playlist-videos-list\"] li").size(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + Element ol = doc.select("ol[class*=\"playlist-videos-list\"]").first(); + collectStreamsFrom(collector, ol); + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() { + return ""; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) { + //Continuations are not implemented + return null; + } + + private void collectStreamsFrom( + @Nonnull StreamInfoItemsCollector collector, + @Nullable Element element) { + collector.reset(); + + if (element == null) { + return; + } + + final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (final Element li : element.children()) { + + collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { + + @Override + public boolean isAd() { + return false; + } + + @Override + public String getUrl() throws ParsingException { + try { + return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getName() throws ParsingException { + try { + return li.attr("data-video-title"); + } catch (Exception e) { + throw new ParsingException("Could not get name", e); + } + } + + @Override + public long getDuration() throws ParsingException { + //Not present in doc + return 0; + } + + @Override + public String getUploaderName() throws ParsingException { + try { + return li.select( + "div[class=\"playlist-video-description\"]" + + "span[class=\"video-uploader-byline\"]") + .first() + .text(); + } catch (Exception e) { + throw new ParsingException("Could not get uploader", e); + } + } + + @Override + public String getUploaderUrl() { + //Not present in doc + return ""; + } + + @Override + public String getTextualUploadDate() { + //Not present in doc + return ""; + } + + @Override + public long getViewCount() { + return -1; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId() + + "/hqdefault.jpg"; + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 56abc194b..3d0ba78a5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -2,10 +2,13 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; +import java.net.MalformedURLException; import java.net.URL; import java.util.List; @@ -67,4 +70,22 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { } return true; } + + @Override + public ListLinkHandler fromUrl(String url) throws ParsingException { + try { + URL urlObj = Utils.stringToURL(url); + String listID = Utils.getQueryValue(urlObj, "list"); + if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { + String videoID = Utils.getQueryValue(urlObj, "v"); + String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; + return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), + getSortFilter(url)); + } + } catch (MalformedURLException exception) { + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); + } + return super.fromUrl(url); + } } From d74265c846854263088ee69dfca551b2f70493d8 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sun, 2 Feb 2020 18:04:15 +0100 Subject: [PATCH 02/14] [Youtube] Extract getThumbnailUrl into method and change getUploaderName --- .../YoutubeMixPlaylistExtractor.java | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 69b3fb987..64281931c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,12 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.StreamingService; @@ -50,7 +46,16 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - return doc.select("ol[class*=\"playlist-videos-list\"] li").first().attr("data-thumbnail-url"); + Element li = doc.select("ol[class*=\"playlist-videos-list\"] li").first(); + String videoId = li.attr("data-video-id"); + if (videoId != null && !videoId.isEmpty()) { + //higher quality + return getThumbnailUrlFromId(videoId); + } else { + //lower quality + return doc.select("ol[class*=\"playlist-videos-list\"] li").first() + .attr("data-thumbnail-url"); + } } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -146,21 +151,18 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } @Override - public long getDuration() throws ParsingException { + public long getDuration() { //Not present in doc return 0; } @Override public String getUploaderName() throws ParsingException { - try { - return li.select( - "div[class=\"playlist-video-description\"]" - + "span[class=\"video-uploader-byline\"]") - .first() - .text(); - } catch (Exception e) { - throw new ParsingException("Could not get uploader", e); + String uploaderName = li.attr("data-video-username"); + if (uploaderName == null || uploaderName.isEmpty()) { + throw new ParsingException("Could not get uploader name"); + } else { + return uploaderName; } } @@ -184,8 +186,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId() - + "/hqdefault.jpg"; + return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -193,4 +194,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { }); } } + + private String getThumbnailUrlFromId(String videoId) { + return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + } } From 327a5730a883aa6725a629763a3df0cebda30d03 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sun, 2 Feb 2020 18:15:47 +0100 Subject: [PATCH 03/14] [Youtube] Add some comments to mix --- .../extractor/services/youtube/YoutubeParsingHelper.java | 6 ++++++ .../youtube/extractors/YoutubeMixPlaylistExtractor.java | 4 ++++ .../linkHandler/YoutubePlaylistLinkHandlerFactory.java | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 42420a1a7..258799692 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -192,6 +192,12 @@ public class YoutubeParsingHelper { } } + /** + * Checks if the given playlist id is a mix (auto-generated playlist) + * Ids from a mix start with "RD" + * @param playlistId + * @return Whether given id belongs to a mix + */ public static boolean isYoutubeMixId(String playlistId) { return playlistId.startsWith("RD"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 64281931c..9da2e98fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -18,6 +18,10 @@ import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingH import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +/** + * A YoutubePlaylistExtractor for a mix (auto-generated playlist). + * It handles urls in the format of "youtube.com/watch?v=videoId&list=playlistId" + */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { private Document doc; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 3d0ba78a5..ae66660c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -71,6 +71,14 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { return true; } + /** + * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like + * youtube.com/watch?v=videoId&list=playlistId + *

Otherwise use super

+ * @param url + * @return + * @throws ParsingException + */ @Override public ListLinkHandler fromUrl(String url) throws ParsingException { try { From a376792a5d557e4d0e9101147af6fe49e72d4c3f Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Tue, 4 Feb 2020 13:49:43 +0100 Subject: [PATCH 04/14] [Youtube] Handle case where url is in "youtube.com/playlist?list=listID" format. This occurs when sharing a mix from the official youtube app. --- .../linkHandler/YoutubePlaylistLinkHandlerFactory.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index ae66660c3..787372dcb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -75,9 +75,6 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like * youtube.com/watch?v=videoId&list=playlistId *

Otherwise use super

- * @param url - * @return - * @throws ParsingException */ @Override public ListLinkHandler fromUrl(String url) throws ParsingException { @@ -86,6 +83,9 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { String listID = Utils.getQueryValue(urlObj, "list"); if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { String videoID = Utils.getQueryValue(urlObj, "v"); + if (videoID == null) { + videoID = listID.substring(2); + } String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), getSortFilter(url)); From 0ff054acb440ff310c543f600b32b594338873c2 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Fri, 6 Mar 2020 20:40:40 +0100 Subject: [PATCH 05/14] [Youtube] Extract initial playlist info --- .../YoutubeMixPlaylistExtractor.java | 156 ++++++++++-------- 1 file changed, 83 insertions(+), 73 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 9da2e98fd..e9bc81623 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,5 +1,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -24,6 +28,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + private JsonObject initialData; private Document doc; public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -32,9 +37,14 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl(); - final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + final String url = getUrl() + "&pbj=1"; + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + initialData = ajaxJson.getObject(3).getObject("response"); + JsonObject a = initialData.getObject("contents"); + JsonObject b = initialData.getObject("contents").getObject("twoColumnWatchNextResults"); + JsonObject c = initialData.getObject("contents"); + JsonObject playlist = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); + System.out.println(); } @Nonnull @@ -127,76 +137,76 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Element li : element.children()) { - - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - - @Override - public boolean isAd() { - return false; - } - - @Override - public String getUrl() throws ParsingException { - try { - return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getName() throws ParsingException { - try { - return li.attr("data-video-title"); - } catch (Exception e) { - throw new ParsingException("Could not get name", e); - } - } - - @Override - public long getDuration() { - //Not present in doc - return 0; - } - - @Override - public String getUploaderName() throws ParsingException { - String uploaderName = li.attr("data-video-username"); - if (uploaderName == null || uploaderName.isEmpty()) { - throw new ParsingException("Could not get uploader name"); - } else { - return uploaderName; - } - } - - @Override - public String getUploaderUrl() { - //Not present in doc - return ""; - } - - @Override - public String getTextualUploadDate() { - //Not present in doc - return ""; - } - - @Override - public long getViewCount() { - return -1; - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); - } +// for (final Element li : element.children()) { +// +// collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { +// +// @Override +// public boolean isAd() { +// return false; +// } +// +// @Override +// public String getUrl() throws ParsingException { +// try { +// return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); +// } catch (Exception e) { +// throw new ParsingException("Could not get web page url for the video", e); +// } +// } +// +// @Override +// public String getName() throws ParsingException { +// try { +// return li.attr("data-video-title"); +// } catch (Exception e) { +// throw new ParsingException("Could not get name", e); +// } +// } +// +// @Override +// public long getDuration() { +// //Not present in doc +// return 0; +// } +// +// @Override +// public String getUploaderName() throws ParsingException { +// String uploaderName = li.attr("data-video-username"); +// if (uploaderName == null || uploaderName.isEmpty()) { +// throw new ParsingException("Could not get uploader name"); +// } else { +// return uploaderName; +// } +// } +// +// @Override +// public String getUploaderUrl() { +// //Not present in doc +// return ""; +// } +// +// @Override +// public String getTextualUploadDate() { +// //Not present in doc +// return ""; +// } +// +// @Override +// public long getViewCount() { +// return -1; +// } +// +// @Override +// public String getThumbnailUrl() throws ParsingException { +// try { +// return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); +// } catch (Exception e) { +// throw new ParsingException("Could not get thumbnail url", e); +// } +// } +// }); +// } } private String getThumbnailUrlFromId(String videoId) { From ec6b99c082267e2df609cac46144fc4f41109591 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 7 Mar 2020 08:42:26 +0100 Subject: [PATCH 06/14] [Youtube] Adjust mix extractor to new user agent Also extract continuation now --- .../YoutubeMixPlaylistExtractor.java | 140 ++++++------------ 1 file changed, 43 insertions(+), 97 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index e9bc81623..af200a9a8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import com.grack.nanojson.JsonArray; @@ -29,7 +30,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { private JsonObject initialData; - private Document doc; + private JsonObject playlistData; public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -40,10 +41,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final String url = getUrl() + "&pbj=1"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(3).getObject("response"); - JsonObject a = initialData.getObject("contents"); - JsonObject b = initialData.getObject("contents").getObject("twoColumnWatchNextResults"); - JsonObject c = initialData.getObject("contents"); - JsonObject playlist = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); + playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); System.out.println(); } @@ -51,7 +49,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getName() throws ParsingException { try { - return doc.select("div[class=\"playlist-info\"] h3[class=\"playlist-title\"]").first().text(); + final String name = playlistData.getString("title"); + if (name!= null) return name; + else return ""; } catch (Exception e) { throw new ParsingException("Could not get playlist name", e); } @@ -60,16 +60,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - Element li = doc.select("ol[class*=\"playlist-videos-list\"] li").first(); - String videoId = li.attr("data-video-id"); - if (videoId != null && !videoId.isEmpty()) { - //higher quality - return getThumbnailUrlFromId(videoId); - } else { - //lower quality - return doc.select("ol[class*=\"playlist-videos-list\"] li").first() - .attr("data-thumbnail-url"); - } + final String videoId = playlistData.getArray("contents").getObject(0).getString("videoId"); + return getThumbnailUrlFromId(videoId); } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -101,112 +93,66 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public long getStreamCount() { // Auto-generated playlist always start with 25 videos and are endless - // But the html doesn't have a continuation url - return doc.select("ol[class*=\"playlist-videos-list\"] li").size(); + return 25; } @Nonnull @Override public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Element ol = doc.select("ol[class*=\"playlist-videos-list\"]").first(); - collectStreamsFrom(collector, ol); + collectStreamsFrom(collector, playlistData.getArray("contents")); return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override public String getNextPageUrl() { - return ""; + final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") + .get(playlistData.getArray("contents").size() - 1)); + final String lastStreamId = lastStream.getObject("playlistPanelVideoRenderer") + .getString("videoId"); + return "https://youtube.com" + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint") + .getObject("commandMetadata").getObject("webCommandMetadata").getString("url") + "&pbj=1"; } @Override - public InfoItemsPage getPage(final String pageUrl) { - //Continuations are not implemented - return null; + public InfoItemsPage getPage(final String pageUrl) + throws ExtractionException, IOException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + playlistData = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray streams = playlistData.getArray("contents"); + //Because continuation requests are created with the last video of previous request as start + streams.remove(0); + collectStreamsFrom(collector, streams); + return new InfoItemsPage<>(collector, getNextPageUrl()); } private void collectStreamsFrom( @Nonnull StreamInfoItemsCollector collector, - @Nullable Element element) { + @Nullable JsonArray streams) { collector.reset(); - if (element == null) { + if (streams == null) { return; } - final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); -// for (final Element li : element.children()) { -// -// collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { -// -// @Override -// public boolean isAd() { -// return false; -// } -// -// @Override -// public String getUrl() throws ParsingException { -// try { -// return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); -// } catch (Exception e) { -// throw new ParsingException("Could not get web page url for the video", e); -// } -// } -// -// @Override -// public String getName() throws ParsingException { -// try { -// return li.attr("data-video-title"); -// } catch (Exception e) { -// throw new ParsingException("Could not get name", e); -// } -// } -// -// @Override -// public long getDuration() { -// //Not present in doc -// return 0; -// } -// -// @Override -// public String getUploaderName() throws ParsingException { -// String uploaderName = li.attr("data-video-username"); -// if (uploaderName == null || uploaderName.isEmpty()) { -// throw new ParsingException("Could not get uploader name"); -// } else { -// return uploaderName; -// } -// } -// -// @Override -// public String getUploaderUrl() { -// //Not present in doc -// return ""; -// } -// -// @Override -// public String getTextualUploadDate() { -// //Not present in doc -// return ""; -// } -// -// @Override -// public long getViewCount() { -// return -1; -// } -// -// @Override -// public String getThumbnailUrl() throws ParsingException { -// try { -// return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); -// } catch (Exception e) { -// throw new ParsingException("Could not get thumbnail url", e); -// } -// } -// }); -// } + for (Object stream : streams) { + if (stream instanceof JsonObject) { + JsonObject streamInfo = ((JsonObject) stream).getObject("playlistPanelVideoRenderer"); + if (streamInfo != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + } + } + } } private String getThumbnailUrlFromId(String videoId) { From 68a3948af6dbd5795c061ff874df67ed1d19cb78 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 7 Mar 2020 15:54:17 +0100 Subject: [PATCH 07/14] [Youtube] Fix get banner url --- .../youtube/extractors/YoutubeMixPlaylistExtractor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index af200a9a8..6687eb844 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -60,7 +60,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - final String videoId = playlistData.getArray("contents").getObject(0).getString("videoId"); + final String videoId = playlistData.getArray("contents").getObject(0) + .getObject("playlistPanelVideoRenderer").getString("videoId"); + if (videoId == null || videoId.isEmpty()) throw new ParsingException(""); return getThumbnailUrlFromId(videoId); } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); From df38b1926c8a5fb9fb510dd15ff75b5f13897877 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 7 Mar 2020 16:03:12 +0100 Subject: [PATCH 08/14] [Youtube] Add tests and take thumbnail image always from first video of mix Also fix getThumbnailUrl for "My Mix" --- .../YoutubeMixPlaylistExtractor.java | 285 ++++++++-------- .../YoutubeMixPlaylistExtractorTest.java | 323 ++++++++++++++++++ ...YoutubePlaylistLinkHandlerFactoryTest.java | 19 ++ .../services/youtube/YoutubeServiceTest.java | 24 ++ 4 files changed, 518 insertions(+), 133 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 6687eb844..f4868a61e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import com.grack.nanojson.JsonArray; @@ -8,156 +7,176 @@ import com.grack.nanojson.JsonObject; import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; /** - * A YoutubePlaylistExtractor for a mix (auto-generated playlist). - * It handles urls in the format of "youtube.com/watch?v=videoId&list=playlistId" + * A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of + * "youtube.com/watch?v=videoId&list=playlistId" */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { - private JsonObject initialData; - private JsonObject playlistData; + private final static String CONTENTS = "contents"; + private final static String RESPONSE = "response"; + private final static String PLAYLIST = "playlist"; + private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults"; + private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer"; - public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { - super(service, linkHandler); - } + private JsonObject playlistData; - @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + "&pbj=1"; - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - initialData = ajaxJson.getObject(3).getObject("response"); - playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); - System.out.println(); - } - - @Nonnull - @Override - public String getName() throws ParsingException { - try { - final String name = playlistData.getString("title"); - if (name!= null) return name; - else return ""; - } catch (Exception e) { - throw new ParsingException("Could not get playlist name", e); - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - final String videoId = playlistData.getArray("contents").getObject(0) - .getObject("playlistPanelVideoRenderer").getString("videoId"); - if (videoId == null || videoId.isEmpty()) throw new ParsingException(""); - return getThumbnailUrlFromId(videoId); - } catch (Exception e) { - throw new ParsingException("Could not get playlist thumbnail", e); - } - } - - @Override - public String getBannerUrl() { - return ""; - } - - @Override - public String getUploaderUrl() { - //Youtube mix are auto-generated - return ""; - } - - @Override - public String getUploaderName() { - //Youtube mix are auto-generated - return ""; - } - - @Override - public String getUploaderAvatarUrl() { - //Youtube mix are auto-generated - return ""; - } - - @Override - public long getStreamCount() { - // Auto-generated playlist always start with 25 videos and are endless - return 25; - } - - @Nonnull - @Override - public InfoItemsPage getInitialPage() { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, playlistData.getArray("contents")); - return new InfoItemsPage<>(collector, getNextPageUrl()); - } - - @Override - public String getNextPageUrl() { - final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") - .get(playlistData.getArray("contents").size() - 1)); - final String lastStreamId = lastStream.getObject("playlistPanelVideoRenderer") - .getString("videoId"); - return "https://youtube.com" + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint") - .getObject("commandMetadata").getObject("webCommandMetadata").getString("url") + "&pbj=1"; - } - - @Override - public InfoItemsPage getPage(final String pageUrl) - throws ExtractionException, IOException { - if (pageUrl == null || pageUrl.isEmpty()) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); } - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - playlistData = - ajaxJson.getObject(3).getObject("response").getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("playlist") - .getObject("playlist"); - final JsonArray streams = playlistData.getArray("contents"); - //Because continuation requests are created with the last video of previous request as start - streams.remove(0); - collectStreamsFrom(collector, streams); - return new InfoItemsPage<>(collector, getNextPageUrl()); - } - - private void collectStreamsFrom( - @Nonnull StreamInfoItemsCollector collector, - @Nullable JsonArray streams) { - collector.reset(); - - if (streams == null) { - return; - } - - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - - for (Object stream : streams) { - if (stream instanceof JsonObject) { - JsonObject streamInfo = ((JsonObject) stream).getObject("playlistPanelVideoRenderer"); - if (streamInfo != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + @Override + public void onFetchPage(@Nonnull Downloader downloader) + throws IOException, ExtractionException { + final String url = getUrl() + "&pbj=1"; + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE); + try { + playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS) + .getObject(PLAYLIST).getObject(PLAYLIST); + } catch (NullPointerException e) { + throw new ExtractionException(e); } - } - } - } - private String getThumbnailUrlFromId(String videoId) { - return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; - } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + try { + final String name = playlistData.getString("title"); + if (name != null) { + return name; + } else { + return ""; + } + } catch (Exception e) { + throw new ParsingException("Could not get playlist name", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final String playlistId = playlistData.getString("playlistId"); + final String videoId; + if (playlistId.startsWith("RDMM")) { + videoId = playlistId.substring(4); + } else { + videoId = playlistId.substring(2); + } + if (videoId.isEmpty()) { + throw new ParsingException(""); + } + return getThumbnailUrlFromId(videoId); + } catch (Exception e) { + throw new ParsingException("Could not get playlist thumbnail", e); + } + } + + @Override + public String getBannerUrl() { + return ""; + } + + @Override + public String getUploaderUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderName() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderAvatarUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public long getStreamCount() { + // Auto-generated playlist always start with 25 videos and are endless + return 25; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, playlistData.getArray(CONTENTS)); + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS) + .get(playlistData.getArray(CONTENTS).size() - 1)); + if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) { + throw new ExtractionException("Could not extract next page url"); + } + return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) + .getObject("navigationEndpoint").getObject("commandMetadata") + .getObject("webCommandMetadata").getString("url") + "&pbj=1"; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) + throws ExtractionException, IOException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException( + new IllegalArgumentException("Page url is empty or null")); + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + playlistData = + ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS) + .getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS).getObject(PLAYLIST) + .getObject(PLAYLIST); + final JsonArray streams = playlistData.getArray(CONTENTS); + //Because continuation requests are created with the last video of previous request as start + streams.remove(0); + collectStreamsFrom(collector, streams); + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + private void collectStreamsFrom( + @Nonnull StreamInfoItemsCollector collector, + @Nullable JsonArray streams) { + collector.reset(); + + if (streams == null) { + return; + } + + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object stream : streams) { + if (stream instanceof JsonObject) { + JsonObject streamInfo = ((JsonObject) stream) + .getObject(PLAYLIST_PANEL_VIDEO_RENDERER); + if (streamInfo != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + } + } + } + } + + private String getThumbnailUrlFromId(String videoId) { + return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java new file mode 100644 index 000000000..0d21ad8c6 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -0,0 +1,323 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +public class YoutubeMixPlaylistExtractorTest { + + private static YoutubeMixPlaylistExtractor extractor; + private static String videoId = "_AzeUSL9lZc"; + private static String videoTitle = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; + + public static class Mix { + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + extractor.fetchPage(); + } + + @Test + public void getServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void getName() throws Exception { + String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(videoTitle)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(videoId)); + } + + @Test + public void getNextPageUrl() throws Exception { + final String nextPageUrl = extractor.getNextPageUrl(); + assertIsSecureUrl(nextPageUrl); + assertThat(nextPageUrl, containsString("list=RD" + videoId)); + } + + @Test + public void getInitialPage() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPageMultipleTimes() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + streams = extractor.getPage(streams.getNextPageUrl()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class MixWithIndex { + + private static String index = "&index=13"; + private static String videoIdNumber13 = "qHtzO49SDmk"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoIdNumber13 + "&list=RD" + videoId + + index); + extractor.fetchPage(); + } + + @Test + public void getName() throws Exception { + String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(videoTitle)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(videoId)); + } + + @Test + public void getNextPageUrl() throws Exception { + final String nextPageUrl = extractor.getNextPageUrl(); + assertIsSecureUrl(nextPageUrl); + assertThat(nextPageUrl, containsString("list=RD" + videoId)); + } + + @Test + public void getInitialPage() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPageMultipleTimes() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + streams = extractor.getPage(streams.getNextPageUrl()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class MyMix { + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + extractor.fetchPage(); + } + + @Test + public void getServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void getName() throws Exception { + String name = extractor.getName(); + assertEquals("My Mix", name); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc")); + } + + @Test + public void getNextPageUrl() throws Exception { + final String nextPageUrl = extractor.getNextPageUrl(); + assertIsSecureUrl(nextPageUrl); + assertThat(nextPageUrl, containsString("list=RDMM" + videoId)); + } + + @Test + public void getInitialPage() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPageMultipleTimes() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + streams = extractor.getPage(streams.getNextPageUrl()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class Invalid { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ExtractionException.class) + public void getPageEmptyUrl() throws Exception { + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + extractor.fetchPage(); + extractor.getPage(""); + } + + @Test(expected = NullPointerException.class) + public void invalidVideoId() throws Exception { + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde"); + extractor.fetchPage(); + } + } + + public static class ChannelMix { + + private static String channelId = "UCXuqSBlHAE6Xw-yeJA0Tunw"; + private static String videoIdOfChannel = "mnk6gnOBYIo"; + private static String channelTitle = "Linus Tech Tips"; + + + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoIdOfChannel + "&list=RDCM" + channelId); + extractor.fetchPage(); + } + + @Test + public void getName() throws Exception { + String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(channelTitle)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + } + + @Test + public void getNextPageUrl() throws Exception { + final String nextPageUrl = extractor.getNextPageUrl(); + assertIsSecureUrl(nextPageUrl); + assertThat(nextPageUrl, containsString("list=RDCM" + channelId)); + } + + @Test + public void getInitialPage() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 636a646f8..443a21adf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -105,4 +105,23 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); } + + @Test + public void fromUrlIsMixVideo() throws Exception { + final String videoId = "_AzeUSL9lZc"; + String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; + assertEquals(url, linkHandler.fromUrl(url).getUrl()); + + final String mixVideoId = "qHtzO49SDmk"; + url = "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId; + assertEquals(url, linkHandler.fromUrl(url).getUrl()); + } + + @Test + public void fromUrlIsMixPlaylist() throws Exception { + final String videoId = "_AzeUSL9lZc"; + final String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; + assertEquals(url, + linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl()); + } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index 6de39e5ce..de4fe53fc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -26,9 +26,13 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** @@ -54,4 +58,24 @@ public class YoutubeServiceTest { public void testGetDefaultKiosk() throws Exception { assertEquals(kioskList.getDefaultKioskExtractor(null).getId(), "Trending"); } + + + @Test + public void getPlayListExtractorIsNormalPlaylist() throws Exception { + PlaylistExtractor extractor = service.getPlaylistExtractor( + "https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS"); + assertTrue(extractor instanceof YoutubePlaylistExtractor); + } + + @Test + public void getPlaylistExtractorIsMix() throws Exception { + String videoId = "_AzeUSL9lZc"; + PlaylistExtractor extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + + extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + } } From 822cf307f723613baaaa2fde0bbb78aaa1d4082a Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Tue, 17 Mar 2020 14:04:46 +0100 Subject: [PATCH 09/14] [Youtube] Add _ITEMS constants and improve code style Move thumbnail id exctraction code to getThumbnailUrlFromId Add test for "My mix" detection to service tests Use ITEM_COUNT_UNKNOWN everywhere instead of -1 and add some tests --- .../MediaCCCConferenceInfoItemExtractor.java | 4 +- .../YoutubeChannelInfoItemExtractor.java | 3 +- .../YoutubeMixPlaylistExtractor.java | 63 +++++++++---------- .../YoutubeMixPlaylistExtractorTest.java | 15 +++++ .../services/youtube/YoutubeServiceTest.java | 8 ++- 5 files changed, 55 insertions(+), 38 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java index 9099cb1a7..cea663edb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -23,7 +25,7 @@ public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtra @Override public long getStreamCount() { - return -1; + return ListExtractor.ITEM_COUNT_UNKNOWN; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 881fbd794..fbab74d83 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; @@ -86,7 +87,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor try { if (!channelInfoItem.has("videoCountText")) { // Video count is not available, channel probably has no public uploads. - return -1; + return ListExtractor.ITEM_COUNT_UNKNOWN; } return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index f4868a61e..060edc20b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -7,6 +7,8 @@ import com.grack.nanojson.JsonObject; import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; + +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -23,6 +25,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + private final static String CONTENTS = "contents"; private final static String RESPONSE = "response"; private final static String PLAYLIST = "playlist"; @@ -41,44 +44,25 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final String url = getUrl() + "&pbj=1"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE); - try { - playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS) - .getObject(PLAYLIST).getObject(PLAYLIST); - } catch (NullPointerException e) { - throw new ExtractionException(e); - } - + playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMN_WATCH_NEXT_RESULTS) + .getObject(PLAYLIST).getObject(PLAYLIST); } @Nonnull @Override public String getName() throws ParsingException { - try { - final String name = playlistData.getString("title"); - if (name != null) { - return name; - } else { - return ""; - } - } catch (Exception e) { - throw new ParsingException("Could not get playlist name", e); + final String name = playlistData.getString("title"); + if (name == null) { + throw new ParsingException("Could not get playlist name"); } + return name; } @Override public String getThumbnailUrl() throws ParsingException { try { final String playlistId = playlistData.getString("playlistId"); - final String videoId; - if (playlistId.startsWith("RDMM")) { - videoId = playlistId.substring(4); - } else { - videoId = playlistId.substring(2); - } - if (videoId.isEmpty()) { - throw new ParsingException(""); - } - return getThumbnailUrlFromId(videoId); + return getThumbnailUrlFromId(playlistId); } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -97,20 +81,20 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderName() { - //Youtube mix are auto-generated - return ""; + //Youtube mix are auto-generated by YouTube + return "YouTube"; } @Override public String getUploaderAvatarUrl() { - //Youtube mix are auto-generated + //Youtube mix are auto-generated by YouTube return ""; } @Override public long getStreamCount() { // Auto-generated playlist always start with 25 videos and are endless - return 25; + return ListExtractor.ITEM_COUNT_INFINITE; } @Nonnull @@ -135,7 +119,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getPage(final String pageUrl) - throws ExtractionException, IOException { + throws ExtractionException, IOException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException( new IllegalArgumentException("Page url is empty or null")); @@ -145,7 +129,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); playlistData = ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS) - .getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS).getObject(PLAYLIST) + .getObject(TWO_COLUMN_WATCH_NEXT_RESULTS).getObject(PLAYLIST) .getObject(PLAYLIST); final JsonArray streams = playlistData.getArray(CONTENTS); //Because continuation requests are created with the last video of previous request as start @@ -155,8 +139,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } private void collectStreamsFrom( - @Nonnull StreamInfoItemsCollector collector, - @Nullable JsonArray streams) { + @Nonnull StreamInfoItemsCollector collector, + @Nullable JsonArray streams) { collector.reset(); if (streams == null) { @@ -176,7 +160,16 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } } - private String getThumbnailUrlFromId(String videoId) { + private String getThumbnailUrlFromId(String playlistId) throws ParsingException { + final String videoId; + if (playlistId.startsWith("RDMM")) { + videoId = playlistId.substring(4); + } else { + videoId = playlistId.substring(2); + } + if (videoId.isEmpty()) { + throw new ParsingException("videoId is empty"); + } return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 0d21ad8c6..d18e4fc55 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -93,6 +93,11 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } } public static class MixWithIndex { @@ -165,6 +170,11 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } } public static class MyMix { @@ -319,5 +329,10 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index de4fe53fc..7b5d2ce73 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -75,7 +75,13 @@ public class YoutubeServiceTest { assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); extractor = YouTube.getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + + final String mixVideoId = "qHtzO49SDmk"; + + extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId); assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); } } From 3ff8619bcc703920705431e217d549726cefb830 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 21 Mar 2020 18:48:12 +0100 Subject: [PATCH 10/14] [Youtube] apply wb9688 suggestion (mix) Channel mix adjusments and test Don't accept youtube music mix urls as playlist Don't override playlistData to keep getInitialPage() Remove json constants Indentation --- .../youtube/YoutubeParsingHelper.java | 23 +++++-- .../services/youtube/YoutubeService.java | 2 +- .../YoutubeMixPlaylistExtractor.java | 67 +++++++++++-------- .../YoutubePlaylistLinkHandlerFactory.java | 5 +- .../YoutubeMixPlaylistExtractorTest.java | 19 +----- 5 files changed, 64 insertions(+), 52 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 258799692..4ac9724b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -193,13 +193,23 @@ public class YoutubeParsingHelper { } /** - * Checks if the given playlist id is a mix (auto-generated playlist) - * Ids from a mix start with "RD" + * Checks if the given playlist id is a youtube mix (auto-generated playlist) + * Ids from a youtube mix start with "RD" * @param playlistId - * @return Whether given id belongs to a mix + * @return Whether given id belongs to a youtube mix */ public static boolean isYoutubeMixId(String playlistId) { - return playlistId.startsWith("RD"); + return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); + } + + /** + * Checks if the given playlist id is a youtube music mix (auto-generated playlist) + * Ids from a youtube music mix start with "RD" + * @param playlistId + * @return Whether given id belongs to a youtube music mix + */ + public static boolean isYoutubeMusicMixId(String playlistId) { + return playlistId.startsWith("RDAMVM"); } public static JsonObject getInitialData(String html) throws ParsingException { @@ -427,9 +437,9 @@ public class YoutubeParsingHelper { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) - url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); return url.toString(); } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + @@ -457,6 +467,7 @@ public class YoutubeParsingHelper { if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); if (!isNullOrEmpty(url)) { + url = url.replaceAll("&", "&"); textBuilder.append("").append(text).append(""); continue; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 7d7a83eba..fce5a1d46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -144,7 +144,7 @@ public class YoutubeService extends StreamingService { public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) - throws ExtractionException { + throws ExtractionException { return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 060edc20b..0aedafd0e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -25,13 +26,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { - - private final static String CONTENTS = "contents"; - private final static String RESPONSE = "response"; - private final static String PLAYLIST = "playlist"; - private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults"; - private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer"; - + private JsonObject initialData; private JsonObject playlistData; public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -43,9 +38,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE); - playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMN_WATCH_NEXT_RESULTS) - .getObject(PLAYLIST).getObject(PLAYLIST); + initialData = ajaxJson.getObject(3).getObject("response"); + playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("playlist").getObject("playlist"); } @Nonnull @@ -62,7 +57,14 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { public String getThumbnailUrl() throws ParsingException { try { final String playlistId = playlistData.getString("playlistId"); - return getThumbnailUrlFromId(playlistId); + try { + return getThumbnailUrlFromPlaylistId(playlistId); + } catch (ParsingException e) { + //fallback to thumbnail of current video. Always the case for channel mix + return getThumbnailUrlFromVideoId( + initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") + .getString("videoId")); + } } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -101,20 +103,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, playlistData.getArray(CONTENTS)); + collectStreamsFrom(collector, playlistData.getArray("contents")); return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override public String getNextPageUrl() throws ExtractionException { - final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS) - .get(playlistData.getArray(CONTENTS).size() - 1)); - if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) { + return getNextPageUrlFrom(playlistData); + } + + private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") + .get(playlistData.getArray("contents").size() - 1)); + if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { throw new ExtractionException("Could not extract next page url"); } - return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) - .getObject("navigationEndpoint").getObject("commandMetadata") - .getObject("webCommandMetadata").getString("url") + "&pbj=1"; + //Index of video in mix is missing, but adding it doesn't appear to have any effect. + //And since the index needs to be tracked by us, it is left out + return getUrlFromNavigationEndpoint( + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) + + "&pbj=1"; } @Override @@ -127,21 +135,20 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - playlistData = - ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS) - .getObject(TWO_COLUMN_WATCH_NEXT_RESULTS).getObject(PLAYLIST) - .getObject(PLAYLIST); - final JsonArray streams = playlistData.getArray(CONTENTS); + JsonObject playlistData = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray streams = playlistData.getArray("contents"); //Because continuation requests are created with the last video of previous request as start streams.remove(0); collectStreamsFrom(collector, streams); - return new InfoItemsPage<>(collector, getNextPageUrl()); + return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData)); } private void collectStreamsFrom( @Nonnull StreamInfoItemsCollector collector, @Nullable JsonArray streams) { - collector.reset(); if (streams == null) { return; @@ -152,7 +159,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { for (Object stream : streams) { if (stream instanceof JsonObject) { JsonObject streamInfo = ((JsonObject) stream) - .getObject(PLAYLIST_PANEL_VIDEO_RENDERER); + .getObject("playlistPanelVideoRenderer"); if (streamInfo != null) { collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); } @@ -160,16 +167,22 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } } - private String getThumbnailUrlFromId(String playlistId) throws ParsingException { + private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException { final String videoId; if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); + } else if (playlistId.startsWith("RDCMUC")) { + throw new ParsingException("is channel mix"); } else { videoId = playlistId.substring(2); } if (videoId.isEmpty()) { throw new ParsingException("videoId is empty"); } + return getThumbnailUrlFromVideoId(videoId); + } + + private String getThumbnailUrlFromVideoId(String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 787372dcb..b565fde62 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -64,11 +64,12 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { try { - getId(url); + String playlistId = getId(url); + //Because youtube music mix are not supported yet. + return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId); } catch (ParsingException e) { return false; } - return true; } /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index d18e4fc55..b612827a7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -93,11 +94,6 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } - - @Test - public void getStreamCount() throws Exception { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } } public static class MixWithIndex { @@ -166,11 +162,6 @@ public class YoutubeMixPlaylistExtractorTest { assertFalse(streams.getItems().isEmpty()); } - @Test - public void getStreamCount() { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } - @Test public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); @@ -264,12 +255,13 @@ public class YoutubeMixPlaylistExtractorTest { extractor.getPage(""); } - @Test(expected = NullPointerException.class) + @Test(expected = ExtractionException.class) public void invalidVideoId() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde"); extractor.fetchPage(); + extractor.getName(); } } @@ -329,10 +321,5 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } - - @Test - public void getStreamCount() throws Exception { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } } } \ No newline at end of file From 421935401f9c8fc45aed92742cb111f6b9e45193 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 21 May 2020 14:52:13 +0200 Subject: [PATCH 11/14] [Youtube] Add subchannel functions to mix and fix imports --- .../services/youtube/YoutubeService.java | 1 + .../YoutubeMixPlaylistExtractor.java | 32 +++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index fce5a1d46..10ac64404 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 0aedafd0e..0bf1fcbe8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,13 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import java.io.IOException; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; @@ -20,6 +14,14 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import java.io.IOException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; + /** * A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of * "youtube.com/watch?v=videoId&list=playlistId" @@ -185,4 +187,22 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { private String getThumbnailUrlFromVideoId(String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } + + @Nonnull + @Override + public String getSubChannelName() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() { + return ""; + } } From 22d2f7e4006036aab8ac63478b791c4ab3de1624 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Thu, 16 Apr 2020 19:28:27 +0200 Subject: [PATCH 12/14] [Youtube] Add cookies to youtube mix request This way youtube wont return duplicates when getting more items of the mix (but youtube can also track us) --- .../youtube/YoutubeParsingHelper.java | 93 +++++++-- .../services/youtube/YoutubeService.java | 2 +- .../YoutubeMixPlaylistExtractor.java | 102 ++++++---- .../YoutubePlaylistLinkHandlerFactory.java | 52 +++--- .../schabi/newpipe/extractor/utils/Utils.java | 12 ++ .../YoutubeMixPlaylistExtractorTest.java | 176 ++++++++++-------- ...YoutubePlaylistLinkHandlerFactoryTest.java | 2 +- .../services/youtube/YoutubeServiceTest.java | 4 +- 8 files changed, 276 insertions(+), 167 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 4ac9724b2..ea8743069 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -5,8 +5,10 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -21,6 +23,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -35,6 +38,7 @@ import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.join; /* * Created by Christian Schabesberger on 02.03.16. @@ -61,6 +65,12 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + /** + * The official youtube app supports intents in this format, where after the ':' is the videoId. + * Accordingly there are other apps sharing streams in this format. + */ + public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube"; + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; @@ -193,22 +203,22 @@ public class YoutubeParsingHelper { } /** - * Checks if the given playlist id is a youtube mix (auto-generated playlist) - * Ids from a youtube mix start with "RD" + * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) + * Ids from a YouTube Mix start with "RD" * @param playlistId - * @return Whether given id belongs to a youtube mix + * @return Whether given id belongs to a YouTube Mix */ - public static boolean isYoutubeMixId(String playlistId) { + public static boolean isYoutubeMixId(final String playlistId) { return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); } /** - * Checks if the given playlist id is a youtube music mix (auto-generated playlist) - * Ids from a youtube music mix start with "RD" + * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) + * Ids from a YouTube Music Mix start with "RD" * @param playlistId - * @return Whether given id belongs to a youtube music mix + * @return Whether given id belongs to a YouTube Music Mix */ - public static boolean isYoutubeMusicMixId(String playlistId) { + public static boolean isYoutubeMusicMixId(final String playlistId) { return playlistId.startsWith("RDAMVM"); } @@ -352,7 +362,7 @@ public class YoutubeParsingHelper { .end() .value("query", "test") .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") - .end().done().getBytes("UTF-8"); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on Map> headers = new HashMap<>(); @@ -436,10 +446,14 @@ public class YoutubeParsingHelper { } else if (navigationEndpoint.has("watchEndpoint")) { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); - if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); - if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) - url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") + .getString("playlistId")); + } + if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint") + .getInt("startTimeSeconds")); + } return url.toString(); } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + @@ -467,7 +481,6 @@ public class YoutubeParsingHelper { if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); if (!isNullOrEmpty(url)) { - url = url.replaceAll("&", "&"); textBuilder.append("").append(text).append(""); continue; } @@ -506,8 +519,8 @@ public class YoutubeParsingHelper { public static String getValidJsonResponseBody(final Response response) throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); } final String responseBody = response.responseBody(); @@ -527,13 +540,39 @@ public class YoutubeParsingHelper { final String responseContentType = response.getHeader("Content-Type"); if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); } return responseBody; } + public static Response getResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(url, headers, localization); + getValidJsonResponseBody(response); + + return response; + } + + public static String extractCookieValue(final String cookieName, final Response response) { + final List cookies = response.responseHeaders().get("Set-Cookie"); + int startIndex; + String result = ""; + for (final String cookie : cookies) { + startIndex = cookie.indexOf(cookieName); + if (startIndex != -1) { + result = cookie.substring(startIndex + cookieName.length() + "=".length(), + cookie.indexOf(";", startIndex)); + } + } + return result; + } + public static JsonArray getJsonResponse(final String url, final Localization localization) throws IOException, ExtractionException { Map> headers = new HashMap<>(); @@ -541,8 +580,24 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); final Response response = getDownloader().get(url, headers, localization); - final String responseBody = getValidJsonResponseBody(response); + return toJsonArray(getValidJsonResponseBody(response)); + } + public static JsonArray getJsonResponse(final Page page, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + if (!isNullOrEmpty(page.getCookies())) { + headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies()))); + } + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(page.getUrl(), headers, localization); + + return toJsonArray(getValidJsonResponseBody(response)); + } + + public static JsonArray toJsonArray(final String responseBody) throws ParsingException { try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 10ac64404..997fd0b73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -110,7 +110,7 @@ public class YoutubeService extends StreamingService { } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { return new YoutubeMixPlaylistExtractor(this, linkHandler); } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 0bf1fcbe8..d5e307453 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -4,8 +4,10 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -15,34 +17,50 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; +import java.util.Collections; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray; /** - * A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of - * "youtube.com/watch?v=videoId&list=playlistId" + * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). + * It handles URLs in the format of + * {@code youtube.com/watch?v=videoId&list=playlistId} */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + /** + * YouTube identifies mixes based on this cookie. With this information it can generate + * continuations without duplicates. + */ + private static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; + private JsonObject initialData; private JsonObject playlistData; + private String cookieValue; - public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + public YoutubeMixPlaylistExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) - throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + final Response response = getResponse(url, getExtractorLocalization()); + final JsonArray ajaxJson = toJsonArray(response.responseBody()); initialData = ajaxJson.getObject(3).getObject("response"); playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("playlist").getObject("playlist"); + .getObject("playlist").getObject("playlist"); + cookieValue = extractCookieValue(COOKIE_NAME, response); } @Nonnull @@ -58,16 +76,15 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - final String playlistId = playlistData.getString("playlistId"); + return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId")); + } catch (final Exception e) { try { - return getThumbnailUrlFromPlaylistId(playlistId); - } catch (ParsingException e) { //fallback to thumbnail of current video. Always the case for channel mix return getThumbnailUrlFromVideoId( initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") .getString("videoId")); + } catch (final Exception ignored) { } - } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } } @@ -104,53 +121,56 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, playlistData.getArray("contents")); - return new InfoItemsPage<>(collector, getNextPageUrl()); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrl(), Collections.singletonMap(COOKIE_NAME, cookieValue))); } - @Override - public String getNextPageUrl() throws ExtractionException { + private String getNextPageUrl() throws ExtractionException { return getNextPageUrlFrom(playlistData); } - private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException { - final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") - .get(playlistData.getArray("contents").size() - 1)); + private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents") + .get(playlistJson.getArray("contents").size() - 1)); if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { throw new ExtractionException("Could not extract next page url"); } - //Index of video in mix is missing, but adding it doesn't appear to have any effect. - //And since the index needs to be tracked by us, it is left out + return getUrlFromNavigationEndpoint( - lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) - + "&pbj=1"; + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) + + "&pbj=1"; } @Override - public InfoItemsPage getPage(final String pageUrl) + public InfoItemsPage getPage(final Page page) throws ExtractionException, IOException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (page == null || page.getUrl().isEmpty()) { throw new ExtractionException( new IllegalArgumentException("Page url is empty or null")); } - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonObject playlistData = - ajaxJson.getObject(3).getObject("response").getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("playlist") - .getObject("playlist"); - final JsonArray streams = playlistData.getArray("contents"); - //Because continuation requests are created with the last video of previous request as start - streams.remove(0); - collectStreamsFrom(collector, streams); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData)); + final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization()); + final JsonObject playlistJson = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray allStreams = playlistJson.getArray("contents"); + // Sublist because youtube returns up to 24 previous streams in the mix + // +1 because the stream of "currentIndex" was already extracted in previous request + final List newStreams = + allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size()); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, newStreams); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrlFrom(playlistJson), page.getCookies())); } private void collectStreamsFrom( - @Nonnull StreamInfoItemsCollector collector, - @Nullable JsonArray streams) { + @Nonnull final StreamInfoItemsCollector collector, + @Nullable final List streams) { if (streams == null) { return; @@ -158,9 +178,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Object stream : streams) { + for (final Object stream : streams) { if (stream instanceof JsonObject) { - JsonObject streamInfo = ((JsonObject) stream) + final JsonObject streamInfo = ((JsonObject) stream) .getObject("playlistPanelVideoRenderer"); if (streamInfo != null) { collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); @@ -169,7 +189,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } } - private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException { + private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { final String videoId; if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); @@ -184,7 +204,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { return getThumbnailUrlFromVideoId(videoId); } - private String getThumbnailUrlFromVideoId(String videoId) { + private String getThumbnailUrlFromVideoId(final String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index b565fde62..096447879 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -14,82 +14,88 @@ import java.util.List; public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { - private static final YoutubePlaylistLinkHandlerFactory instance = new YoutubePlaylistLinkHandlerFactory(); + private static final YoutubePlaylistLinkHandlerFactory INSTANCE = + new YoutubePlaylistLinkHandlerFactory(); public static YoutubePlaylistLinkHandlerFactory getInstance() { - return instance; + return INSTANCE; } @Override - public String getUrl(String id, List contentFilters, String sortFilter) { + public String getUrl(final String id, final List contentFilters, + final String sortFilter) { return "https://www.youtube.com/playlist?list=" + id; } @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); + final URL urlObj = Utils.stringToURL(url); if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj))) { throw new ParsingException("the url given is not a Youtube-URL"); } - String path = urlObj.getPath(); + final String path = urlObj.getPath(); if (!path.equals("/watch") && !path.equals("/playlist")) { throw new ParsingException("the url given is neither a video nor a playlist URL"); } - String listID = Utils.getQueryValue(urlObj, "list"); + final String listID = Utils.getQueryValue(urlObj, "list"); if (listID == null) { throw new ParsingException("the url given does not include a playlist"); } if (!listID.matches("[a-zA-Z0-9_-]{10,}")) { - throw new ParsingException("the list-ID given in the URL does not match the list pattern"); + throw new ParsingException( + "the list-ID given in the URL does not match the list pattern"); } - // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists - if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) { - throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported"); + if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) { + throw new ContentNotSupportedException( + "YouTube Music Mix playlists are not yet supported"); } return listID; } catch (final Exception exception) { - throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); } } @Override public boolean onAcceptUrl(final String url) { try { - String playlistId = getId(url); - //Because youtube music mix are not supported yet. - return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId); + getId(url); } catch (ParsingException e) { return false; } + return true; } /** - * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like - * youtube.com/watch?v=videoId&list=playlistId + * * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is + * like + * https://youtube.com/watch?v=videoId&list=playlistId. *

Otherwise use super

*/ @Override - public ListLinkHandler fromUrl(String url) throws ParsingException { + public ListLinkHandler fromUrl(final String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); - String listID = Utils.getQueryValue(urlObj, "list"); + final URL urlObj = Utils.stringToURL(url); + final String listID = Utils.getQueryValue(urlObj, "list"); if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { String videoID = Utils.getQueryValue(urlObj, "v"); if (videoID == null) { videoID = listID.substring(2); } - String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; - return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), - getSortFilter(url)); + final String newUrl = "https://www.youtube.com/watch?v=" + videoID + + "&list=" + listID; + return new ListLinkHandler(new LinkHandler(url, newUrl, listID), + getContentFilter(url), + getSortFilter(url)); } } catch (MalformedURLException exception) { throw new ParsingException("Error could not parse url :" + exception.getMessage(), diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 959202700..3c2bc7d9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -8,6 +8,7 @@ import java.net.URL; import java.net.URLDecoder; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -260,4 +261,15 @@ public class Utils { } return stringBuilder.toString(); } + + public static String join(final String delimiter, final String mapJoin, + final Map elements) { + final List list = new LinkedList<>(); + for (final Map.Entry entry : elements + .entrySet()) { + list.add(entry.getKey() + mapJoin + entry.getValue()); + } + return join(delimiter, list); + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index b612827a7..8e41f12d8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -2,26 +2,45 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.startsWith; -import static org.junit.Assert.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import java.util.HashSet; +import java.util.Set; +import org.hamcrest.MatcherAssert; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.ChannelMix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Invalid; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Mix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MixWithIndex; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MyMix; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +@RunWith(Suite.class) +@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class}) public class YoutubeMixPlaylistExtractorTest { + public static final String PBJ = "&pbj=1"; + private static final String VIDEO_ID = "_AzeUSL9lZc"; + private static final String VIDEO_TITLE = + "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; + private static YoutubeMixPlaylistExtractor extractor; - private static String videoId = "_AzeUSL9lZc"; - private static String videoTitle = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; public static class Mix { @@ -29,8 +48,8 @@ public class YoutubeMixPlaylistExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); extractor.fetchPage(); } @@ -41,81 +60,82 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(videoTitle)); + assertThat(name, containsString(VIDEO_TITLE)); } @Test public void getThumbnailUrl() throws Exception { final String thumbnailUrl = extractor.getThumbnailUrl(); assertIsSecureUrl(thumbnailUrl); - assertThat(thumbnailUrl, containsString("yt")); - assertThat(thumbnailUrl, containsString(videoId)); - } - - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RD" + videoId)); + MatcherAssert.assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); } @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID + + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); - streams = extractor.getPage(streams.getNextPageUrl()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } public static class MixWithIndex { - private static String index = "&index=13"; - private static String videoIdNumber13 = "qHtzO49SDmk"; + private static final String INDEX = "&index=13"; + private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoIdNumber13 + "&list=RD" + videoId - + index); + "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX); extractor.fetchPage(); } @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(videoTitle)); + assertThat(name, containsString(VIDEO_TITLE)); } @Test @@ -123,47 +143,47 @@ public class YoutubeMixPlaylistExtractorTest { final String thumbnailUrl = extractor.getThumbnailUrl(); assertIsSecureUrl(thumbnailUrl); assertThat(thumbnailUrl, containsString("yt")); - assertThat(thumbnailUrl, containsString(videoId)); - } - - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RD" + videoId)); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); } @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } - streams = extractor.getPage(streams.getNextPageUrl()); + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } @@ -175,7 +195,8 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM" + + VIDEO_ID); extractor.fetchPage(); } @@ -186,7 +207,7 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertEquals("My Mix", name); } @@ -197,44 +218,44 @@ public class YoutubeMixPlaylistExtractorTest { assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc")); } - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RDMM" + videoId)); - } - @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = + extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + + "&list=RDMM" + VIDEO_ID + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } - @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); - streams = extractor.getPage(streams.getNextPageUrl()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } @@ -249,10 +270,10 @@ public class YoutubeMixPlaylistExtractorTest { @Test(expected = ExtractionException.class) public void getPageEmptyUrl() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); extractor.fetchPage(); - extractor.getPage(""); + extractor.getPage(new Page("")); } @Test(expected = ExtractionException.class) @@ -267,10 +288,9 @@ public class YoutubeMixPlaylistExtractorTest { public static class ChannelMix { - private static String channelId = "UCXuqSBlHAE6Xw-yeJA0Tunw"; - private static String videoIdOfChannel = "mnk6gnOBYIo"; - private static String channelTitle = "Linus Tech Tips"; - + private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw"; + private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo"; + private static final String CHANNEL_TITLE = "Linus Tech Tips"; @BeforeClass @@ -278,15 +298,16 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoIdOfChannel + "&list=RDCM" + channelId); + "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID); extractor.fetchPage(); } @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(channelTitle)); + assertThat(name, containsString(CHANNEL_TITLE)); } @Test @@ -296,30 +317,25 @@ public class YoutubeMixPlaylistExtractorTest { assertThat(thumbnailUrl, containsString("yt")); } - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RDCM" + channelId)); - } - @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } -} \ No newline at end of file +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 443a21adf..e7bec115d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -56,7 +56,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=RDCLAK5uy_ly6s4irLuZAcjEDwJmqcA_UtSipMyGgbQ")); // YouTube Music playlist - assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index 7b5d2ce73..4354fa258 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -62,14 +62,14 @@ public class YoutubeServiceTest { @Test public void getPlayListExtractorIsNormalPlaylist() throws Exception { - PlaylistExtractor extractor = service.getPlaylistExtractor( + final PlaylistExtractor extractor = service.getPlaylistExtractor( "https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS"); assertTrue(extractor instanceof YoutubePlaylistExtractor); } @Test public void getPlaylistExtractorIsMix() throws Exception { - String videoId = "_AzeUSL9lZc"; + final String videoId = "_AzeUSL9lZc"; PlaylistExtractor extractor = YouTube.getPlaylistExtractor( "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); From a338e4e08e385e741c18ce2d30ae1e5016599cf0 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 26 Sep 2020 11:22:24 +0200 Subject: [PATCH 13/14] [Youtube] Apply review suggestions and avoid channel mix edge case --- .../youtube/YoutubeParsingHelper.java | 35 +++++++++++++++++-- .../YoutubeMixPlaylistExtractor.java | 17 +++++---- .../YoutubePlaylistLinkHandlerFactory.java | 15 +++++--- .../YoutubeMixPlaylistExtractorTest.java | 34 ++++++++++-------- 4 files changed, 70 insertions(+), 31 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index ea8743069..06f421a69 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -214,13 +214,44 @@ public class YoutubeParsingHelper { /** * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) - * Ids from a YouTube Music Mix start with "RD" + * Ids from a YouTube Music Mix start with "RDAMVM" * @param playlistId * @return Whether given id belongs to a YouTube Music Mix */ public static boolean isYoutubeMusicMixId(final String playlistId) { return playlistId.startsWith("RDAMVM"); } + /** + * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) + * Ids from a YouTube channel Mix start with "RDCM" + * @return Whether given id belongs to a YouTube Channel Mix + */ + public static boolean isYoutubeChannelMixId(final String playlistId) { + return playlistId.startsWith("RDCM"); + } + + /** + * Extracts the video id from the playlist id for Mixes. + * @throws ParsingException If the playlistId is a Channel Mix or not a mix. + */ + public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException { + if (playlistId.startsWith("RDMM")) { //My Mix + return playlistId.substring(4); + + } else if (playlistId.startsWith("RDAMVM")) { //Music mix + return playlistId.substring(6); + + } else if (playlistId.startsWith("RMCM")) { //Channel mix + //Channel mix are build with RMCM{channelId}, so videoId can't be determined + throw new ParsingException("Video id could not be determined from mix id: " + playlistId); + + } else if (playlistId.startsWith("RD")) { // Normal mix + return playlistId.substring(2); + + } else { //not a mix + throw new ParsingException("Video id could not be determined from mix id: " + playlistId); + } + } public static JsonObject getInitialData(String html) throws ParsingException { try { @@ -362,7 +393,7 @@ public class YoutubeParsingHelper { .end() .value("query", "test") .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") - .end().done().getBytes(StandardCharsets.UTF_8); + .end().done().getBytes("UTF-8"); // @formatter:on Map> headers = new HashMap<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index d5e307453..d424129ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -28,6 +28,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /** * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). @@ -40,7 +41,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { * YouTube identifies mixes based on this cookie. With this information it can generate * continuations without duplicates. */ - private static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; + public static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; private JsonObject initialData; private JsonObject playlistData; @@ -124,11 +125,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, playlistData.getArray("contents")); return new InfoItemsPage<>(collector, - new Page(getNextPageUrl(), Collections.singletonMap(COOKIE_NAME, cookieValue))); - } - - private String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(playlistData); + new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue))); } private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { @@ -146,9 +143,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getPage(final Page page) throws ExtractionException, IOException { - if (page == null || page.getUrl().isEmpty()) { - throw new ExtractionException( - new IllegalArgumentException("Page url is empty or null")); + if (page == null || isNullOrEmpty(page.getUrl())) { + throw new IllegalArgumentException("Page url is empty or null"); + } + if (!page.getCookies().containsKey(COOKIE_NAME)) { + throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing"); } final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 096447879..aa2908e64 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,5 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -8,10 +11,6 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.List; - public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { private static final YoutubePlaylistLinkHandlerFactory INSTANCE = @@ -58,6 +57,12 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { "YouTube Music Mix playlists are not yet supported"); } + if (YoutubeParsingHelper.isYoutubeChannelMixId(listID) + && Utils.getQueryValue(urlObj, "v") == null) { + //Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId + throw new ContentNotSupportedException("Channel Mix without a video id are not supported"); + } + return listID; } catch (final Exception exception) { throw new ParsingException("Error could not parse url :" + exception.getMessage(), @@ -89,7 +94,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { String videoID = Utils.getQueryValue(urlObj, "v"); if (videoID == null) { - videoID = listID.substring(2); + videoID = YoutubeParsingHelper.extractVideoIdFromMixId(listID); } final String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 8e41f12d8..49fb3fe04 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -1,15 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.startsWith; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - +import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Set; import org.hamcrest.MatcherAssert; import org.junit.BeforeClass; @@ -31,6 +24,15 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractor import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + @RunWith(Suite.class) @SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class}) public class YoutubeMixPlaylistExtractorTest { @@ -41,6 +43,8 @@ public class YoutubeMixPlaylistExtractorTest { "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; private static YoutubeMixPlaylistExtractor extractor; + private static Map dummyCookie + = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); public static class Mix { @@ -83,8 +87,8 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { final InfoItemsPage streams = extractor.getPage( - new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID - + PBJ)); + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID + + PBJ, dummyCookie)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -157,7 +161,7 @@ public class YoutubeMixPlaylistExtractorTest { public void getPage() throws Exception { final InfoItemsPage streams = extractor.getPage( new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" - + VIDEO_ID + INDEX + PBJ)); + + VIDEO_ID + INDEX + PBJ, dummyCookie)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -229,7 +233,7 @@ public class YoutubeMixPlaylistExtractorTest { public void getPage() throws Exception { final InfoItemsPage streams = extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID - + "&list=RDMM" + VIDEO_ID + PBJ)); + + "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -267,7 +271,7 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); } - @Test(expected = ExtractionException.class) + @Test(expected = IllegalArgumentException.class) public void getPageEmptyUrl() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( @@ -328,7 +332,7 @@ public class YoutubeMixPlaylistExtractorTest { public void getPage() throws Exception { final InfoItemsPage streams = extractor.getPage( new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL - + "&list=RDCM" + CHANNEL_ID + PBJ)); + + "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } From f90f6fcf92b355f668656e9e277bdb2086d11ab5 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 12 Dec 2020 20:40:13 +0100 Subject: [PATCH 14/14] [YouTube] Don't escape & in getUrlFromNavigationEndpoint for playlists --- .../extractor/services/youtube/YoutubeParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 06f421a69..835ddd309 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -478,7 +478,7 @@ public class YoutubeParsingHelper { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") .getString("playlistId")); } if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) {