From 22d2f7e4006036aab8ac63478b791c4ab3de1624 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Thu, 16 Apr 2020 19:28:27 +0200 Subject: [PATCH] [Youtube] Add cookies to youtube mix request This way youtube wont return duplicates when getting more items of the mix (but youtube can also track us) --- .../youtube/YoutubeParsingHelper.java | 93 +++++++-- .../services/youtube/YoutubeService.java | 2 +- .../YoutubeMixPlaylistExtractor.java | 102 ++++++---- .../YoutubePlaylistLinkHandlerFactory.java | 52 +++--- .../schabi/newpipe/extractor/utils/Utils.java | 12 ++ .../YoutubeMixPlaylistExtractorTest.java | 176 ++++++++++-------- ...YoutubePlaylistLinkHandlerFactoryTest.java | 2 +- .../services/youtube/YoutubeServiceTest.java | 4 +- 8 files changed, 276 insertions(+), 167 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 4ac9724b2..ea8743069 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -5,8 +5,10 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -21,6 +23,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -35,6 +38,7 @@ import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.join; /* * Created by Christian Schabesberger on 02.03.16. @@ -61,6 +65,12 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + /** + * The official youtube app supports intents in this format, where after the ':' is the videoId. + * Accordingly there are other apps sharing streams in this format. + */ + public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube"; + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; @@ -193,22 +203,22 @@ public class YoutubeParsingHelper { } /** - * Checks if the given playlist id is a youtube mix (auto-generated playlist) - * Ids from a youtube mix start with "RD" + * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) + * Ids from a YouTube Mix start with "RD" * @param playlistId - * @return Whether given id belongs to a youtube mix + * @return Whether given id belongs to a YouTube Mix */ - public static boolean isYoutubeMixId(String playlistId) { + public static boolean isYoutubeMixId(final String playlistId) { return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); } /** - * Checks if the given playlist id is a youtube music mix (auto-generated playlist) - * Ids from a youtube music mix start with "RD" + * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) + * Ids from a YouTube Music Mix start with "RD" * @param playlistId - * @return Whether given id belongs to a youtube music mix + * @return Whether given id belongs to a YouTube Music Mix */ - public static boolean isYoutubeMusicMixId(String playlistId) { + public static boolean isYoutubeMusicMixId(final String playlistId) { return playlistId.startsWith("RDAMVM"); } @@ -352,7 +362,7 @@ public class YoutubeParsingHelper { .end() .value("query", "test") .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") - .end().done().getBytes("UTF-8"); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on Map> headers = new HashMap<>(); @@ -436,10 +446,14 @@ public class YoutubeParsingHelper { } else if (navigationEndpoint.has("watchEndpoint")) { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); - if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); - if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) - url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") + .getString("playlistId")); + } + if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint") + .getInt("startTimeSeconds")); + } return url.toString(); } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + @@ -467,7 +481,6 @@ public class YoutubeParsingHelper { if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); if (!isNullOrEmpty(url)) { - url = url.replaceAll("&", "&"); textBuilder.append("").append(text).append(""); continue; } @@ -506,8 +519,8 @@ public class YoutubeParsingHelper { public static String getValidJsonResponseBody(final Response response) throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); } final String responseBody = response.responseBody(); @@ -527,13 +540,39 @@ public class YoutubeParsingHelper { final String responseContentType = response.getHeader("Content-Type"); if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); } return responseBody; } + public static Response getResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(url, headers, localization); + getValidJsonResponseBody(response); + + return response; + } + + public static String extractCookieValue(final String cookieName, final Response response) { + final List cookies = response.responseHeaders().get("Set-Cookie"); + int startIndex; + String result = ""; + for (final String cookie : cookies) { + startIndex = cookie.indexOf(cookieName); + if (startIndex != -1) { + result = cookie.substring(startIndex + cookieName.length() + "=".length(), + cookie.indexOf(";", startIndex)); + } + } + return result; + } + public static JsonArray getJsonResponse(final String url, final Localization localization) throws IOException, ExtractionException { Map> headers = new HashMap<>(); @@ -541,8 +580,24 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); final Response response = getDownloader().get(url, headers, localization); - final String responseBody = getValidJsonResponseBody(response); + return toJsonArray(getValidJsonResponseBody(response)); + } + public static JsonArray getJsonResponse(final Page page, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + if (!isNullOrEmpty(page.getCookies())) { + headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies()))); + } + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(page.getUrl(), headers, localization); + + return toJsonArray(getValidJsonResponseBody(response)); + } + + public static JsonArray toJsonArray(final String responseBody) throws ParsingException { try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 10ac64404..997fd0b73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -110,7 +110,7 @@ public class YoutubeService extends StreamingService { } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { return new YoutubeMixPlaylistExtractor(this, linkHandler); } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 0bf1fcbe8..d5e307453 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -4,8 +4,10 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -15,34 +17,50 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; +import java.util.Collections; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray; /** - * A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of - * "youtube.com/watch?v=videoId&list=playlistId" + * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). + * It handles URLs in the format of + * {@code youtube.com/watch?v=videoId&list=playlistId} */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + /** + * YouTube identifies mixes based on this cookie. With this information it can generate + * continuations without duplicates. + */ + private static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; + private JsonObject initialData; private JsonObject playlistData; + private String cookieValue; - public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + public YoutubeMixPlaylistExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) - throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + final Response response = getResponse(url, getExtractorLocalization()); + final JsonArray ajaxJson = toJsonArray(response.responseBody()); initialData = ajaxJson.getObject(3).getObject("response"); playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("playlist").getObject("playlist"); + .getObject("playlist").getObject("playlist"); + cookieValue = extractCookieValue(COOKIE_NAME, response); } @Nonnull @@ -58,16 +76,15 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - final String playlistId = playlistData.getString("playlistId"); + return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId")); + } catch (final Exception e) { try { - return getThumbnailUrlFromPlaylistId(playlistId); - } catch (ParsingException e) { //fallback to thumbnail of current video. Always the case for channel mix return getThumbnailUrlFromVideoId( initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") .getString("videoId")); + } catch (final Exception ignored) { } - } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } } @@ -104,53 +121,56 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, playlistData.getArray("contents")); - return new InfoItemsPage<>(collector, getNextPageUrl()); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrl(), Collections.singletonMap(COOKIE_NAME, cookieValue))); } - @Override - public String getNextPageUrl() throws ExtractionException { + private String getNextPageUrl() throws ExtractionException { return getNextPageUrlFrom(playlistData); } - private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException { - final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") - .get(playlistData.getArray("contents").size() - 1)); + private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents") + .get(playlistJson.getArray("contents").size() - 1)); if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { throw new ExtractionException("Could not extract next page url"); } - //Index of video in mix is missing, but adding it doesn't appear to have any effect. - //And since the index needs to be tracked by us, it is left out + return getUrlFromNavigationEndpoint( - lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) - + "&pbj=1"; + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) + + "&pbj=1"; } @Override - public InfoItemsPage getPage(final String pageUrl) + public InfoItemsPage getPage(final Page page) throws ExtractionException, IOException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (page == null || page.getUrl().isEmpty()) { throw new ExtractionException( new IllegalArgumentException("Page url is empty or null")); } - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonObject playlistData = - ajaxJson.getObject(3).getObject("response").getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("playlist") - .getObject("playlist"); - final JsonArray streams = playlistData.getArray("contents"); - //Because continuation requests are created with the last video of previous request as start - streams.remove(0); - collectStreamsFrom(collector, streams); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData)); + final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization()); + final JsonObject playlistJson = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray allStreams = playlistJson.getArray("contents"); + // Sublist because youtube returns up to 24 previous streams in the mix + // +1 because the stream of "currentIndex" was already extracted in previous request + final List newStreams = + allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size()); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, newStreams); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrlFrom(playlistJson), page.getCookies())); } private void collectStreamsFrom( - @Nonnull StreamInfoItemsCollector collector, - @Nullable JsonArray streams) { + @Nonnull final StreamInfoItemsCollector collector, + @Nullable final List streams) { if (streams == null) { return; @@ -158,9 +178,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Object stream : streams) { + for (final Object stream : streams) { if (stream instanceof JsonObject) { - JsonObject streamInfo = ((JsonObject) stream) + final JsonObject streamInfo = ((JsonObject) stream) .getObject("playlistPanelVideoRenderer"); if (streamInfo != null) { collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); @@ -169,7 +189,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } } - private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException { + private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { final String videoId; if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); @@ -184,7 +204,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { return getThumbnailUrlFromVideoId(videoId); } - private String getThumbnailUrlFromVideoId(String videoId) { + private String getThumbnailUrlFromVideoId(final String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index b565fde62..096447879 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -14,82 +14,88 @@ import java.util.List; public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { - private static final YoutubePlaylistLinkHandlerFactory instance = new YoutubePlaylistLinkHandlerFactory(); + private static final YoutubePlaylistLinkHandlerFactory INSTANCE = + new YoutubePlaylistLinkHandlerFactory(); public static YoutubePlaylistLinkHandlerFactory getInstance() { - return instance; + return INSTANCE; } @Override - public String getUrl(String id, List contentFilters, String sortFilter) { + public String getUrl(final String id, final List contentFilters, + final String sortFilter) { return "https://www.youtube.com/playlist?list=" + id; } @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); + final URL urlObj = Utils.stringToURL(url); if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj))) { throw new ParsingException("the url given is not a Youtube-URL"); } - String path = urlObj.getPath(); + final String path = urlObj.getPath(); if (!path.equals("/watch") && !path.equals("/playlist")) { throw new ParsingException("the url given is neither a video nor a playlist URL"); } - String listID = Utils.getQueryValue(urlObj, "list"); + final String listID = Utils.getQueryValue(urlObj, "list"); if (listID == null) { throw new ParsingException("the url given does not include a playlist"); } if (!listID.matches("[a-zA-Z0-9_-]{10,}")) { - throw new ParsingException("the list-ID given in the URL does not match the list pattern"); + throw new ParsingException( + "the list-ID given in the URL does not match the list pattern"); } - // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists - if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) { - throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported"); + if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) { + throw new ContentNotSupportedException( + "YouTube Music Mix playlists are not yet supported"); } return listID; } catch (final Exception exception) { - throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); } } @Override public boolean onAcceptUrl(final String url) { try { - String playlistId = getId(url); - //Because youtube music mix are not supported yet. - return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId); + getId(url); } catch (ParsingException e) { return false; } + return true; } /** - * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like - * youtube.com/watch?v=videoId&list=playlistId + * * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is + * like + * https://youtube.com/watch?v=videoId&list=playlistId. *

Otherwise use super

*/ @Override - public ListLinkHandler fromUrl(String url) throws ParsingException { + public ListLinkHandler fromUrl(final String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); - String listID = Utils.getQueryValue(urlObj, "list"); + final URL urlObj = Utils.stringToURL(url); + final String listID = Utils.getQueryValue(urlObj, "list"); if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { String videoID = Utils.getQueryValue(urlObj, "v"); if (videoID == null) { videoID = listID.substring(2); } - String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; - return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), - getSortFilter(url)); + final String newUrl = "https://www.youtube.com/watch?v=" + videoID + + "&list=" + listID; + return new ListLinkHandler(new LinkHandler(url, newUrl, listID), + getContentFilter(url), + getSortFilter(url)); } } catch (MalformedURLException exception) { throw new ParsingException("Error could not parse url :" + exception.getMessage(), diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 959202700..3c2bc7d9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -8,6 +8,7 @@ import java.net.URL; import java.net.URLDecoder; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -260,4 +261,15 @@ public class Utils { } return stringBuilder.toString(); } + + public static String join(final String delimiter, final String mapJoin, + final Map elements) { + final List list = new LinkedList<>(); + for (final Map.Entry entry : elements + .entrySet()) { + list.add(entry.getKey() + mapJoin + entry.getValue()); + } + return join(delimiter, list); + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index b612827a7..8e41f12d8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -2,26 +2,45 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.startsWith; -import static org.junit.Assert.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import java.util.HashSet; +import java.util.Set; +import org.hamcrest.MatcherAssert; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.ChannelMix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Invalid; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Mix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MixWithIndex; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MyMix; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +@RunWith(Suite.class) +@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class}) public class YoutubeMixPlaylistExtractorTest { + public static final String PBJ = "&pbj=1"; + private static final String VIDEO_ID = "_AzeUSL9lZc"; + private static final String VIDEO_TITLE = + "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; + private static YoutubeMixPlaylistExtractor extractor; - private static String videoId = "_AzeUSL9lZc"; - private static String videoTitle = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; public static class Mix { @@ -29,8 +48,8 @@ public class YoutubeMixPlaylistExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); extractor.fetchPage(); } @@ -41,81 +60,82 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(videoTitle)); + assertThat(name, containsString(VIDEO_TITLE)); } @Test public void getThumbnailUrl() throws Exception { final String thumbnailUrl = extractor.getThumbnailUrl(); assertIsSecureUrl(thumbnailUrl); - assertThat(thumbnailUrl, containsString("yt")); - assertThat(thumbnailUrl, containsString(videoId)); - } - - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RD" + videoId)); + MatcherAssert.assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); } @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID + + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); - streams = extractor.getPage(streams.getNextPageUrl()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } public static class MixWithIndex { - private static String index = "&index=13"; - private static String videoIdNumber13 = "qHtzO49SDmk"; + private static final String INDEX = "&index=13"; + private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoIdNumber13 + "&list=RD" + videoId - + index); + "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX); extractor.fetchPage(); } @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(videoTitle)); + assertThat(name, containsString(VIDEO_TITLE)); } @Test @@ -123,47 +143,47 @@ public class YoutubeMixPlaylistExtractorTest { final String thumbnailUrl = extractor.getThumbnailUrl(); assertIsSecureUrl(thumbnailUrl); assertThat(thumbnailUrl, containsString("yt")); - assertThat(thumbnailUrl, containsString(videoId)); - } - - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RD" + videoId)); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); } @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } - streams = extractor.getPage(streams.getNextPageUrl()); + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } @@ -175,7 +195,8 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM" + + VIDEO_ID); extractor.fetchPage(); } @@ -186,7 +207,7 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertEquals("My Mix", name); } @@ -197,44 +218,44 @@ public class YoutubeMixPlaylistExtractorTest { assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc")); } - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RDMM" + videoId)); - } - @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = + extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + + "&list=RDMM" + VIDEO_ID + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } - @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); - streams = extractor.getPage(streams.getNextPageUrl()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } @@ -249,10 +270,10 @@ public class YoutubeMixPlaylistExtractorTest { @Test(expected = ExtractionException.class) public void getPageEmptyUrl() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); extractor.fetchPage(); - extractor.getPage(""); + extractor.getPage(new Page("")); } @Test(expected = ExtractionException.class) @@ -267,10 +288,9 @@ public class YoutubeMixPlaylistExtractorTest { public static class ChannelMix { - private static String channelId = "UCXuqSBlHAE6Xw-yeJA0Tunw"; - private static String videoIdOfChannel = "mnk6gnOBYIo"; - private static String channelTitle = "Linus Tech Tips"; - + private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw"; + private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo"; + private static final String CHANNEL_TITLE = "Linus Tech Tips"; @BeforeClass @@ -278,15 +298,16 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoIdOfChannel + "&list=RDCM" + channelId); + "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID); extractor.fetchPage(); } @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(channelTitle)); + assertThat(name, containsString(CHANNEL_TITLE)); } @Test @@ -296,30 +317,25 @@ public class YoutubeMixPlaylistExtractorTest { assertThat(thumbnailUrl, containsString("yt")); } - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RDCM" + channelId)); - } - @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } -} \ No newline at end of file +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 443a21adf..e7bec115d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -56,7 +56,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=RDCLAK5uy_ly6s4irLuZAcjEDwJmqcA_UtSipMyGgbQ")); // YouTube Music playlist - assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index 7b5d2ce73..4354fa258 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -62,14 +62,14 @@ public class YoutubeServiceTest { @Test public void getPlayListExtractorIsNormalPlaylist() throws Exception { - PlaylistExtractor extractor = service.getPlaylistExtractor( + final PlaylistExtractor extractor = service.getPlaylistExtractor( "https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS"); assertTrue(extractor instanceof YoutubePlaylistExtractor); } @Test public void getPlaylistExtractorIsMix() throws Exception { - String videoId = "_AzeUSL9lZc"; + final String videoId = "_AzeUSL9lZc"; PlaylistExtractor extractor = YouTube.getPlaylistExtractor( "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); assertTrue(extractor instanceof YoutubeMixPlaylistExtractor);