From c0afd5213a516c78b3e47877836c6deb5d1c0396 Mon Sep 17 00:00:00 2001 From: opusforlife2 <53176348+opusforlife2@users.noreply.github.com> Date: Wed, 11 Nov 2020 14:54:16 +0000 Subject: [PATCH 01/29] Update Invidious URL list in Link Handler Factory --- .../YoutubeStreamLinkHandlerFactory.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index efc06da2a..c37dff517 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -186,20 +186,18 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { case "WWW.INVIDIO.US": case "DEV.INVIDIO.US": case "INVIDIO.US": + case "VID.ENCRYPTIONIN.SPACE": case "INVIDIOUS.SNOPYTA.ORG": - case "FI.INVIDIOUS.SNOPYTA.ORG": case "YEWTU.BE": - case "INVIDIOUS.GGC-PROJECT.DE": - case "YT.MAISPUTAIN.OVH": - case "INVIDIOUS.13AD.DE": - case "INVIDIOUS.TOOT.KOELN": - case "INVIDIOUS.FDN.FR": - case "WATCH.NETTOHIKARI.COM": - case "INVIDIOUS.SNWMDS.NET": - case "INVIDIOUS.SNWMDS.ORG": - case "INVIDIOUS.SNWMDS.COM": - case "INVIDIOUS.SUNSETRAVENS.COM": - case "INVIDIOUS.GACHIRANGERS.COM": { // code-block for hooktube.com and Invidious instances + case "TUBE.CONNECT.CAFE": + case "INVIDIOUS.ZAPASHCANON.FR": + case "INVIDIOUS.KAVIN.ROCKS": + case "INVIDIOUS.TUBE": + case "INVIDIOUS.SITE": + case "INVIDIOUS.XYZ": + case "VID.MINT.LGBT": + case "INVIDIOU.SITE": + case "INVIDIOUS.FDN.FR": { // code-block for hooktube.com and Invidious instances if (path.equals("watch")) { String viewQueryValue = Utils.getQueryValue(url, "v"); if (viewQueryValue != null) { From 2174055ef8cc20b0059dc0e82680ceccc47b492e Mon Sep 17 00:00:00 2001 From: opusforlife2 <53176348+opusforlife2@users.noreply.github.com> Date: Wed, 11 Nov 2020 15:12:31 +0000 Subject: [PATCH 02/29] Update Invidious URL list in Parsing Helper --- .../youtube/YoutubeParsingHelper.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 8b924cf93..6ea588341 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -110,20 +110,18 @@ public class YoutubeParsingHelper { return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") + || host.equalsIgnoreCase("vid.encryptionin.space") || host.equalsIgnoreCase("invidious.snopyta.org") - || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("yewtu.be") - || host.equalsIgnoreCase("invidious.ggc-project.de") - || host.equalsIgnoreCase("yt.maisputain.ovh") - || host.equalsIgnoreCase("invidious.13ad.de") - || host.equalsIgnoreCase("invidious.toot.koeln") - || host.equalsIgnoreCase("invidious.fdn.fr") - || host.equalsIgnoreCase("watch.nettohikari.com") - || host.equalsIgnoreCase("invidious.snwmds.net") - || host.equalsIgnoreCase("invidious.snwmds.org") - || host.equalsIgnoreCase("invidious.snwmds.com") - || host.equalsIgnoreCase("invidious.sunsetravens.com") - || host.equalsIgnoreCase("invidious.gachirangers.com"); + || host.equalsIgnoreCase("tube.connect.cafe") + || host.equalsIgnoreCase("invidious.zapashcanon.fr") + || host.equalsIgnoreCase("invidious.kavin.rocks") + || host.equalsIgnoreCase("invidious.tube") + || host.equalsIgnoreCase("invidious.site") + || host.equalsIgnoreCase("invidious.xyz") + || host.equalsIgnoreCase("vid.mint.lgbt") + || host.equalsIgnoreCase("invidiou.site") + || host.equalsIgnoreCase("invidious.fdn.fr"); } /** From 8347e1495299475577b792358d109dc4b7e07f7c Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Tue, 17 Nov 2020 21:04:53 +0100 Subject: [PATCH 03/29] [YouTube] Fix playlist continuations extraction --- .../extractors/YoutubePlaylistExtractor.java | 32 +++++++++++-------- .../youtube/YoutubePlaylistExtractorTest.java | 9 ++++++ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index c55707233..55d760cb0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -190,9 +190,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return new InfoItemsPage<>(collector, null); } else if (contents.getObject(0).has("playlistVideoListRenderer")) { final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer"); - collectStreamsFrom(collector, videos.getArray("contents")); + final JsonArray videosArray = videos.getArray("contents"); + collectStreamsFrom(collector, videosArray); - nextPage = getNextPageFrom(videos.getArray("continuations")); + nextPage = getNextPageFrom(videosArray); } return new InfoItemsPage<>(collector, nextPage); @@ -207,24 +208,29 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization()); - final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("playlistVideoListContinuation"); + final JsonArray continuation = ajaxJson.getObject(1) + .getObject("response") + .getArray("onResponseReceivedActions") + .getObject(0) + .getObject("appendContinuationItemsAction") + .getArray("continuationItems"); - collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); + collectStreamsFrom(collector, continuation); - return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations"))); + return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); } - private Page getNextPageFrom(final JsonArray continuations) { - if (isNullOrEmpty(continuations)) { + private Page getNextPageFrom(final JsonArray contents) { + if (isNullOrEmpty(contents)) { return null; } - final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); - final String continuation = nextContinuationData.getString("continuation"); - final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams); + final String continuation = contents.getObject(contents.size() - 1) + .getObject("continuationItemRenderer") + .getObject("continuationEndpoint") + .getObject("continuationCommand") + .getString("token"); + return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation); } private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 2b03579f9..a67884dd1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -3,6 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; @@ -11,6 +14,10 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -23,6 +30,8 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; /** * Test for {@link YoutubePlaylistExtractor} */ +@RunWith(Suite.class) +@SuiteClasses({NotAvailable.class, TimelessPopHits.class, HugePlaylist.class, LearningPlaylist.class}) public class YoutubePlaylistExtractorTest { public static class NotAvailable { From 5bceff0083f8b9ab40cc83c2a1d64e880052d2a9 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Wed, 18 Nov 2020 19:03:12 +0100 Subject: [PATCH 04/29] [YouTube] Fix extraction of next page url for the last page of playlist --- .../extractors/YoutubePlaylistExtractor.java | 17 +++++--- .../youtube/YoutubePlaylistExtractorTest.java | 42 +++++++++++++++++-- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 55d760cb0..1cc29eb63 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -225,12 +225,17 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return null; } - final String continuation = contents.getObject(contents.size() - 1) - .getObject("continuationItemRenderer") - .getObject("continuationEndpoint") - .getObject("continuationCommand") - .getString("token"); - return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation); + final JsonObject lastElement = contents.getObject(contents.size() - 1); + if (lastElement.has("continuationItemRenderer")) { + final String continuation = lastElement + .getObject("continuationItemRenderer") + .getObject("continuationEndpoint") + .getObject("continuationCommand") + .getString("token"); + return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation); + } else { + return null; + } } private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index a67884dd1..4a68f4a0a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -14,6 +14,7 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests; import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist; import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist; import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable; @@ -21,7 +22,9 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTes import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -31,7 +34,8 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link YoutubePlaylistExtractor} */ @RunWith(Suite.class) -@SuiteClasses({NotAvailable.class, TimelessPopHits.class, HugePlaylist.class, LearningPlaylist.class}) +@SuiteClasses({NotAvailable.class, TimelessPopHits.class, HugePlaylist.class, + LearningPlaylist.class, ContinuationsTests.class}) public class YoutubePlaylistExtractorTest { public static class NotAvailable { @@ -123,7 +127,7 @@ public class YoutubePlaylistExtractorTest { @Ignore @Test - public void testBannerUrl() throws Exception { + public void testBannerUrl() { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); assertTrue(bannerUrl, bannerUrl.contains("yt")); @@ -236,7 +240,7 @@ public class YoutubePlaylistExtractorTest { @Ignore @Test - public void testBannerUrl() throws Exception { + public void testBannerUrl() { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); assertTrue(bannerUrl, bannerUrl.contains("yt")); @@ -333,7 +337,7 @@ public class YoutubePlaylistExtractorTest { @Ignore @Test - public void testBannerUrl() throws Exception { + public void testBannerUrl() { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); assertTrue(bannerUrl, bannerUrl.contains("yt")); @@ -361,4 +365,34 @@ public class YoutubePlaylistExtractorTest { assertTrue("Error in the streams count", extractor.getStreamCount() > 40); } } + + public static class ContinuationsTests { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testNoContinuations() throws Exception { + final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/playlist?list=PLXJg25X-OulsVsnvZ7RVtSDW-id9_RzAO"); + extractor.fetchPage(); + + assertNoMoreItems(extractor); + } + + @Test + public void testOnlySingleContinuation() throws Exception { + final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX"); + extractor.fetchPage(); + + final ListExtractor.InfoItemsPage page = defaultTestMoreItems( + extractor); + assertFalse("More items available when it shouldn't", page.hasNextPage()); + } + } } From 334e1e9b53eb3a41a26be9760a937eddb3e9b0ab Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 20 Nov 2020 19:53:53 +0100 Subject: [PATCH 05/29] Fix YouTube comments test --- .../services/youtube/YoutubeCommentsExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 62da50413..55669b68e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -30,7 +30,7 @@ public class YoutubeCommentsExtractorTest { private static final String url = "https://www.youtube.com/watch?v=D00Au7k3i6o"; private static YoutubeCommentsExtractor extractor; - private static final String commentContent = "sub 4 sub"; + private static final String commentContent = "Category: Education"; @BeforeClass public static void setUp() throws Exception { From 5a3f96b967bbd97bb6ccd58dbfa15e9ecc648c30 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 22 Nov 2020 22:29:08 +0100 Subject: [PATCH 06/29] Release 0.20.4 --- README.md | 2 +- build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1911c7531..cbc5d9c1b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.3'`the `dependencies` in your `build.gradle`. Replace `v0.20.3` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.4'`the `dependencies` in your `build.gradle`. Replace `v0.20.4` with the latest release. **Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. diff --git a/build.gradle b/build.gradle index c20b35c97..2e2e63489 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.8 targetCompatibility = 1.8 - version 'v0.20.3' + version 'v0.20.4' group 'com.github.TeamNewPipe' repositories { From 175df679e05b24b6094570d719cc11f8dfc17c68 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 1 Dec 2020 08:18:29 +0100 Subject: [PATCH 07/29] Release 0.20.5 --- README.md | 2 +- build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cbc5d9c1b..076123d9f 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.4'`the `dependencies` in your `build.gradle`. Replace `v0.20.4` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.5'`the `dependencies` in your `build.gradle`. Replace `v0.20.5` with the latest release. **Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. diff --git a/build.gradle b/build.gradle index 2e2e63489..a9a217fbb 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.8 targetCompatibility = 1.8 - version 'v0.20.4' + version 'v0.20.5' group 'com.github.TeamNewPipe' repositories { From ba3e2302bc3a6f6688a87fe67bf3588b3c5c343f Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Tue, 1 Dec 2020 23:54:27 +0100 Subject: [PATCH 08/29] Fix name of media.ccc.de service --- README.md | 2 +- .../newpipe/extractor/services/media_ccc/MediaCCCService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 076123d9f..57a1c18e1 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ The following sites are currently supported: - YouTube - SoundCloud -- MediaCCC +- media.ccc.de - PeerTube (no P2P) ## License diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 523584e60..24dd20ba3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -32,7 +32,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap public class MediaCCCService extends StreamingService { public MediaCCCService(final int id) { - super(id, "MediaCCC", asList(AUDIO, VIDEO)); + super(id, "media.ccc.de", asList(AUDIO, VIDEO)); } @Override From 9dbacbc6188d88f540870be8f6b652b3e95ccc2d Mon Sep 17 00:00:00 2001 From: vkay94 Date: Wed, 9 Dec 2020 14:57:38 +0100 Subject: [PATCH 09/29] Fix comments parsing --- .../extractors/YoutubeCommentsExtractor.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index bd2af3c67..6b5a992c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -160,8 +160,15 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private String findValue(String doc, String start, String end) { - final int beginIndex = doc.indexOf(start) + start.length(); - final int endIndex = doc.indexOf(end, beginIndex); - return doc.substring(beginIndex, endIndex); + final String unescaped = doc + .replaceAll("\\\\x22", "\"") + .replaceAll("\\\\x7b", "{") + .replaceAll("\\\\x7d", "}") + .replaceAll("\\\\x5b", "[") + .replaceAll("\\\\x5d", "]"); + + final int beginIndex = unescaped.indexOf(start) + start.length(); + final int endIndex = unescaped.indexOf(end, beginIndex); + return unescaped.substring(beginIndex, endIndex); } } From b3835bd616ab28b861c83dcefd56e1754c6d20be Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 9 Dec 2020 23:30:23 +0100 Subject: [PATCH 10/29] Release 0.20.6 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57a1c18e1..642cdf8c1 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.5'`the `dependencies` in your `build.gradle`. Replace `v0.20.5` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.6'`the `dependencies` in your `build.gradle`. Replace `v0.20.6` with the latest release. **Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. From b8f64595a24cbd41f5d19f8e72e2c959f6787606 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Mon, 19 Oct 2020 16:52:21 +0530 Subject: [PATCH 11/29] Use Objects' static equals() and hashCode() methods. --- .../newpipe/extractor/localization/Localization.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java index 02b890870..a3af889ac 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; public class Localization implements Serializable { public static final Localization DEFAULT = new Localization("en", "GB"); @@ -89,14 +90,14 @@ public class Localization implements Serializable { Localization that = (Localization) o; - if (!languageCode.equals(that.languageCode)) return false; - return countryCode != null ? countryCode.equals(that.countryCode) : that.countryCode == null; + return languageCode.equals(that.languageCode) && + Objects.equals(countryCode, that.countryCode); } @Override public int hashCode() { int result = languageCode.hashCode(); - result = 31 * result + (countryCode != null ? countryCode.hashCode() : 0); + result = 31 * result + Objects.hashCode(countryCode); return result; } } From 4c19a88612ec315cd1234551f70ce168a3a6cb48 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Mon, 19 Oct 2020 17:17:41 +0530 Subject: [PATCH 12/29] Use Objects.toString(). --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 608ae47cd..5eaf80852 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -55,6 +55,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; @@ -861,7 +862,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } finally { Context.exit(); } - return result == null ? "" : result.toString(); + return Objects.toString(result, ""); } /*////////////////////////////////////////////////////////////////////////// From 57be1f1123e7b3707ab8a98a0a4cddfd1dcce4c1 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Mon, 19 Oct 2020 17:43:34 +0530 Subject: [PATCH 13/29] Use Objects.requireNonNull(). --- .../java/org/schabi/newpipe/extractor/Extractor.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index df1c22617..9389e255d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.util.Objects; public abstract class Extractor { /** @@ -29,12 +30,9 @@ public abstract class Extractor { private final Downloader downloader; public Extractor(final StreamingService service, final LinkHandler linkHandler) { - if (service == null) throw new NullPointerException("service is null"); - if (linkHandler == null) throw new NullPointerException("LinkHandler is null"); - this.service = service; - this.linkHandler = linkHandler; - this.downloader = NewPipe.getDownloader(); - if (downloader == null) throw new NullPointerException("downloader is null"); + this.service = Objects.requireNonNull(service, "service is null"); + this.linkHandler = Objects.requireNonNull(linkHandler, "LinkHandler is null"); + this.downloader = Objects.requireNonNull(NewPipe.getDownloader(), "downloader is null"); } /** From 0efb854d27deeee0f9f85d6f5537aa137aeb07c0 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sun, 2 Feb 2020 14:19:48 +0100 Subject: [PATCH 14/29] [Youtube] Implement mix extractor for auto-generated playlists. -New YoutubeMixPlaylistExtractor, that extracts from a mix (auto-generated playlist). -The url has the format of "youtube.com/watch?v=videoID&playlistID", where playlistID always starts with "RD" and usually followed by the videoID. -Change YoutubePlaylistLinkHandlerFactory to create a linkhandler with the given url if it is a mix. -Change YoutubeService to return YoutubeMixPlaylistExtractor if the url is a mix. --- .../youtube/YoutubeParsingHelper.java | 4 + .../services/youtube/YoutubeService.java | 8 +- .../YoutubeMixPlaylistExtractor.java | 196 ++++++++++++++++++ .../YoutubePlaylistLinkHandlerFactory.java | 21 ++ 4 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 6ea588341..42420a1a7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -192,6 +192,10 @@ public class YoutubeParsingHelper { } } + public static boolean isYoutubeMixId(String playlistId) { + return playlistId.startsWith("RD"); + } + public static JsonObject getInitialData(String html) throws ParsingException { try { try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 519672141..7d7a83eba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -110,7 +110,11 @@ public class YoutubeService extends StreamingService { @Override public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { - return new YoutubePlaylistExtractor(this, linkHandler); + if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { + return new YoutubeMixPlaylistExtractor(this, linkHandler); + } else { + return new YoutubePlaylistExtractor(this, linkHandler); + } } @Override @@ -140,7 +144,7 @@ public class YoutubeService extends StreamingService { public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) - throws ExtractionException { + throws ExtractionException { return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java new file mode 100644 index 000000000..69b3fb987 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -0,0 +1,196 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import java.io.IOException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + + private Document doc; + + public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String url = getUrl(); + final Response response = downloader.get(url, getExtractorLocalization()); + doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + try { + return doc.select("div[class=\"playlist-info\"] h3[class=\"playlist-title\"]").first().text(); + } catch (Exception e) { + throw new ParsingException("Could not get playlist name", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return doc.select("ol[class*=\"playlist-videos-list\"] li").first().attr("data-thumbnail-url"); + } catch (Exception e) { + throw new ParsingException("Could not get playlist thumbnail", e); + } + } + + @Override + public String getBannerUrl() { + return ""; + } + + @Override + public String getUploaderUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderName() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderAvatarUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public long getStreamCount() { + // Auto-generated playlist always start with 25 videos and are endless + // But the html doesn't have a continuation url + return doc.select("ol[class*=\"playlist-videos-list\"] li").size(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + Element ol = doc.select("ol[class*=\"playlist-videos-list\"]").first(); + collectStreamsFrom(collector, ol); + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() { + return ""; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) { + //Continuations are not implemented + return null; + } + + private void collectStreamsFrom( + @Nonnull StreamInfoItemsCollector collector, + @Nullable Element element) { + collector.reset(); + + if (element == null) { + return; + } + + final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (final Element li : element.children()) { + + collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { + + @Override + public boolean isAd() { + return false; + } + + @Override + public String getUrl() throws ParsingException { + try { + return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getName() throws ParsingException { + try { + return li.attr("data-video-title"); + } catch (Exception e) { + throw new ParsingException("Could not get name", e); + } + } + + @Override + public long getDuration() throws ParsingException { + //Not present in doc + return 0; + } + + @Override + public String getUploaderName() throws ParsingException { + try { + return li.select( + "div[class=\"playlist-video-description\"]" + + "span[class=\"video-uploader-byline\"]") + .first() + .text(); + } catch (Exception e) { + throw new ParsingException("Could not get uploader", e); + } + } + + @Override + public String getUploaderUrl() { + //Not present in doc + return ""; + } + + @Override + public String getTextualUploadDate() { + //Not present in doc + return ""; + } + + @Override + public long getViewCount() { + return -1; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId() + + "/hqdefault.jpg"; + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 56abc194b..3d0ba78a5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -2,10 +2,13 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; +import java.net.MalformedURLException; import java.net.URL; import java.util.List; @@ -67,4 +70,22 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { } return true; } + + @Override + public ListLinkHandler fromUrl(String url) throws ParsingException { + try { + URL urlObj = Utils.stringToURL(url); + String listID = Utils.getQueryValue(urlObj, "list"); + if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { + String videoID = Utils.getQueryValue(urlObj, "v"); + String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; + return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), + getSortFilter(url)); + } + } catch (MalformedURLException exception) { + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); + } + return super.fromUrl(url); + } } From d74265c846854263088ee69dfca551b2f70493d8 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sun, 2 Feb 2020 18:04:15 +0100 Subject: [PATCH 15/29] [Youtube] Extract getThumbnailUrl into method and change getUploaderName --- .../YoutubeMixPlaylistExtractor.java | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 69b3fb987..64281931c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,12 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.StreamingService; @@ -50,7 +46,16 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - return doc.select("ol[class*=\"playlist-videos-list\"] li").first().attr("data-thumbnail-url"); + Element li = doc.select("ol[class*=\"playlist-videos-list\"] li").first(); + String videoId = li.attr("data-video-id"); + if (videoId != null && !videoId.isEmpty()) { + //higher quality + return getThumbnailUrlFromId(videoId); + } else { + //lower quality + return doc.select("ol[class*=\"playlist-videos-list\"] li").first() + .attr("data-thumbnail-url"); + } } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -146,21 +151,18 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } @Override - public long getDuration() throws ParsingException { + public long getDuration() { //Not present in doc return 0; } @Override public String getUploaderName() throws ParsingException { - try { - return li.select( - "div[class=\"playlist-video-description\"]" - + "span[class=\"video-uploader-byline\"]") - .first() - .text(); - } catch (Exception e) { - throw new ParsingException("Could not get uploader", e); + String uploaderName = li.attr("data-video-username"); + if (uploaderName == null || uploaderName.isEmpty()) { + throw new ParsingException("Could not get uploader name"); + } else { + return uploaderName; } } @@ -184,8 +186,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId() - + "/hqdefault.jpg"; + return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -193,4 +194,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { }); } } + + private String getThumbnailUrlFromId(String videoId) { + return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + } } From 327a5730a883aa6725a629763a3df0cebda30d03 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sun, 2 Feb 2020 18:15:47 +0100 Subject: [PATCH 16/29] [Youtube] Add some comments to mix --- .../extractor/services/youtube/YoutubeParsingHelper.java | 6 ++++++ .../youtube/extractors/YoutubeMixPlaylistExtractor.java | 4 ++++ .../linkHandler/YoutubePlaylistLinkHandlerFactory.java | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 42420a1a7..258799692 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -192,6 +192,12 @@ public class YoutubeParsingHelper { } } + /** + * Checks if the given playlist id is a mix (auto-generated playlist) + * Ids from a mix start with "RD" + * @param playlistId + * @return Whether given id belongs to a mix + */ public static boolean isYoutubeMixId(String playlistId) { return playlistId.startsWith("RD"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 64281931c..9da2e98fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -18,6 +18,10 @@ import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingH import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +/** + * A YoutubePlaylistExtractor for a mix (auto-generated playlist). + * It handles urls in the format of "youtube.com/watch?v=videoId&list=playlistId" + */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { private Document doc; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 3d0ba78a5..ae66660c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -71,6 +71,14 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { return true; } + /** + * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like + * youtube.com/watch?v=videoId&list=playlistId + *

Otherwise use super

+ * @param url + * @return + * @throws ParsingException + */ @Override public ListLinkHandler fromUrl(String url) throws ParsingException { try { From a376792a5d557e4d0e9101147af6fe49e72d4c3f Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Tue, 4 Feb 2020 13:49:43 +0100 Subject: [PATCH 17/29] [Youtube] Handle case where url is in "youtube.com/playlist?list=listID" format. This occurs when sharing a mix from the official youtube app. --- .../linkHandler/YoutubePlaylistLinkHandlerFactory.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index ae66660c3..787372dcb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -75,9 +75,6 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like * youtube.com/watch?v=videoId&list=playlistId *

Otherwise use super

- * @param url - * @return - * @throws ParsingException */ @Override public ListLinkHandler fromUrl(String url) throws ParsingException { @@ -86,6 +83,9 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { String listID = Utils.getQueryValue(urlObj, "list"); if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { String videoID = Utils.getQueryValue(urlObj, "v"); + if (videoID == null) { + videoID = listID.substring(2); + } String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), getSortFilter(url)); From 0ff054acb440ff310c543f600b32b594338873c2 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Fri, 6 Mar 2020 20:40:40 +0100 Subject: [PATCH 18/29] [Youtube] Extract initial playlist info --- .../YoutubeMixPlaylistExtractor.java | 156 ++++++++++-------- 1 file changed, 83 insertions(+), 73 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 9da2e98fd..e9bc81623 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,5 +1,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -24,6 +28,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + private JsonObject initialData; private Document doc; public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -32,9 +37,14 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl(); - final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + final String url = getUrl() + "&pbj=1"; + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + initialData = ajaxJson.getObject(3).getObject("response"); + JsonObject a = initialData.getObject("contents"); + JsonObject b = initialData.getObject("contents").getObject("twoColumnWatchNextResults"); + JsonObject c = initialData.getObject("contents"); + JsonObject playlist = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); + System.out.println(); } @Nonnull @@ -127,76 +137,76 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Element li : element.children()) { - - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - - @Override - public boolean isAd() { - return false; - } - - @Override - public String getUrl() throws ParsingException { - try { - return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getName() throws ParsingException { - try { - return li.attr("data-video-title"); - } catch (Exception e) { - throw new ParsingException("Could not get name", e); - } - } - - @Override - public long getDuration() { - //Not present in doc - return 0; - } - - @Override - public String getUploaderName() throws ParsingException { - String uploaderName = li.attr("data-video-username"); - if (uploaderName == null || uploaderName.isEmpty()) { - throw new ParsingException("Could not get uploader name"); - } else { - return uploaderName; - } - } - - @Override - public String getUploaderUrl() { - //Not present in doc - return ""; - } - - @Override - public String getTextualUploadDate() { - //Not present in doc - return ""; - } - - @Override - public long getViewCount() { - return -1; - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); - } +// for (final Element li : element.children()) { +// +// collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { +// +// @Override +// public boolean isAd() { +// return false; +// } +// +// @Override +// public String getUrl() throws ParsingException { +// try { +// return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); +// } catch (Exception e) { +// throw new ParsingException("Could not get web page url for the video", e); +// } +// } +// +// @Override +// public String getName() throws ParsingException { +// try { +// return li.attr("data-video-title"); +// } catch (Exception e) { +// throw new ParsingException("Could not get name", e); +// } +// } +// +// @Override +// public long getDuration() { +// //Not present in doc +// return 0; +// } +// +// @Override +// public String getUploaderName() throws ParsingException { +// String uploaderName = li.attr("data-video-username"); +// if (uploaderName == null || uploaderName.isEmpty()) { +// throw new ParsingException("Could not get uploader name"); +// } else { +// return uploaderName; +// } +// } +// +// @Override +// public String getUploaderUrl() { +// //Not present in doc +// return ""; +// } +// +// @Override +// public String getTextualUploadDate() { +// //Not present in doc +// return ""; +// } +// +// @Override +// public long getViewCount() { +// return -1; +// } +// +// @Override +// public String getThumbnailUrl() throws ParsingException { +// try { +// return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); +// } catch (Exception e) { +// throw new ParsingException("Could not get thumbnail url", e); +// } +// } +// }); +// } } private String getThumbnailUrlFromId(String videoId) { From ec6b99c082267e2df609cac46144fc4f41109591 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 7 Mar 2020 08:42:26 +0100 Subject: [PATCH 19/29] [Youtube] Adjust mix extractor to new user agent Also extract continuation now --- .../YoutubeMixPlaylistExtractor.java | 140 ++++++------------ 1 file changed, 43 insertions(+), 97 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index e9bc81623..af200a9a8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import com.grack.nanojson.JsonArray; @@ -29,7 +30,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { private JsonObject initialData; - private Document doc; + private JsonObject playlistData; public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -40,10 +41,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final String url = getUrl() + "&pbj=1"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(3).getObject("response"); - JsonObject a = initialData.getObject("contents"); - JsonObject b = initialData.getObject("contents").getObject("twoColumnWatchNextResults"); - JsonObject c = initialData.getObject("contents"); - JsonObject playlist = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); + playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); System.out.println(); } @@ -51,7 +49,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getName() throws ParsingException { try { - return doc.select("div[class=\"playlist-info\"] h3[class=\"playlist-title\"]").first().text(); + final String name = playlistData.getString("title"); + if (name!= null) return name; + else return ""; } catch (Exception e) { throw new ParsingException("Could not get playlist name", e); } @@ -60,16 +60,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - Element li = doc.select("ol[class*=\"playlist-videos-list\"] li").first(); - String videoId = li.attr("data-video-id"); - if (videoId != null && !videoId.isEmpty()) { - //higher quality - return getThumbnailUrlFromId(videoId); - } else { - //lower quality - return doc.select("ol[class*=\"playlist-videos-list\"] li").first() - .attr("data-thumbnail-url"); - } + final String videoId = playlistData.getArray("contents").getObject(0).getString("videoId"); + return getThumbnailUrlFromId(videoId); } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -101,112 +93,66 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public long getStreamCount() { // Auto-generated playlist always start with 25 videos and are endless - // But the html doesn't have a continuation url - return doc.select("ol[class*=\"playlist-videos-list\"] li").size(); + return 25; } @Nonnull @Override public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Element ol = doc.select("ol[class*=\"playlist-videos-list\"]").first(); - collectStreamsFrom(collector, ol); + collectStreamsFrom(collector, playlistData.getArray("contents")); return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override public String getNextPageUrl() { - return ""; + final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") + .get(playlistData.getArray("contents").size() - 1)); + final String lastStreamId = lastStream.getObject("playlistPanelVideoRenderer") + .getString("videoId"); + return "https://youtube.com" + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint") + .getObject("commandMetadata").getObject("webCommandMetadata").getString("url") + "&pbj=1"; } @Override - public InfoItemsPage getPage(final String pageUrl) { - //Continuations are not implemented - return null; + public InfoItemsPage getPage(final String pageUrl) + throws ExtractionException, IOException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + playlistData = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray streams = playlistData.getArray("contents"); + //Because continuation requests are created with the last video of previous request as start + streams.remove(0); + collectStreamsFrom(collector, streams); + return new InfoItemsPage<>(collector, getNextPageUrl()); } private void collectStreamsFrom( @Nonnull StreamInfoItemsCollector collector, - @Nullable Element element) { + @Nullable JsonArray streams) { collector.reset(); - if (element == null) { + if (streams == null) { return; } - final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); -// for (final Element li : element.children()) { -// -// collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { -// -// @Override -// public boolean isAd() { -// return false; -// } -// -// @Override -// public String getUrl() throws ParsingException { -// try { -// return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); -// } catch (Exception e) { -// throw new ParsingException("Could not get web page url for the video", e); -// } -// } -// -// @Override -// public String getName() throws ParsingException { -// try { -// return li.attr("data-video-title"); -// } catch (Exception e) { -// throw new ParsingException("Could not get name", e); -// } -// } -// -// @Override -// public long getDuration() { -// //Not present in doc -// return 0; -// } -// -// @Override -// public String getUploaderName() throws ParsingException { -// String uploaderName = li.attr("data-video-username"); -// if (uploaderName == null || uploaderName.isEmpty()) { -// throw new ParsingException("Could not get uploader name"); -// } else { -// return uploaderName; -// } -// } -// -// @Override -// public String getUploaderUrl() { -// //Not present in doc -// return ""; -// } -// -// @Override -// public String getTextualUploadDate() { -// //Not present in doc -// return ""; -// } -// -// @Override -// public long getViewCount() { -// return -1; -// } -// -// @Override -// public String getThumbnailUrl() throws ParsingException { -// try { -// return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); -// } catch (Exception e) { -// throw new ParsingException("Could not get thumbnail url", e); -// } -// } -// }); -// } + for (Object stream : streams) { + if (stream instanceof JsonObject) { + JsonObject streamInfo = ((JsonObject) stream).getObject("playlistPanelVideoRenderer"); + if (streamInfo != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + } + } + } } private String getThumbnailUrlFromId(String videoId) { From 68a3948af6dbd5795c061ff874df67ed1d19cb78 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 7 Mar 2020 15:54:17 +0100 Subject: [PATCH 20/29] [Youtube] Fix get banner url --- .../youtube/extractors/YoutubeMixPlaylistExtractor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index af200a9a8..6687eb844 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -60,7 +60,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - final String videoId = playlistData.getArray("contents").getObject(0).getString("videoId"); + final String videoId = playlistData.getArray("contents").getObject(0) + .getObject("playlistPanelVideoRenderer").getString("videoId"); + if (videoId == null || videoId.isEmpty()) throw new ParsingException(""); return getThumbnailUrlFromId(videoId); } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); From df38b1926c8a5fb9fb510dd15ff75b5f13897877 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 7 Mar 2020 16:03:12 +0100 Subject: [PATCH 21/29] [Youtube] Add tests and take thumbnail image always from first video of mix Also fix getThumbnailUrl for "My Mix" --- .../YoutubeMixPlaylistExtractor.java | 285 ++++++++-------- .../YoutubeMixPlaylistExtractorTest.java | 323 ++++++++++++++++++ ...YoutubePlaylistLinkHandlerFactoryTest.java | 19 ++ .../services/youtube/YoutubeServiceTest.java | 24 ++ 4 files changed, 518 insertions(+), 133 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 6687eb844..f4868a61e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import com.grack.nanojson.JsonArray; @@ -8,156 +7,176 @@ import com.grack.nanojson.JsonObject; import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; /** - * A YoutubePlaylistExtractor for a mix (auto-generated playlist). - * It handles urls in the format of "youtube.com/watch?v=videoId&list=playlistId" + * A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of + * "youtube.com/watch?v=videoId&list=playlistId" */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { - private JsonObject initialData; - private JsonObject playlistData; + private final static String CONTENTS = "contents"; + private final static String RESPONSE = "response"; + private final static String PLAYLIST = "playlist"; + private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults"; + private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer"; - public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { - super(service, linkHandler); - } + private JsonObject playlistData; - @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + "&pbj=1"; - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - initialData = ajaxJson.getObject(3).getObject("response"); - playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); - System.out.println(); - } - - @Nonnull - @Override - public String getName() throws ParsingException { - try { - final String name = playlistData.getString("title"); - if (name!= null) return name; - else return ""; - } catch (Exception e) { - throw new ParsingException("Could not get playlist name", e); - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - final String videoId = playlistData.getArray("contents").getObject(0) - .getObject("playlistPanelVideoRenderer").getString("videoId"); - if (videoId == null || videoId.isEmpty()) throw new ParsingException(""); - return getThumbnailUrlFromId(videoId); - } catch (Exception e) { - throw new ParsingException("Could not get playlist thumbnail", e); - } - } - - @Override - public String getBannerUrl() { - return ""; - } - - @Override - public String getUploaderUrl() { - //Youtube mix are auto-generated - return ""; - } - - @Override - public String getUploaderName() { - //Youtube mix are auto-generated - return ""; - } - - @Override - public String getUploaderAvatarUrl() { - //Youtube mix are auto-generated - return ""; - } - - @Override - public long getStreamCount() { - // Auto-generated playlist always start with 25 videos and are endless - return 25; - } - - @Nonnull - @Override - public InfoItemsPage getInitialPage() { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, playlistData.getArray("contents")); - return new InfoItemsPage<>(collector, getNextPageUrl()); - } - - @Override - public String getNextPageUrl() { - final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") - .get(playlistData.getArray("contents").size() - 1)); - final String lastStreamId = lastStream.getObject("playlistPanelVideoRenderer") - .getString("videoId"); - return "https://youtube.com" + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint") - .getObject("commandMetadata").getObject("webCommandMetadata").getString("url") + "&pbj=1"; - } - - @Override - public InfoItemsPage getPage(final String pageUrl) - throws ExtractionException, IOException { - if (pageUrl == null || pageUrl.isEmpty()) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); } - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - playlistData = - ajaxJson.getObject(3).getObject("response").getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("playlist") - .getObject("playlist"); - final JsonArray streams = playlistData.getArray("contents"); - //Because continuation requests are created with the last video of previous request as start - streams.remove(0); - collectStreamsFrom(collector, streams); - return new InfoItemsPage<>(collector, getNextPageUrl()); - } - - private void collectStreamsFrom( - @Nonnull StreamInfoItemsCollector collector, - @Nullable JsonArray streams) { - collector.reset(); - - if (streams == null) { - return; - } - - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - - for (Object stream : streams) { - if (stream instanceof JsonObject) { - JsonObject streamInfo = ((JsonObject) stream).getObject("playlistPanelVideoRenderer"); - if (streamInfo != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + @Override + public void onFetchPage(@Nonnull Downloader downloader) + throws IOException, ExtractionException { + final String url = getUrl() + "&pbj=1"; + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE); + try { + playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS) + .getObject(PLAYLIST).getObject(PLAYLIST); + } catch (NullPointerException e) { + throw new ExtractionException(e); } - } - } - } - private String getThumbnailUrlFromId(String videoId) { - return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; - } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + try { + final String name = playlistData.getString("title"); + if (name != null) { + return name; + } else { + return ""; + } + } catch (Exception e) { + throw new ParsingException("Could not get playlist name", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final String playlistId = playlistData.getString("playlistId"); + final String videoId; + if (playlistId.startsWith("RDMM")) { + videoId = playlistId.substring(4); + } else { + videoId = playlistId.substring(2); + } + if (videoId.isEmpty()) { + throw new ParsingException(""); + } + return getThumbnailUrlFromId(videoId); + } catch (Exception e) { + throw new ParsingException("Could not get playlist thumbnail", e); + } + } + + @Override + public String getBannerUrl() { + return ""; + } + + @Override + public String getUploaderUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderName() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderAvatarUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public long getStreamCount() { + // Auto-generated playlist always start with 25 videos and are endless + return 25; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, playlistData.getArray(CONTENTS)); + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS) + .get(playlistData.getArray(CONTENTS).size() - 1)); + if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) { + throw new ExtractionException("Could not extract next page url"); + } + return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) + .getObject("navigationEndpoint").getObject("commandMetadata") + .getObject("webCommandMetadata").getString("url") + "&pbj=1"; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) + throws ExtractionException, IOException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException( + new IllegalArgumentException("Page url is empty or null")); + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + playlistData = + ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS) + .getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS).getObject(PLAYLIST) + .getObject(PLAYLIST); + final JsonArray streams = playlistData.getArray(CONTENTS); + //Because continuation requests are created with the last video of previous request as start + streams.remove(0); + collectStreamsFrom(collector, streams); + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + private void collectStreamsFrom( + @Nonnull StreamInfoItemsCollector collector, + @Nullable JsonArray streams) { + collector.reset(); + + if (streams == null) { + return; + } + + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object stream : streams) { + if (stream instanceof JsonObject) { + JsonObject streamInfo = ((JsonObject) stream) + .getObject(PLAYLIST_PANEL_VIDEO_RENDERER); + if (streamInfo != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + } + } + } + } + + private String getThumbnailUrlFromId(String videoId) { + return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java new file mode 100644 index 000000000..0d21ad8c6 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -0,0 +1,323 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +public class YoutubeMixPlaylistExtractorTest { + + private static YoutubeMixPlaylistExtractor extractor; + private static String videoId = "_AzeUSL9lZc"; + private static String videoTitle = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; + + public static class Mix { + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + extractor.fetchPage(); + } + + @Test + public void getServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void getName() throws Exception { + String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(videoTitle)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(videoId)); + } + + @Test + public void getNextPageUrl() throws Exception { + final String nextPageUrl = extractor.getNextPageUrl(); + assertIsSecureUrl(nextPageUrl); + assertThat(nextPageUrl, containsString("list=RD" + videoId)); + } + + @Test + public void getInitialPage() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPageMultipleTimes() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + streams = extractor.getPage(streams.getNextPageUrl()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class MixWithIndex { + + private static String index = "&index=13"; + private static String videoIdNumber13 = "qHtzO49SDmk"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoIdNumber13 + "&list=RD" + videoId + + index); + extractor.fetchPage(); + } + + @Test + public void getName() throws Exception { + String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(videoTitle)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(videoId)); + } + + @Test + public void getNextPageUrl() throws Exception { + final String nextPageUrl = extractor.getNextPageUrl(); + assertIsSecureUrl(nextPageUrl); + assertThat(nextPageUrl, containsString("list=RD" + videoId)); + } + + @Test + public void getInitialPage() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPageMultipleTimes() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + streams = extractor.getPage(streams.getNextPageUrl()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class MyMix { + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + extractor.fetchPage(); + } + + @Test + public void getServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void getName() throws Exception { + String name = extractor.getName(); + assertEquals("My Mix", name); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc")); + } + + @Test + public void getNextPageUrl() throws Exception { + final String nextPageUrl = extractor.getNextPageUrl(); + assertIsSecureUrl(nextPageUrl); + assertThat(nextPageUrl, containsString("list=RDMM" + videoId)); + } + + @Test + public void getInitialPage() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPageMultipleTimes() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + streams = extractor.getPage(streams.getNextPageUrl()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class Invalid { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ExtractionException.class) + public void getPageEmptyUrl() throws Exception { + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + extractor.fetchPage(); + extractor.getPage(""); + } + + @Test(expected = NullPointerException.class) + public void invalidVideoId() throws Exception { + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde"); + extractor.fetchPage(); + } + } + + public static class ChannelMix { + + private static String channelId = "UCXuqSBlHAE6Xw-yeJA0Tunw"; + private static String videoIdOfChannel = "mnk6gnOBYIo"; + private static String channelTitle = "Linus Tech Tips"; + + + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoIdOfChannel + "&list=RDCM" + channelId); + extractor.fetchPage(); + } + + @Test + public void getName() throws Exception { + String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(channelTitle)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + } + + @Test + public void getNextPageUrl() throws Exception { + final String nextPageUrl = extractor.getNextPageUrl(); + assertIsSecureUrl(nextPageUrl); + assertThat(nextPageUrl, containsString("list=RDCM" + channelId)); + } + + @Test + public void getInitialPage() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 636a646f8..443a21adf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -105,4 +105,23 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); } + + @Test + public void fromUrlIsMixVideo() throws Exception { + final String videoId = "_AzeUSL9lZc"; + String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; + assertEquals(url, linkHandler.fromUrl(url).getUrl()); + + final String mixVideoId = "qHtzO49SDmk"; + url = "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId; + assertEquals(url, linkHandler.fromUrl(url).getUrl()); + } + + @Test + public void fromUrlIsMixPlaylist() throws Exception { + final String videoId = "_AzeUSL9lZc"; + final String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; + assertEquals(url, + linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl()); + } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index 6de39e5ce..de4fe53fc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -26,9 +26,13 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** @@ -54,4 +58,24 @@ public class YoutubeServiceTest { public void testGetDefaultKiosk() throws Exception { assertEquals(kioskList.getDefaultKioskExtractor(null).getId(), "Trending"); } + + + @Test + public void getPlayListExtractorIsNormalPlaylist() throws Exception { + PlaylistExtractor extractor = service.getPlaylistExtractor( + "https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS"); + assertTrue(extractor instanceof YoutubePlaylistExtractor); + } + + @Test + public void getPlaylistExtractorIsMix() throws Exception { + String videoId = "_AzeUSL9lZc"; + PlaylistExtractor extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + + extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + } } From 822cf307f723613baaaa2fde0bbb78aaa1d4082a Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Tue, 17 Mar 2020 14:04:46 +0100 Subject: [PATCH 22/29] [Youtube] Add _ITEMS constants and improve code style Move thumbnail id exctraction code to getThumbnailUrlFromId Add test for "My mix" detection to service tests Use ITEM_COUNT_UNKNOWN everywhere instead of -1 and add some tests --- .../MediaCCCConferenceInfoItemExtractor.java | 4 +- .../YoutubeChannelInfoItemExtractor.java | 3 +- .../YoutubeMixPlaylistExtractor.java | 63 +++++++++---------- .../YoutubeMixPlaylistExtractorTest.java | 15 +++++ .../services/youtube/YoutubeServiceTest.java | 8 ++- 5 files changed, 55 insertions(+), 38 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java index 9099cb1a7..cea663edb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -23,7 +25,7 @@ public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtra @Override public long getStreamCount() { - return -1; + return ListExtractor.ITEM_COUNT_UNKNOWN; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 881fbd794..fbab74d83 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; @@ -86,7 +87,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor try { if (!channelInfoItem.has("videoCountText")) { // Video count is not available, channel probably has no public uploads. - return -1; + return ListExtractor.ITEM_COUNT_UNKNOWN; } return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index f4868a61e..060edc20b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -7,6 +7,8 @@ import com.grack.nanojson.JsonObject; import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; + +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -23,6 +25,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + private final static String CONTENTS = "contents"; private final static String RESPONSE = "response"; private final static String PLAYLIST = "playlist"; @@ -41,44 +44,25 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final String url = getUrl() + "&pbj=1"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE); - try { - playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS) - .getObject(PLAYLIST).getObject(PLAYLIST); - } catch (NullPointerException e) { - throw new ExtractionException(e); - } - + playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMN_WATCH_NEXT_RESULTS) + .getObject(PLAYLIST).getObject(PLAYLIST); } @Nonnull @Override public String getName() throws ParsingException { - try { - final String name = playlistData.getString("title"); - if (name != null) { - return name; - } else { - return ""; - } - } catch (Exception e) { - throw new ParsingException("Could not get playlist name", e); + final String name = playlistData.getString("title"); + if (name == null) { + throw new ParsingException("Could not get playlist name"); } + return name; } @Override public String getThumbnailUrl() throws ParsingException { try { final String playlistId = playlistData.getString("playlistId"); - final String videoId; - if (playlistId.startsWith("RDMM")) { - videoId = playlistId.substring(4); - } else { - videoId = playlistId.substring(2); - } - if (videoId.isEmpty()) { - throw new ParsingException(""); - } - return getThumbnailUrlFromId(videoId); + return getThumbnailUrlFromId(playlistId); } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -97,20 +81,20 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderName() { - //Youtube mix are auto-generated - return ""; + //Youtube mix are auto-generated by YouTube + return "YouTube"; } @Override public String getUploaderAvatarUrl() { - //Youtube mix are auto-generated + //Youtube mix are auto-generated by YouTube return ""; } @Override public long getStreamCount() { // Auto-generated playlist always start with 25 videos and are endless - return 25; + return ListExtractor.ITEM_COUNT_INFINITE; } @Nonnull @@ -135,7 +119,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getPage(final String pageUrl) - throws ExtractionException, IOException { + throws ExtractionException, IOException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException( new IllegalArgumentException("Page url is empty or null")); @@ -145,7 +129,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); playlistData = ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS) - .getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS).getObject(PLAYLIST) + .getObject(TWO_COLUMN_WATCH_NEXT_RESULTS).getObject(PLAYLIST) .getObject(PLAYLIST); final JsonArray streams = playlistData.getArray(CONTENTS); //Because continuation requests are created with the last video of previous request as start @@ -155,8 +139,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } private void collectStreamsFrom( - @Nonnull StreamInfoItemsCollector collector, - @Nullable JsonArray streams) { + @Nonnull StreamInfoItemsCollector collector, + @Nullable JsonArray streams) { collector.reset(); if (streams == null) { @@ -176,7 +160,16 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } } - private String getThumbnailUrlFromId(String videoId) { + private String getThumbnailUrlFromId(String playlistId) throws ParsingException { + final String videoId; + if (playlistId.startsWith("RDMM")) { + videoId = playlistId.substring(4); + } else { + videoId = playlistId.substring(2); + } + if (videoId.isEmpty()) { + throw new ParsingException("videoId is empty"); + } return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 0d21ad8c6..d18e4fc55 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -93,6 +93,11 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } } public static class MixWithIndex { @@ -165,6 +170,11 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } } public static class MyMix { @@ -319,5 +329,10 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } + + @Test + public void getStreamCount() throws Exception { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index de4fe53fc..7b5d2ce73 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -75,7 +75,13 @@ public class YoutubeServiceTest { assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); extractor = YouTube.getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + + final String mixVideoId = "qHtzO49SDmk"; + + extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId); assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); } } From 3ff8619bcc703920705431e217d549726cefb830 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 21 Mar 2020 18:48:12 +0100 Subject: [PATCH 23/29] [Youtube] apply wb9688 suggestion (mix) Channel mix adjusments and test Don't accept youtube music mix urls as playlist Don't override playlistData to keep getInitialPage() Remove json constants Indentation --- .../youtube/YoutubeParsingHelper.java | 23 +++++-- .../services/youtube/YoutubeService.java | 2 +- .../YoutubeMixPlaylistExtractor.java | 67 +++++++++++-------- .../YoutubePlaylistLinkHandlerFactory.java | 5 +- .../YoutubeMixPlaylistExtractorTest.java | 19 +----- 5 files changed, 64 insertions(+), 52 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 258799692..4ac9724b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -193,13 +193,23 @@ public class YoutubeParsingHelper { } /** - * Checks if the given playlist id is a mix (auto-generated playlist) - * Ids from a mix start with "RD" + * Checks if the given playlist id is a youtube mix (auto-generated playlist) + * Ids from a youtube mix start with "RD" * @param playlistId - * @return Whether given id belongs to a mix + * @return Whether given id belongs to a youtube mix */ public static boolean isYoutubeMixId(String playlistId) { - return playlistId.startsWith("RD"); + return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); + } + + /** + * Checks if the given playlist id is a youtube music mix (auto-generated playlist) + * Ids from a youtube music mix start with "RD" + * @param playlistId + * @return Whether given id belongs to a youtube music mix + */ + public static boolean isYoutubeMusicMixId(String playlistId) { + return playlistId.startsWith("RDAMVM"); } public static JsonObject getInitialData(String html) throws ParsingException { @@ -427,9 +437,9 @@ public class YoutubeParsingHelper { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) - url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); return url.toString(); } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + @@ -457,6 +467,7 @@ public class YoutubeParsingHelper { if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); if (!isNullOrEmpty(url)) { + url = url.replaceAll("&", "&"); textBuilder.append("").append(text).append(""); continue; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 7d7a83eba..fce5a1d46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -144,7 +144,7 @@ public class YoutubeService extends StreamingService { public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) - throws ExtractionException { + throws ExtractionException { return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 060edc20b..0aedafd0e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -25,13 +26,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { - - private final static String CONTENTS = "contents"; - private final static String RESPONSE = "response"; - private final static String PLAYLIST = "playlist"; - private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults"; - private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer"; - + private JsonObject initialData; private JsonObject playlistData; public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -43,9 +38,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE); - playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMN_WATCH_NEXT_RESULTS) - .getObject(PLAYLIST).getObject(PLAYLIST); + initialData = ajaxJson.getObject(3).getObject("response"); + playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("playlist").getObject("playlist"); } @Nonnull @@ -62,7 +57,14 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { public String getThumbnailUrl() throws ParsingException { try { final String playlistId = playlistData.getString("playlistId"); - return getThumbnailUrlFromId(playlistId); + try { + return getThumbnailUrlFromPlaylistId(playlistId); + } catch (ParsingException e) { + //fallback to thumbnail of current video. Always the case for channel mix + return getThumbnailUrlFromVideoId( + initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") + .getString("videoId")); + } } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -101,20 +103,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, playlistData.getArray(CONTENTS)); + collectStreamsFrom(collector, playlistData.getArray("contents")); return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override public String getNextPageUrl() throws ExtractionException { - final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS) - .get(playlistData.getArray(CONTENTS).size() - 1)); - if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) { + return getNextPageUrlFrom(playlistData); + } + + private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") + .get(playlistData.getArray("contents").size() - 1)); + if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { throw new ExtractionException("Could not extract next page url"); } - return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) - .getObject("navigationEndpoint").getObject("commandMetadata") - .getObject("webCommandMetadata").getString("url") + "&pbj=1"; + //Index of video in mix is missing, but adding it doesn't appear to have any effect. + //And since the index needs to be tracked by us, it is left out + return getUrlFromNavigationEndpoint( + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) + + "&pbj=1"; } @Override @@ -127,21 +135,20 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - playlistData = - ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS) - .getObject(TWO_COLUMN_WATCH_NEXT_RESULTS).getObject(PLAYLIST) - .getObject(PLAYLIST); - final JsonArray streams = playlistData.getArray(CONTENTS); + JsonObject playlistData = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray streams = playlistData.getArray("contents"); //Because continuation requests are created with the last video of previous request as start streams.remove(0); collectStreamsFrom(collector, streams); - return new InfoItemsPage<>(collector, getNextPageUrl()); + return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData)); } private void collectStreamsFrom( @Nonnull StreamInfoItemsCollector collector, @Nullable JsonArray streams) { - collector.reset(); if (streams == null) { return; @@ -152,7 +159,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { for (Object stream : streams) { if (stream instanceof JsonObject) { JsonObject streamInfo = ((JsonObject) stream) - .getObject(PLAYLIST_PANEL_VIDEO_RENDERER); + .getObject("playlistPanelVideoRenderer"); if (streamInfo != null) { collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); } @@ -160,16 +167,22 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } } - private String getThumbnailUrlFromId(String playlistId) throws ParsingException { + private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException { final String videoId; if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); + } else if (playlistId.startsWith("RDCMUC")) { + throw new ParsingException("is channel mix"); } else { videoId = playlistId.substring(2); } if (videoId.isEmpty()) { throw new ParsingException("videoId is empty"); } + return getThumbnailUrlFromVideoId(videoId); + } + + private String getThumbnailUrlFromVideoId(String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 787372dcb..b565fde62 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -64,11 +64,12 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { try { - getId(url); + String playlistId = getId(url); + //Because youtube music mix are not supported yet. + return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId); } catch (ParsingException e) { return false; } - return true; } /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index d18e4fc55..b612827a7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -93,11 +94,6 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } - - @Test - public void getStreamCount() throws Exception { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } } public static class MixWithIndex { @@ -166,11 +162,6 @@ public class YoutubeMixPlaylistExtractorTest { assertFalse(streams.getItems().isEmpty()); } - @Test - public void getStreamCount() { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } - @Test public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); @@ -264,12 +255,13 @@ public class YoutubeMixPlaylistExtractorTest { extractor.getPage(""); } - @Test(expected = NullPointerException.class) + @Test(expected = ExtractionException.class) public void invalidVideoId() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde"); extractor.fetchPage(); + extractor.getName(); } } @@ -329,10 +321,5 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } - - @Test - public void getStreamCount() throws Exception { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } } } \ No newline at end of file From 421935401f9c8fc45aed92742cb111f6b9e45193 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 21 May 2020 14:52:13 +0200 Subject: [PATCH 24/29] [Youtube] Add subchannel functions to mix and fix imports --- .../services/youtube/YoutubeService.java | 1 + .../YoutubeMixPlaylistExtractor.java | 32 +++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index fce5a1d46..10ac64404 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 0aedafd0e..0bf1fcbe8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,13 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import java.io.IOException; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; @@ -20,6 +14,14 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import java.io.IOException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; + /** * A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of * "youtube.com/watch?v=videoId&list=playlistId" @@ -185,4 +187,22 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { private String getThumbnailUrlFromVideoId(String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } + + @Nonnull + @Override + public String getSubChannelName() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() { + return ""; + } } From 22d2f7e4006036aab8ac63478b791c4ab3de1624 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Thu, 16 Apr 2020 19:28:27 +0200 Subject: [PATCH 25/29] [Youtube] Add cookies to youtube mix request This way youtube wont return duplicates when getting more items of the mix (but youtube can also track us) --- .../youtube/YoutubeParsingHelper.java | 93 +++++++-- .../services/youtube/YoutubeService.java | 2 +- .../YoutubeMixPlaylistExtractor.java | 102 ++++++---- .../YoutubePlaylistLinkHandlerFactory.java | 52 +++--- .../schabi/newpipe/extractor/utils/Utils.java | 12 ++ .../YoutubeMixPlaylistExtractorTest.java | 176 ++++++++++-------- ...YoutubePlaylistLinkHandlerFactoryTest.java | 2 +- .../services/youtube/YoutubeServiceTest.java | 4 +- 8 files changed, 276 insertions(+), 167 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 4ac9724b2..ea8743069 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -5,8 +5,10 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -21,6 +23,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -35,6 +38,7 @@ import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.join; /* * Created by Christian Schabesberger on 02.03.16. @@ -61,6 +65,12 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + /** + * The official youtube app supports intents in this format, where after the ':' is the videoId. + * Accordingly there are other apps sharing streams in this format. + */ + public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube"; + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; @@ -193,22 +203,22 @@ public class YoutubeParsingHelper { } /** - * Checks if the given playlist id is a youtube mix (auto-generated playlist) - * Ids from a youtube mix start with "RD" + * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) + * Ids from a YouTube Mix start with "RD" * @param playlistId - * @return Whether given id belongs to a youtube mix + * @return Whether given id belongs to a YouTube Mix */ - public static boolean isYoutubeMixId(String playlistId) { + public static boolean isYoutubeMixId(final String playlistId) { return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); } /** - * Checks if the given playlist id is a youtube music mix (auto-generated playlist) - * Ids from a youtube music mix start with "RD" + * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) + * Ids from a YouTube Music Mix start with "RD" * @param playlistId - * @return Whether given id belongs to a youtube music mix + * @return Whether given id belongs to a YouTube Music Mix */ - public static boolean isYoutubeMusicMixId(String playlistId) { + public static boolean isYoutubeMusicMixId(final String playlistId) { return playlistId.startsWith("RDAMVM"); } @@ -352,7 +362,7 @@ public class YoutubeParsingHelper { .end() .value("query", "test") .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") - .end().done().getBytes("UTF-8"); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on Map> headers = new HashMap<>(); @@ -436,10 +446,14 @@ public class YoutubeParsingHelper { } else if (navigationEndpoint.has("watchEndpoint")) { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); - if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); - if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) - url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") + .getString("playlistId")); + } + if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint") + .getInt("startTimeSeconds")); + } return url.toString(); } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + @@ -467,7 +481,6 @@ public class YoutubeParsingHelper { if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); if (!isNullOrEmpty(url)) { - url = url.replaceAll("&", "&"); textBuilder.append("").append(text).append(""); continue; } @@ -506,8 +519,8 @@ public class YoutubeParsingHelper { public static String getValidJsonResponseBody(final Response response) throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); } final String responseBody = response.responseBody(); @@ -527,13 +540,39 @@ public class YoutubeParsingHelper { final String responseContentType = response.getHeader("Content-Type"); if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); } return responseBody; } + public static Response getResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(url, headers, localization); + getValidJsonResponseBody(response); + + return response; + } + + public static String extractCookieValue(final String cookieName, final Response response) { + final List cookies = response.responseHeaders().get("Set-Cookie"); + int startIndex; + String result = ""; + for (final String cookie : cookies) { + startIndex = cookie.indexOf(cookieName); + if (startIndex != -1) { + result = cookie.substring(startIndex + cookieName.length() + "=".length(), + cookie.indexOf(";", startIndex)); + } + } + return result; + } + public static JsonArray getJsonResponse(final String url, final Localization localization) throws IOException, ExtractionException { Map> headers = new HashMap<>(); @@ -541,8 +580,24 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); final Response response = getDownloader().get(url, headers, localization); - final String responseBody = getValidJsonResponseBody(response); + return toJsonArray(getValidJsonResponseBody(response)); + } + public static JsonArray getJsonResponse(final Page page, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + if (!isNullOrEmpty(page.getCookies())) { + headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies()))); + } + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(page.getUrl(), headers, localization); + + return toJsonArray(getValidJsonResponseBody(response)); + } + + public static JsonArray toJsonArray(final String responseBody) throws ParsingException { try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 10ac64404..997fd0b73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -110,7 +110,7 @@ public class YoutubeService extends StreamingService { } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { return new YoutubeMixPlaylistExtractor(this, linkHandler); } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 0bf1fcbe8..d5e307453 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -4,8 +4,10 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -15,34 +17,50 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; +import java.util.Collections; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray; /** - * A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of - * "youtube.com/watch?v=videoId&list=playlistId" + * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). + * It handles URLs in the format of + * {@code youtube.com/watch?v=videoId&list=playlistId} */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + /** + * YouTube identifies mixes based on this cookie. With this information it can generate + * continuations without duplicates. + */ + private static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; + private JsonObject initialData; private JsonObject playlistData; + private String cookieValue; - public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + public YoutubeMixPlaylistExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) - throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + final Response response = getResponse(url, getExtractorLocalization()); + final JsonArray ajaxJson = toJsonArray(response.responseBody()); initialData = ajaxJson.getObject(3).getObject("response"); playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("playlist").getObject("playlist"); + .getObject("playlist").getObject("playlist"); + cookieValue = extractCookieValue(COOKIE_NAME, response); } @Nonnull @@ -58,16 +76,15 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - final String playlistId = playlistData.getString("playlistId"); + return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId")); + } catch (final Exception e) { try { - return getThumbnailUrlFromPlaylistId(playlistId); - } catch (ParsingException e) { //fallback to thumbnail of current video. Always the case for channel mix return getThumbnailUrlFromVideoId( initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") .getString("videoId")); + } catch (final Exception ignored) { } - } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } } @@ -104,53 +121,56 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, playlistData.getArray("contents")); - return new InfoItemsPage<>(collector, getNextPageUrl()); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrl(), Collections.singletonMap(COOKIE_NAME, cookieValue))); } - @Override - public String getNextPageUrl() throws ExtractionException { + private String getNextPageUrl() throws ExtractionException { return getNextPageUrlFrom(playlistData); } - private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException { - final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") - .get(playlistData.getArray("contents").size() - 1)); + private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents") + .get(playlistJson.getArray("contents").size() - 1)); if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { throw new ExtractionException("Could not extract next page url"); } - //Index of video in mix is missing, but adding it doesn't appear to have any effect. - //And since the index needs to be tracked by us, it is left out + return getUrlFromNavigationEndpoint( - lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) - + "&pbj=1"; + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) + + "&pbj=1"; } @Override - public InfoItemsPage getPage(final String pageUrl) + public InfoItemsPage getPage(final Page page) throws ExtractionException, IOException { - if (pageUrl == null || pageUrl.isEmpty()) { + if (page == null || page.getUrl().isEmpty()) { throw new ExtractionException( new IllegalArgumentException("Page url is empty or null")); } - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonObject playlistData = - ajaxJson.getObject(3).getObject("response").getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("playlist") - .getObject("playlist"); - final JsonArray streams = playlistData.getArray("contents"); - //Because continuation requests are created with the last video of previous request as start - streams.remove(0); - collectStreamsFrom(collector, streams); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData)); + final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization()); + final JsonObject playlistJson = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray allStreams = playlistJson.getArray("contents"); + // Sublist because youtube returns up to 24 previous streams in the mix + // +1 because the stream of "currentIndex" was already extracted in previous request + final List newStreams = + allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size()); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, newStreams); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrlFrom(playlistJson), page.getCookies())); } private void collectStreamsFrom( - @Nonnull StreamInfoItemsCollector collector, - @Nullable JsonArray streams) { + @Nonnull final StreamInfoItemsCollector collector, + @Nullable final List streams) { if (streams == null) { return; @@ -158,9 +178,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Object stream : streams) { + for (final Object stream : streams) { if (stream instanceof JsonObject) { - JsonObject streamInfo = ((JsonObject) stream) + final JsonObject streamInfo = ((JsonObject) stream) .getObject("playlistPanelVideoRenderer"); if (streamInfo != null) { collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); @@ -169,7 +189,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } } - private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException { + private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { final String videoId; if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); @@ -184,7 +204,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { return getThumbnailUrlFromVideoId(videoId); } - private String getThumbnailUrlFromVideoId(String videoId) { + private String getThumbnailUrlFromVideoId(final String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index b565fde62..096447879 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -14,82 +14,88 @@ import java.util.List; public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { - private static final YoutubePlaylistLinkHandlerFactory instance = new YoutubePlaylistLinkHandlerFactory(); + private static final YoutubePlaylistLinkHandlerFactory INSTANCE = + new YoutubePlaylistLinkHandlerFactory(); public static YoutubePlaylistLinkHandlerFactory getInstance() { - return instance; + return INSTANCE; } @Override - public String getUrl(String id, List contentFilters, String sortFilter) { + public String getUrl(final String id, final List contentFilters, + final String sortFilter) { return "https://www.youtube.com/playlist?list=" + id; } @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); + final URL urlObj = Utils.stringToURL(url); if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj))) { throw new ParsingException("the url given is not a Youtube-URL"); } - String path = urlObj.getPath(); + final String path = urlObj.getPath(); if (!path.equals("/watch") && !path.equals("/playlist")) { throw new ParsingException("the url given is neither a video nor a playlist URL"); } - String listID = Utils.getQueryValue(urlObj, "list"); + final String listID = Utils.getQueryValue(urlObj, "list"); if (listID == null) { throw new ParsingException("the url given does not include a playlist"); } if (!listID.matches("[a-zA-Z0-9_-]{10,}")) { - throw new ParsingException("the list-ID given in the URL does not match the list pattern"); + throw new ParsingException( + "the list-ID given in the URL does not match the list pattern"); } - // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists - if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) { - throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported"); + if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) { + throw new ContentNotSupportedException( + "YouTube Music Mix playlists are not yet supported"); } return listID; } catch (final Exception exception) { - throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); } } @Override public boolean onAcceptUrl(final String url) { try { - String playlistId = getId(url); - //Because youtube music mix are not supported yet. - return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId); + getId(url); } catch (ParsingException e) { return false; } + return true; } /** - * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like - * youtube.com/watch?v=videoId&list=playlistId + * * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is + * like + * https://youtube.com/watch?v=videoId&list=playlistId. *

Otherwise use super

*/ @Override - public ListLinkHandler fromUrl(String url) throws ParsingException { + public ListLinkHandler fromUrl(final String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); - String listID = Utils.getQueryValue(urlObj, "list"); + final URL urlObj = Utils.stringToURL(url); + final String listID = Utils.getQueryValue(urlObj, "list"); if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { String videoID = Utils.getQueryValue(urlObj, "v"); if (videoID == null) { videoID = listID.substring(2); } - String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; - return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), - getSortFilter(url)); + final String newUrl = "https://www.youtube.com/watch?v=" + videoID + + "&list=" + listID; + return new ListLinkHandler(new LinkHandler(url, newUrl, listID), + getContentFilter(url), + getSortFilter(url)); } } catch (MalformedURLException exception) { throw new ParsingException("Error could not parse url :" + exception.getMessage(), diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 959202700..3c2bc7d9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -8,6 +8,7 @@ import java.net.URL; import java.net.URLDecoder; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -260,4 +261,15 @@ public class Utils { } return stringBuilder.toString(); } + + public static String join(final String delimiter, final String mapJoin, + final Map elements) { + final List list = new LinkedList<>(); + for (final Map.Entry entry : elements + .entrySet()) { + list.add(entry.getKey() + mapJoin + entry.getValue()); + } + return join(delimiter, list); + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index b612827a7..8e41f12d8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -2,26 +2,45 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.startsWith; -import static org.junit.Assert.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import java.util.HashSet; +import java.util.Set; +import org.hamcrest.MatcherAssert; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.ChannelMix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Invalid; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Mix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MixWithIndex; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MyMix; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +@RunWith(Suite.class) +@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class}) public class YoutubeMixPlaylistExtractorTest { + public static final String PBJ = "&pbj=1"; + private static final String VIDEO_ID = "_AzeUSL9lZc"; + private static final String VIDEO_TITLE = + "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; + private static YoutubeMixPlaylistExtractor extractor; - private static String videoId = "_AzeUSL9lZc"; - private static String videoTitle = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; public static class Mix { @@ -29,8 +48,8 @@ public class YoutubeMixPlaylistExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); extractor.fetchPage(); } @@ -41,81 +60,82 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(videoTitle)); + assertThat(name, containsString(VIDEO_TITLE)); } @Test public void getThumbnailUrl() throws Exception { final String thumbnailUrl = extractor.getThumbnailUrl(); assertIsSecureUrl(thumbnailUrl); - assertThat(thumbnailUrl, containsString("yt")); - assertThat(thumbnailUrl, containsString(videoId)); - } - - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RD" + videoId)); + MatcherAssert.assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); } @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID + + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); - streams = extractor.getPage(streams.getNextPageUrl()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } public static class MixWithIndex { - private static String index = "&index=13"; - private static String videoIdNumber13 = "qHtzO49SDmk"; + private static final String INDEX = "&index=13"; + private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoIdNumber13 + "&list=RD" + videoId - + index); + "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX); extractor.fetchPage(); } @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(videoTitle)); + assertThat(name, containsString(VIDEO_TITLE)); } @Test @@ -123,47 +143,47 @@ public class YoutubeMixPlaylistExtractorTest { final String thumbnailUrl = extractor.getThumbnailUrl(); assertIsSecureUrl(thumbnailUrl); assertThat(thumbnailUrl, containsString("yt")); - assertThat(thumbnailUrl, containsString(videoId)); - } - - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RD" + videoId)); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); } @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } - streams = extractor.getPage(streams.getNextPageUrl()); + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } @@ -175,7 +195,8 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM" + + VIDEO_ID); extractor.fetchPage(); } @@ -186,7 +207,7 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertEquals("My Mix", name); } @@ -197,44 +218,44 @@ public class YoutubeMixPlaylistExtractorTest { assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc")); } - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RDMM" + videoId)); - } - @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = + extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + + "&list=RDMM" + VIDEO_ID + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } - @Test - public void getPageMultipleTimes() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); //Should work infinitely, but for testing purposes only 3 times for (int i = 0; i < 3; i++) { assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); - streams = extractor.getPage(streams.getNextPageUrl()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); } assertTrue(streams.hasNextPage()); assertFalse(streams.getItems().isEmpty()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } @@ -249,10 +270,10 @@ public class YoutubeMixPlaylistExtractorTest { @Test(expected = ExtractionException.class) public void getPageEmptyUrl() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); extractor.fetchPage(); - extractor.getPage(""); + extractor.getPage(new Page("")); } @Test(expected = ExtractionException.class) @@ -267,10 +288,9 @@ public class YoutubeMixPlaylistExtractorTest { public static class ChannelMix { - private static String channelId = "UCXuqSBlHAE6Xw-yeJA0Tunw"; - private static String videoIdOfChannel = "mnk6gnOBYIo"; - private static String channelTitle = "Linus Tech Tips"; - + private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw"; + private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo"; + private static final String CHANNEL_TITLE = "Linus Tech Tips"; @BeforeClass @@ -278,15 +298,16 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + videoIdOfChannel + "&list=RDCM" + channelId); + "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID); extractor.fetchPage(); } @Test public void getName() throws Exception { - String name = extractor.getName(); + final String name = extractor.getName(); assertThat(name, startsWith("Mix")); - assertThat(name, containsString(channelTitle)); + assertThat(name, containsString(CHANNEL_TITLE)); } @Test @@ -296,30 +317,25 @@ public class YoutubeMixPlaylistExtractorTest { assertThat(thumbnailUrl, containsString("yt")); } - @Test - public void getNextPageUrl() throws Exception { - final String nextPageUrl = extractor.getNextPageUrl(); - assertIsSecureUrl(nextPageUrl); - assertThat(nextPageUrl, containsString("list=RDCM" + channelId)); - } - @Test public void getInitialPage() throws Exception { - InfoItemsPage streams = extractor.getInitialPage(); + final InfoItemsPage streams = extractor.getInitialPage(); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test public void getPage() throws Exception { - InfoItemsPage streams = extractor.getPage(extractor.getNextPageUrl()); + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID + PBJ)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @Test - public void getStreamCount() throws Exception { + public void getStreamCount() { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } } -} \ No newline at end of file +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 443a21adf..e7bec115d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -56,7 +56,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=RDCLAK5uy_ly6s4irLuZAcjEDwJmqcA_UtSipMyGgbQ")); // YouTube Music playlist - assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index 7b5d2ce73..4354fa258 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -62,14 +62,14 @@ public class YoutubeServiceTest { @Test public void getPlayListExtractorIsNormalPlaylist() throws Exception { - PlaylistExtractor extractor = service.getPlaylistExtractor( + final PlaylistExtractor extractor = service.getPlaylistExtractor( "https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS"); assertTrue(extractor instanceof YoutubePlaylistExtractor); } @Test public void getPlaylistExtractorIsMix() throws Exception { - String videoId = "_AzeUSL9lZc"; + final String videoId = "_AzeUSL9lZc"; PlaylistExtractor extractor = YouTube.getPlaylistExtractor( "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); From a338e4e08e385e741c18ce2d30ae1e5016599cf0 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 26 Sep 2020 11:22:24 +0200 Subject: [PATCH 26/29] [Youtube] Apply review suggestions and avoid channel mix edge case --- .../youtube/YoutubeParsingHelper.java | 35 +++++++++++++++++-- .../YoutubeMixPlaylistExtractor.java | 17 +++++---- .../YoutubePlaylistLinkHandlerFactory.java | 15 +++++--- .../YoutubeMixPlaylistExtractorTest.java | 34 ++++++++++-------- 4 files changed, 70 insertions(+), 31 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index ea8743069..06f421a69 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -214,13 +214,44 @@ public class YoutubeParsingHelper { /** * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) - * Ids from a YouTube Music Mix start with "RD" + * Ids from a YouTube Music Mix start with "RDAMVM" * @param playlistId * @return Whether given id belongs to a YouTube Music Mix */ public static boolean isYoutubeMusicMixId(final String playlistId) { return playlistId.startsWith("RDAMVM"); } + /** + * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) + * Ids from a YouTube channel Mix start with "RDCM" + * @return Whether given id belongs to a YouTube Channel Mix + */ + public static boolean isYoutubeChannelMixId(final String playlistId) { + return playlistId.startsWith("RDCM"); + } + + /** + * Extracts the video id from the playlist id for Mixes. + * @throws ParsingException If the playlistId is a Channel Mix or not a mix. + */ + public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException { + if (playlistId.startsWith("RDMM")) { //My Mix + return playlistId.substring(4); + + } else if (playlistId.startsWith("RDAMVM")) { //Music mix + return playlistId.substring(6); + + } else if (playlistId.startsWith("RMCM")) { //Channel mix + //Channel mix are build with RMCM{channelId}, so videoId can't be determined + throw new ParsingException("Video id could not be determined from mix id: " + playlistId); + + } else if (playlistId.startsWith("RD")) { // Normal mix + return playlistId.substring(2); + + } else { //not a mix + throw new ParsingException("Video id could not be determined from mix id: " + playlistId); + } + } public static JsonObject getInitialData(String html) throws ParsingException { try { @@ -362,7 +393,7 @@ public class YoutubeParsingHelper { .end() .value("query", "test") .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") - .end().done().getBytes(StandardCharsets.UTF_8); + .end().done().getBytes("UTF-8"); // @formatter:on Map> headers = new HashMap<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index d5e307453..d424129ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -28,6 +28,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /** * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). @@ -40,7 +41,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { * YouTube identifies mixes based on this cookie. With this information it can generate * continuations without duplicates. */ - private static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; + public static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; private JsonObject initialData; private JsonObject playlistData; @@ -124,11 +125,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, playlistData.getArray("contents")); return new InfoItemsPage<>(collector, - new Page(getNextPageUrl(), Collections.singletonMap(COOKIE_NAME, cookieValue))); - } - - private String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(playlistData); + new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue))); } private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { @@ -146,9 +143,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getPage(final Page page) throws ExtractionException, IOException { - if (page == null || page.getUrl().isEmpty()) { - throw new ExtractionException( - new IllegalArgumentException("Page url is empty or null")); + if (page == null || isNullOrEmpty(page.getUrl())) { + throw new IllegalArgumentException("Page url is empty or null"); + } + if (!page.getCookies().containsKey(COOKIE_NAME)) { + throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing"); } final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 096447879..aa2908e64 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,5 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -8,10 +11,6 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.List; - public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { private static final YoutubePlaylistLinkHandlerFactory INSTANCE = @@ -58,6 +57,12 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { "YouTube Music Mix playlists are not yet supported"); } + if (YoutubeParsingHelper.isYoutubeChannelMixId(listID) + && Utils.getQueryValue(urlObj, "v") == null) { + //Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId + throw new ContentNotSupportedException("Channel Mix without a video id are not supported"); + } + return listID; } catch (final Exception exception) { throw new ParsingException("Error could not parse url :" + exception.getMessage(), @@ -89,7 +94,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { String videoID = Utils.getQueryValue(urlObj, "v"); if (videoID == null) { - videoID = listID.substring(2); + videoID = YoutubeParsingHelper.extractVideoIdFromMixId(listID); } final String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 8e41f12d8..49fb3fe04 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -1,15 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.startsWith; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - +import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Set; import org.hamcrest.MatcherAssert; import org.junit.BeforeClass; @@ -31,6 +24,15 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractor import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + @RunWith(Suite.class) @SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class}) public class YoutubeMixPlaylistExtractorTest { @@ -41,6 +43,8 @@ public class YoutubeMixPlaylistExtractorTest { "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; private static YoutubeMixPlaylistExtractor extractor; + private static Map dummyCookie + = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); public static class Mix { @@ -83,8 +87,8 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { final InfoItemsPage streams = extractor.getPage( - new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID - + PBJ)); + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID + + PBJ, dummyCookie)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -157,7 +161,7 @@ public class YoutubeMixPlaylistExtractorTest { public void getPage() throws Exception { final InfoItemsPage streams = extractor.getPage( new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" - + VIDEO_ID + INDEX + PBJ)); + + VIDEO_ID + INDEX + PBJ, dummyCookie)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -229,7 +233,7 @@ public class YoutubeMixPlaylistExtractorTest { public void getPage() throws Exception { final InfoItemsPage streams = extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID - + "&list=RDMM" + VIDEO_ID + PBJ)); + + "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -267,7 +271,7 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); } - @Test(expected = ExtractionException.class) + @Test(expected = IllegalArgumentException.class) public void getPageEmptyUrl() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( @@ -328,7 +332,7 @@ public class YoutubeMixPlaylistExtractorTest { public void getPage() throws Exception { final InfoItemsPage streams = extractor.getPage( new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL - + "&list=RDCM" + CHANNEL_ID + PBJ)); + + "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } From f90f6fcf92b355f668656e9e277bdb2086d11ab5 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 12 Dec 2020 20:40:13 +0100 Subject: [PATCH 27/29] [YouTube] Don't escape & in getUrlFromNavigationEndpoint for playlists --- .../extractor/services/youtube/YoutubeParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 06f421a69..835ddd309 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -478,7 +478,7 @@ public class YoutubeParsingHelper { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") .getString("playlistId")); } if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { From ba8782a9ed20271bca01b87ec22d8edc8652c45e Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Wed, 16 Dec 2020 08:32:04 +0100 Subject: [PATCH 28/29] Use lowercase string for extracting cookies The map in a real response happens to ignore upper-/lowercase differences. Other maps used in unit test may not have that behaviour. --- .../extractor/services/youtube/YoutubeParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 835ddd309..69d877128 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -591,7 +591,7 @@ public class YoutubeParsingHelper { } public static String extractCookieValue(final String cookieName, final Response response) { - final List cookies = response.responseHeaders().get("Set-Cookie"); + final List cookies = response.responseHeaders().get("set-cookie"); int startIndex; String result = ""; for (final String cookie : cookies) { From cdcb66b93e5a57f6a676b27f3a2ba9a1b923dc01 Mon Sep 17 00:00:00 2001 From: XiangRongLin <41164160+XiangRongLin@users.noreply.github.com> Date: Wed, 16 Dec 2020 08:33:32 +0100 Subject: [PATCH 29/29] Clean up stuff from mix pl Coming from unclean merge and missing hints from IDE --- .../extractor/services/youtube/YoutubeParsingHelper.java | 6 ------ .../services/youtube/YoutubeMixPlaylistExtractorTest.java | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 69d877128..b36fe039a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -65,12 +65,6 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } - /** - * The official youtube app supports intents in this format, where after the ':' is the videoId. - * Accordingly there are other apps sharing streams in this format. - */ - public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube"; - private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 49fb3fe04..883218911 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -41,10 +41,10 @@ public class YoutubeMixPlaylistExtractorTest { private static final String VIDEO_ID = "_AzeUSL9lZc"; private static final String VIDEO_TITLE = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; + private static final Map dummyCookie + = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); private static YoutubeMixPlaylistExtractor extractor; - private static Map dummyCookie - = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); public static class Mix {