From 00d1ed439b49cdbabcd6933afd7945bc45e9aefc Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 14 Mar 2020 02:29:43 -0300 Subject: [PATCH 01/66] [YouTube] Fix channel extraction when redirects are in the response Some redirects were embed directly into the response as instructions for the page, instead of the usual http redirects. --- .../extractors/YoutubeChannelExtractor.java | 49 ++++++++-- .../extractors/YoutubeStreamExtractor.java | 2 +- .../newpipe/extractor/utils/JsonUtils.java | 4 +- .../youtube/YoutubeChannelExtractorTest.java | 95 +++++++++++++++++++ 4 files changed, 140 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 11dd8985f..1aeb65ed4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -16,13 +15,11 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.io.IOException; -import javax.annotation.Nonnull; - -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.utils.JsonUtils.*; /* * Created by Christian Schabesberger on 25.07.16. @@ -55,9 +52,45 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; + String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; + JsonArray ajaxJson = null; + + int level = 0; + while (level < 3) { + final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization()); + + final JsonObject endpoint = jsonResponse.getObject(1, EMPTY_OBJECT) + .getObject("response", EMPTY_OBJECT).getArray("onResponseReceivedActions", EMPTY_ARRAY) + .getObject(0, EMPTY_OBJECT).getObject("navigateAction", EMPTY_OBJECT) + .getObject("endpoint", EMPTY_OBJECT); + + final String webPageType = endpoint + .getObject("commandMetadata", EMPTY_OBJECT) + .getObject("webCommandMetadata", EMPTY_OBJECT) + .getString("webPageType", EMPTY_STRING); + + final String browseId = endpoint + .getObject("browseEndpoint", EMPTY_OBJECT) + .getString("browseId", EMPTY_STRING); + + if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) { + + if (!browseId.startsWith("UC")) { + throw new ExtractionException("Redirected id is not pointing to a channel"); + } + + url = "https://www.youtube.com/channel/" + browseId + "/videos?pbj=1&view=0&flow=grid"; + level++; + } else { + ajaxJson = jsonResponse; + break; + } + } + + if (ajaxJson == null) { + throw new ExtractionException("Could not fetch initial JSON data"); + } - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(1).getObject("response"); YoutubeParsingHelper.defaultAlertsCheck(initialData); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8ee502013..475557867 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -621,7 +621,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerResponse = getPlayerResponse(); - final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY); + final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.EMPTY_OBJECT); final String status = playabilityStatus.getString("status"); // If status exist, and is not "OK", throw a ContentNotAvailableException with the reason. if (status != null && !status.toLowerCase().equals("ok")) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index 25bb3f6c0..e69240a01 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -11,7 +11,9 @@ import java.util.Arrays; import java.util.List; public class JsonUtils { - public static final JsonObject DEFAULT_EMPTY = new JsonObject(); + public static final JsonObject EMPTY_OBJECT = new JsonObject(); + public static final JsonArray EMPTY_ARRAY = new JsonArray(); + public static final String EMPTY_STRING = ""; private JsonUtils() { } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index dfcaee40a..3615bc63c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -5,12 +5,16 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.channel.ChannelInfo; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import java.util.List; + import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.*; @@ -505,6 +509,97 @@ public class YoutubeChannelExtractorTest { } } + /** + * Some VEVO channels will redirect to a new page with a new channel id. + *

+ * Though, it isn't a simple redirect, but a redirect instruction embed in the response itself, this + * test assure that we account for that. + */ + public static class RedirectedChannel implements BaseChannelExtractorTest { + private static YoutubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeChannelExtractor) YouTube + .getChannelExtractor("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("LordiVEVO", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + assertNoMoreItems(extractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws Exception { + assertEmpty(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws Exception { + String avatarUrl = extractor.getAvatarUrl(); + assertIsSecureUrl(avatarUrl); + assertTrue(avatarUrl, avatarUrl.contains("yt3")); + } + + @Test + public void testBannerUrl() throws Exception { + assertEmpty(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws Exception { + assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws Exception { + assertEquals(-1, extractor.getSubscriberCount()); + } + } public static class RandomChannel implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; From b086e9db3fc73c631da8c7c35974b06b3d5a5392 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 14 Mar 2020 02:34:29 -0300 Subject: [PATCH 02/66] [YouTube] Fix id extraction for some channels Some channels had no reliable way to get the redirected id in the response, so saving it for later was a valid alternative. --- .../extractors/YoutubeChannelExtractor.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 1aeb65ed4..385980745 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -46,6 +46,18 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private JsonObject initialData; private JsonObject videoTab; + /** + * Some channels have response redirects and the only way to reliably get the id is by saving it. + *

+ * "Movies & Shows": + * 

+     * UCuJcl0Ju-gPDoksRjK1ya-w ┐
+     * UChBfWrfBXL9wS6tQtgjt_OQ ├ UClgRkhTL3_hImCAmdLfDE4g
+     * UCok7UTQQEP1Rsctxiv3gwSQ ┘
+     *
+ */ + private String redirectedChannelId; + public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); } @@ -80,6 +92,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } url = "https://www.youtube.com/channel/" + browseId + "/videos?pbj=1&view=0&flow=grid"; + redirectedChannelId = browseId; level++; } else { ajaxJson = jsonResponse; @@ -117,10 +130,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public String getId() throws ParsingException { - try { - return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("channelId"); - } catch (Exception e) { - throw new ParsingException("Could not get channel id", e); + final String channelId = initialData + .getObject("header", EMPTY_OBJECT) + .getObject("c4TabbedHeaderRenderer", EMPTY_OBJECT) + .getString("channelId", EMPTY_STRING); + + if (!channelId.isEmpty()) { + return channelId; + } else if (redirectedChannelId != null && !redirectedChannelId.isEmpty()) { + return redirectedChannelId; + } else { + throw new ParsingException("Could not get channel id"); } } From 5f39bc4e9907e9af013bb7c5f241942ecb06221d Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 14 Mar 2020 12:41:44 +0100 Subject: [PATCH 03/66] [SoundCloud] Update hardcoded client id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index ed50ea544..b8f6af246 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20 + private static final String HARDCODED_CLIENT_ID = "t0h1jzYMsaZXy6ggnZO71gHK3Ms6CFwE"; // Updated on 14/03/20 private static String clientId; private SoundcloudParsingHelper() { From dc20f2aa0cb674918a3ddfbf654ebde55a969d62 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Mar 2020 17:27:26 +0100 Subject: [PATCH 04/66] version 0.18.7 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 977594438..8e9045652 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.18.6' + version 'v0.18.7' group 'com.github.TeamNewPipe' repositories { From efad0be99ffaeff71900bb206a1216ece7c0a14e Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Mar 2020 19:42:35 +0100 Subject: [PATCH 05/66] Test if the extractor can get AudiStreams with the hard-coded cliend_id SoundCloud client_id are changed on a regular basis now. We might want to remove the hardcoded id completely on a later point of time when client_id is changed to often --- .../services/soundcloud/SoundcloudParsingHelper.java | 10 +++++----- .../soundcloud/SoundcloudParsingHelperTest.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index b8f6af246..11cd609c6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -42,7 +42,7 @@ public class SoundcloudParsingHelper { Downloader dl = NewPipe.getDownloader(); clientId = HARDCODED_CLIENT_ID; - if (checkIfHardcodedClientIdIsValid(dl)) { + if (checkIfHardcodedClientIdIsValid()) { return clientId; } @@ -73,11 +73,11 @@ public class SoundcloudParsingHelper { throw new ExtractionException("Couldn't extract client id"); } - static boolean checkIfHardcodedClientIdIsValid(Downloader dl) { - final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID; + static boolean checkIfHardcodedClientIdIsValid() { try { - // Should return 200 to indicate that the client id is valid, a 401 is returned otherwise. - return dl.head(apiUrl).responseCode() == 200; + SoundcloudStreamExtractor e = (SoundcloudStreamExtractor) SoundCloud + .getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); + return e.getAudioStreams().size() >= 1; } catch (Exception ignored) { // No need to throw an exception here. If something went wrong, the client_id is wrong return false; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index 16089aab1..bbbcb8618 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -17,7 +17,7 @@ public class SoundcloudParsingHelperTest { @Test public void assertThatHardcodedClientIdIsValid() throws Exception { assertTrue("Hardcoded client id is not valid anymore", - SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid(DownloaderTestImpl.getInstance())); + SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid()); } @Test From edb57840d72c919c032b21e54e359d22a7f5b0ef Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Mar 2020 19:43:36 +0100 Subject: [PATCH 06/66] [SoundCloud] Update hard-coded client_id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 11cd609c6..977d01070 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "t0h1jzYMsaZXy6ggnZO71gHK3Ms6CFwE"; // Updated on 14/03/20 + private static final String HARDCODED_CLIENT_ID = "Uz4aPhG7GAl1VYGOnvOPW1wQ0M6xKtA9"; // Updated on 16/03/20 private static String clientId; private SoundcloudParsingHelper() { From 9eca7df947cd10eb1bf68c20d55a4aa04b0cc85b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Mar 2020 20:50:08 +0100 Subject: [PATCH 07/66] Forget to fetch the page --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 977d01070..6362f13cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -77,6 +77,7 @@ public class SoundcloudParsingHelper { try { SoundcloudStreamExtractor e = (SoundcloudStreamExtractor) SoundCloud .getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); + e.fetchPage(); return e.getAudioStreams().size() >= 1; } catch (Exception ignored) { // No need to throw an exception here. If something went wrong, the client_id is wrong From f3095713f9cc158a092eb24d3338c0c8fafd4a8d Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:12:13 +0100 Subject: [PATCH 08/66] [SoundCloud] Use api-v2 in PlaylistExtractor Rewrote methods to calculate next page url and to get items from it. `api-v2` is different from `api` since the initial playlist page contains (usually) the full info of the first 3 streams and only the id of the other. Then the single tracks can be requested in batch using `/tracks?ids=id1,id2,...`. --- .../SoundcloudPlaylistExtractor.java | 77 +++++++++++++++---- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index ca08f66ce..363a223a2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -1,8 +1,11 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -13,15 +16,23 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import java.io.IOException; +import java.util.ArrayList; +import java.util.List; @SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { + private static final int streamsPerRequestedPage = 15; + private String playlistId; private JsonObject playlist; - private StreamInfoItemsCollector streamInfoItemsCollector = null; - private String nextPageUrl = null; + private StreamInfoItemsCollector streamInfoItemsCollector; + private List nextTrackIds; + private int nextTrackIdsIndex; + private String nextPageUrl; public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -31,7 +42,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { playlistId = getLinkHandler().getId(); - String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId + + String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId + "?client_id=" + SoundcloudParsingHelper.clientId() + "&representation=compact"; @@ -110,27 +121,55 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if (streamInfoItemsCollector == null) { - computeStreamsAndNextPageUrl(); + computeInitialTracksAndNextIds(); } return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); } - private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException { + private void computeInitialTracksAndNextIds() { streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); + nextTrackIds = new ArrayList<>(); + nextTrackIdsIndex = 0; - // Note the "api", NOT "api-v2" - String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks" - + "?client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=20" - + "&linked_partitioning=1"; + JsonArray tracks = playlist.getArray("tracks"); + for (Object o : tracks) { + if (o instanceof JsonObject) { + JsonObject track = (JsonObject) o; + if (track.has("title")) { // i.e. if full info is available + streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); + } else { + nextTrackIds.add(track.getInt("id")); + } + } + } + } - nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl); + private void computeAnotherNextPageUrl() throws IOException, ExtractionException { + if (nextTrackIdsIndex >= nextTrackIds.size()) { + nextPageUrl = ""; // there are no more tracks + return; + } + + StringBuilder urlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id="); + urlBuilder.append(SoundcloudParsingHelper.clientId()); + urlBuilder.append("&ids="); + + int upperIndex = Math.min(nextTrackIdsIndex + streamsPerRequestedPage, nextTrackIds.size()); + for (int i = nextTrackIdsIndex; i < upperIndex; ++i) { + urlBuilder.append(nextTrackIds.get(i)); + urlBuilder.append(","); // a , at the end is ok + } + + nextPageUrl = urlBuilder.toString(); } @Override public String getNextPageUrl() throws IOException, ExtractionException { if (nextPageUrl == null) { - computeStreamsAndNextPageUrl(); + if (nextTrackIds == null) { + computeInitialTracksAndNextIds(); + } + computeAnotherNextPageUrl(); } return nextPageUrl; } @@ -142,8 +181,20 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); + String response = NewPipe.getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); + try { + JsonArray tracks = JsonParser.array().from(response); + for (Object track : tracks) { + if (track instanceof JsonObject) { + collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track)); + } + } + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } + + computeAnotherNextPageUrl(); return new InfoItemsPage<>(collector, nextPageUrl); } } From d0e66cc600ad43a48e17c65e29e897f4b549a3d0 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:13:28 +0100 Subject: [PATCH 09/66] [SoundCloud] Improve thumbnail url extraction in playlists Prevent NullPointerExceptions and remove duplicate code --- .../soundcloud/SoundcloudPlaylistExtractor.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 363a223a2..f503ac0fe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -66,6 +66,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { return playlist.getString("title"); } + @Nullable @Override public String getThumbnailUrl() { String artworkUrl = playlist.getString("artwork_url"); @@ -75,21 +76,20 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { // if it also fails, return null try { final InfoItemsPage infoItems = getInitialPage(); - if (infoItems.getItems().isEmpty()) return null; for (StreamInfoItem item : infoItems.getItems()) { - final String thumbnailUrl = item.getThumbnailUrl(); - if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue; - - String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg"); - return thumbnailUrlBetterResolution; + artworkUrl = item.getThumbnailUrl(); + if (artworkUrl != null && !artworkUrl.isEmpty()) break; } } catch (Exception ignored) { } + + if (artworkUrl == null) { + return null; + } } - String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); - return artworkUrlBetterResolution; + return artworkUrl.replace("large.jpg", "crop.jpg"); } @Override From c3d811fde5ced6518381173b0d4c0e73a13d0b08 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:22:25 +0100 Subject: [PATCH 10/66] [SoundCloud] Use api-v2 in SubscriptionExtractor Also added --- .../services/soundcloud/SoundcloudSubscriptionExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java index f65bf98b7..cf01b0c93 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java @@ -36,7 +36,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { throw new InvalidSourceException(e); } - String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings" + String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings" + "?client_id=" + SoundcloudParsingHelper.clientId() + "&limit=200"; ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId()); From 4b1121aac7d48e84f97048028aa6005f2acc1d5f Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:23:13 +0100 Subject: [PATCH 11/66] [SoundCloud] Add tests for api-v2 channel urls --- .../services/soundcloud/SoundcloudParsingHelperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index bbbcb8618..130b0de30 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -24,6 +24,8 @@ public class SoundcloudParsingHelperTest { public void resolveUrlWithEmbedPlayerTest() throws Exception { Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743")); Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159")); + Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/26057743")); + Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/16069159")); } @Test From d4aa4a076375af22256fb622547bab71a3eb4a3e Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:25:11 +0100 Subject: [PATCH 12/66] [SoundCloud] Fix typo in ChartsExtractor --- .../services/soundcloud/SoundcloudChartsExtractor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index b9f4894d3..a08502d70 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -44,7 +44,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } - private void computNextPageAndStreams() throws IOException, ExtractionException { + private void computeNextPageAndStreams() throws IOException, ExtractionException { collector = new StreamInfoItemsCollector(getServiceId()); String apiUrl = "https://api-v2.soundcloud.com/charts" + @@ -69,7 +69,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Override public String getNextPageUrl() throws IOException, ExtractionException { if (nextPageUrl == null) { - computNextPageAndStreams(); + computeNextPageAndStreams(); } return nextPageUrl; } @@ -78,7 +78,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if (collector == null) { - computNextPageAndStreams(); + computeNextPageAndStreams(); } return new InfoItemsPage<>(collector, getNextPageUrl()); } From ca8bf53b61131a8849789e76156f490d12a15751 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:35:33 +0100 Subject: [PATCH 13/66] [SoundCloud] Fix playlist test: number of streams changed --- .../services/soundcloud/SoundcloudPlaylistExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index b93c2cfb9..067633bbe 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -317,7 +317,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testStreamCount() { - assertTrue("Error in the streams count", extractor.getStreamCount() >= 3900); + assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 370); } } } From 1558da6f6b2aa4613c6af45f302104b1464d28a0 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:41:16 +0100 Subject: [PATCH 14/66] [SoundCloud] Fix playlist next page generation --- .../services/soundcloud/SoundcloudPlaylistExtractor.java | 3 ++- .../soundcloud/SoundcloudPlaylistExtractorTest.java | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index f503ac0fe..a57abea8b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -145,7 +145,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } private void computeAnotherNextPageUrl() throws IOException, ExtractionException { - if (nextTrackIdsIndex >= nextTrackIds.size()) { + if (nextTrackIds == null || nextTrackIdsIndex >= nextTrackIds.size()) { nextPageUrl = ""; // there are no more tracks return; } @@ -160,6 +160,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { urlBuilder.append(","); // a , at the end is ok } + nextTrackIdsIndex = upperIndex; nextPageUrl = urlBuilder.toString(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 067633bbe..f0832ef28 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -7,10 +7,13 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.io.IOException; + import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -70,7 +73,9 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testMoreRelatedItems() { + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + try { defaultTestMoreItems(extractor); } catch (Throwable ignored) { From 65bdb3bc9dd45c8d8fd198064497e9473be6e16f Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:49:58 +0100 Subject: [PATCH 15/66] [SoundCloud] Unignore ignored playlist tests --- .../services/soundcloud/SoundcloudPlaylistExtractorTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index f0832ef28..d4ab882fa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -269,14 +269,11 @@ public class SoundcloudPlaylistExtractorTest { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ - @Ignore @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } - //TODO: FUCK THIS: This triggers a 500 at sever - @Ignore @Test public void testMoreRelatedItems() throws Exception { ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor); @@ -291,7 +288,6 @@ public class SoundcloudPlaylistExtractorTest { // PlaylistExtractor //////////////////////////////////////////////////////////////////////////*/ - @Ignore @Test public void testThumbnailUrl() { assertIsSecureUrl(extractor.getThumbnailUrl()); From 0e1b4bbf17d460b081b409c35d276d4ba0eed15f Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:53:25 +0100 Subject: [PATCH 16/66] [SoundCloud] Test playlists banner: it should not exist --- .../soundcloud/SoundcloudPlaylistExtractorTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index d4ab882fa..6f1af547f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -94,10 +94,10 @@ public class SoundcloudPlaylistExtractorTest { assertIsSecureUrl(extractor.getThumbnailUrl()); } - @Ignore @Test public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + // SoundCloud playlists do not have a banner + assertNull(extractor.getBannerUrl()); } @Test @@ -186,10 +186,10 @@ public class SoundcloudPlaylistExtractorTest { assertIsSecureUrl(extractor.getThumbnailUrl()); } - @Ignore("not implemented") @Test public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + // SoundCloud playlists do not have a banner + assertNull(extractor.getBannerUrl()); } @Test @@ -293,10 +293,10 @@ public class SoundcloudPlaylistExtractorTest { assertIsSecureUrl(extractor.getThumbnailUrl()); } - @Ignore @Test public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + // SoundCloud playlists do not have a banner + assertNull(extractor.getBannerUrl()); } @Test From 5e4ddb368fe25b809e28428119b032f93b4ce28d Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 18:04:40 +0100 Subject: [PATCH 17/66] [SoundCloud] Fix extractors built from next playlist pages They didn't have the information to calculate another next page url. So now `nextPageUrl` contains a full link with all video ids, and `getPage` takes the first part of the url (containing 15 streams) and produces another `nextPageUrl` with the remaining streams. Also add a test for this. --- .../SoundcloudPlaylistExtractor.java | 62 +++++++++---------- .../SoundcloudPlaylistExtractorTest.java | 9 +++ 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index a57abea8b..c82180773 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -30,8 +30,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private JsonObject playlist; private StreamInfoItemsCollector streamInfoItemsCollector; - private List nextTrackIds; - private int nextTrackIdsIndex; private String nextPageUrl; public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -121,15 +119,16 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if (streamInfoItemsCollector == null) { - computeInitialTracksAndNextIds(); + computeInitialTracksAndNextPageUrl(); } - return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); + return new InfoItemsPage<>(streamInfoItemsCollector, nextPageUrl); } - private void computeInitialTracksAndNextIds() { + private void computeInitialTracksAndNextPageUrl() throws IOException, ExtractionException { streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); - nextTrackIds = new ArrayList<>(); - nextTrackIdsIndex = 0; + StringBuilder nextPageUrlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id="); + nextPageUrlBuilder.append(SoundcloudParsingHelper.clientId()); + nextPageUrlBuilder.append("&ids="); JsonArray tracks = playlist.getArray("tracks"); for (Object o : tracks) { @@ -138,39 +137,23 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { if (track.has("title")) { // i.e. if full info is available streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); } else { - nextTrackIds.add(track.getInt("id")); + // %09d would be enough, but a 0 before the number does not create problems, so let's be sure + nextPageUrlBuilder.append(String.format("%010d,", track.getInt("id"))); } } } - } - private void computeAnotherNextPageUrl() throws IOException, ExtractionException { - if (nextTrackIds == null || nextTrackIdsIndex >= nextTrackIds.size()) { - nextPageUrl = ""; // there are no more tracks - return; + nextPageUrl = nextPageUrlBuilder.toString(); + if (nextPageUrl.endsWith("&ids=")) { + // there are no other videos + nextPageUrl = ""; } - - StringBuilder urlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id="); - urlBuilder.append(SoundcloudParsingHelper.clientId()); - urlBuilder.append("&ids="); - - int upperIndex = Math.min(nextTrackIdsIndex + streamsPerRequestedPage, nextTrackIds.size()); - for (int i = nextTrackIdsIndex; i < upperIndex; ++i) { - urlBuilder.append(nextTrackIds.get(i)); - urlBuilder.append(","); // a , at the end is ok - } - - nextTrackIdsIndex = upperIndex; - nextPageUrl = urlBuilder.toString(); } @Override public String getNextPageUrl() throws IOException, ExtractionException { if (nextPageUrl == null) { - if (nextTrackIds == null) { - computeInitialTracksAndNextIds(); - } - computeAnotherNextPageUrl(); + computeInitialTracksAndNextPageUrl(); } return nextPageUrl; } @@ -181,8 +164,24 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } + // see computeInitialTracksAndNextPageUrl + final int lengthFirstPartOfUrl = ("https://api-v2.soundcloud.com/tracks?client_id=" + + SoundcloudParsingHelper.clientId() + + "&ids=").length(); + final int lengthOfEveryStream = 11; + + String currentPageUrl; + int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage; + if (pageUrl.length() <= lengthMaxStreams) { + currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max + nextPageUrl = ""; // afterwards the list is complete + } else { + currentPageUrl = pageUrl.substring(0, lengthMaxStreams); + nextPageUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams); + } + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String response = NewPipe.getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); + String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody(); try { JsonArray tracks = JsonParser.array().from(response); @@ -195,7 +194,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { throw new ParsingException("Could not parse json response", e); } - computeAnotherNextPageUrl(); return new InfoItemsPage<>(collector, nextPageUrl); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 6f1af547f..700d2e6dc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -234,6 +234,15 @@ public class SoundcloudPlaylistExtractorTest { public void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor); + String page1 = newExtractor.getNextPageUrl(); + defaultTestMoreItems(newExtractor); // there has to be another page + String page2 = newExtractor.getNextPageUrl(); + defaultTestMoreItems(newExtractor); // and another one + String page3 = newExtractor.getNextPageUrl(); + + assertNotEquals("Same pages", page1, page2); + assertNotEquals("Same pages", page2, page3); + assertNotEquals("Same pages", page3, page1); } /*////////////////////////////////////////////////////////////////////////// From 4389fd3b7b37608be495a17f93e63cf1b6feba8b Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 18:06:13 +0100 Subject: [PATCH 18/66] [SoundCloud] Migrate StreamExtractor to api-v2 --- .../services/soundcloud/SoundcloudParsingHelper.java | 2 +- .../services/soundcloud/SoundcloudStreamExtractor.java | 8 +++++--- .../soundcloud/SoundcloudStreamExtractorDefaultTest.java | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 6362f13cb..a957ed810 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -108,7 +108,7 @@ public class SoundcloudParsingHelper { * See https://developers.soundcloud.com/docs/api/reference#resolve */ public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { - String apiUrl = "https://api.soundcloud.com/resolve" + String apiUrl = "https://api-v2.soundcloud.com/resolve" + "?url=" + URLEncoder.encode(url, "UTF-8") + "&client_id=" + clientId(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 2919c89be..805db216b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -19,6 +19,8 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -55,14 +57,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getTextualUploadDate() { - return track.getString("created_at"); + public String getTextualUploadDate() throws ParsingException { + return track.getString("created_at").replace("T"," ").replace("Z", ""); } @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate())); + return new DateWrapper(SoundcloudParsingHelper.parseDate(track.getString("created_at"))); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 0ffe06223..86203f4c3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -74,7 +74,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2016/07/31 18:18:07 +0000", extractor.getTextualUploadDate()); + Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate()); } @Test From ae47c9587c186013bb278d9aaafd0a9a1d8c0946 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 20:31:01 +0100 Subject: [PATCH 19/66] [SoundCloud] Optimize imports in edited files --- .../services/soundcloud/SoundcloudPlaylistExtractor.java | 6 ++---- .../services/soundcloud/SoundcloudStreamExtractor.java | 6 +++--- .../soundcloud/SoundcloudPlaylistExtractorTest.java | 4 ---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index c82180773..a121e4db0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -15,13 +15,11 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import java.io.IOException; + import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - @SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private static final int streamsPerRequestedPage = 15; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 805db216b..5c176720c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -15,17 +16,16 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; -import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.text.SimpleDateFormat; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import javax.annotation.Nonnull; + public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 700d2e6dc..28634207f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -2,18 +2,14 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.hamcrest.CoreMatchers; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import java.io.IOException; - import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; From c505d4e2b77b12e4d2d259121d66937ded756de1 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 20:36:59 +0100 Subject: [PATCH 20/66] [SoundCloud] Remove trailing , in playlist page urls --- .../services/soundcloud/SoundcloudPlaylistExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index a121e4db0..051bfc006 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -141,8 +141,9 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } } + nextPageUrlBuilder.setLength(nextPageUrlBuilder.length() - 1); // remove trailing , nextPageUrl = nextPageUrlBuilder.toString(); - if (nextPageUrl.endsWith("&ids=")) { + if (nextPageUrl.endsWith("&ids")) { // there are no other videos nextPageUrl = ""; } From 45bb646480b13c59d5c6f9aef53b257c51fbf0d5 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 20:56:47 +0100 Subject: [PATCH 21/66] [SoundCloud] Do not overwrite nextPageUrl in PlaylistExtractor Consistent with YouTube and the documentation --- .../services/soundcloud/SoundcloudPlaylistExtractor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 051bfc006..060b45caf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -169,14 +169,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { + "&ids=").length(); final int lengthOfEveryStream = 11; - String currentPageUrl; + String currentPageUrl, nextUrl; int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage; if (pageUrl.length() <= lengthMaxStreams) { currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max - nextPageUrl = ""; // afterwards the list is complete + nextUrl = ""; // afterwards the list is complete } else { currentPageUrl = pageUrl.substring(0, lengthMaxStreams); - nextPageUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams); + nextUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams); } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); @@ -193,6 +193,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { throw new ParsingException("Could not parse json response", e); } - return new InfoItemsPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextUrl); } } From 265cfb61f37c6a626b56f552e5619c95b6a3a34c Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 21:03:50 +0100 Subject: [PATCH 22/66] [SoundCloud] Add test for small playlist, use defaultTestGetPageInNewExtractor --- .../SoundcloudPlaylistExtractorTest.java | 127 ++++++++++++++---- 1 file changed, 104 insertions(+), 23 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 28634207f..59ecff492 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.hamcrest.CoreMatchers; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +9,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static org.hamcrest.CoreMatchers.*; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -71,14 +71,6 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testMoreRelatedItems() throws Exception { defaultTestMoreItems(extractor); - - try { - defaultTestMoreItems(extractor); - } catch (Throwable ignored) { - return; - } - - fail("This playlist doesn't have more items, it should throw an error"); } /*////////////////////////////////////////////////////////////////////////// @@ -100,7 +92,7 @@ public class SoundcloudPlaylistExtractorTest { public void testUploaderUrl() { final String uploaderUrl = extractor.getUploaderUrl(); assertIsSecureUrl(uploaderUrl); - assertTrue(uploaderUrl, uploaderUrl.contains("liluzivert")); + assertThat(uploaderUrl, containsString("liluzivert")); } @Test @@ -115,7 +107,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testStreamCount() { - assertTrue("Error in the streams count", extractor.getStreamCount() >= 10); + assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 10); } } @@ -192,7 +184,7 @@ public class SoundcloudPlaylistExtractorTest { public void testUploaderUrl() { final String uploaderUrl = extractor.getUploaderUrl(); assertIsSecureUrl(uploaderUrl); - assertThat(uploaderUrl, CoreMatchers.containsString("micky96")); + assertThat(uploaderUrl, containsString("micky96")); } @Test @@ -207,7 +199,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testStreamCount() { - assertTrue("Error in the streams count", extractor.getStreamCount() >= 10); + assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 10); } } @@ -228,17 +220,8 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); + PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor); - String page1 = newExtractor.getNextPageUrl(); - defaultTestMoreItems(newExtractor); // there has to be another page - String page2 = newExtractor.getNextPageUrl(); - defaultTestMoreItems(newExtractor); // and another one - String page3 = newExtractor.getNextPageUrl(); - - assertNotEquals("Same pages", page1, page2); - assertNotEquals("Same pages", page2, page3); - assertNotEquals("Same pages", page3, page1); } /*////////////////////////////////////////////////////////////////////////// @@ -326,4 +309,102 @@ public class SoundcloudPlaylistExtractorTest { assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 370); } } + + public static class SmallPlaylist implements BasePlaylistExtractorTest { + private static SoundcloudPlaylistExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (SoundcloudPlaylistExtractor) SoundCloud + .getPlaylistExtractor("https://soundcloud.com/breezy-123/sets/empty-playlist?test=123"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(SoundCloud.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() { + assertEquals("EMPTY PLAYLIST", extractor.getName()); + } + + @Test + public void testId() { + assertEquals("23483459", extractor.getId()); + } + + @Test + public void testUrl() throws Exception { + assertEquals("https://soundcloud.com/breezy-123/sets/empty-playlist", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws Exception { + assertEquals("https://soundcloud.com/breezy-123/sets/empty-playlist?test=123", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + try { + defaultTestMoreItems(extractor); + } catch (Throwable ignored) { + return; + } + + fail("This playlist doesn't have more items, it should throw an error"); + } + + /*////////////////////////////////////////////////////////////////////////// + // PlaylistExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testThumbnailUrl() { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testBannerUrl() { + // SoundCloud playlists do not have a banner + assertNull(extractor.getBannerUrl()); + } + + @Test + public void testUploaderUrl() { + final String uploaderUrl = extractor.getUploaderUrl(); + assertIsSecureUrl(uploaderUrl); + assertThat(uploaderUrl, containsString("breezy-123")); + } + + @Test + public void testUploaderName() { + assertEquals("breezy-123", extractor.getUploaderName()); + } + + @Test + public void testUploaderAvatarUrl() { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testStreamCount() { + assertEquals(2, extractor.getStreamCount()); + } + } } From 904c9d681fb37449231909ac8452cf95835c5136 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 17 Mar 2020 15:00:07 +0100 Subject: [PATCH 23/66] Fix YouTube videos with no views --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 04818de07..e309ed863 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -279,6 +279,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (views == null) throw new ParsingException("Could not get view count"); } + if (views.toLowerCase().contains("no views")) return 0; + return Long.parseLong(Utils.removeNonDigitCharacters(views)); } From 222d659d9e6ebebed5ea9a690518b1a7c1ac0c2d Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 18 Mar 2020 11:01:46 +0100 Subject: [PATCH 24/66] [SoundCloud] Don't make separate request for getAudioStreams() in StreamExtractor Signed-off-by: Stypox --- .../soundcloud/SoundcloudStreamExtractor.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 5c176720c..16eaad012 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -14,7 +14,14 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -148,24 +155,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor { List audioStreams = new ArrayList<>(); Downloader dl = NewPipe.getDownloader(); - String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) - + "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId()); - - String response = dl.get(apiUrl, getExtractorLocalization()).responseBody(); - JsonObject responseObject; - try { - responseObject = JsonParser.object().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json response", e); - } - // Streams can be streamable and downloadable - or explicitly not. // For playing the track, it is only necessary to have a streamable track. // If this is not the case, this track might not be published yet. - if (!responseObject.getBoolean("streamable")) return audioStreams; + if (!track.getBoolean("streamable")) return audioStreams; try { - JsonArray transcodings = responseObject.getObject("media").getArray("transcodings"); + JsonArray transcodings = track.getObject("media").getArray("transcodings"); // get information about what stream formats are available for (Object transcoding : transcodings) { From 0c27198ba17bc8721fb712fd187dab8153743fdb Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 02:21:23 -0300 Subject: [PATCH 25/66] Include errors from child collectors in the search collector --- .../search/InfoItemsSearchCollector.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java index 206fb68fb..bfa8201a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java @@ -11,6 +11,10 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + /* * Created by Christian Schabesberger on 12.02.17. * @@ -55,6 +59,24 @@ public class InfoItemsSearchCollector extends InfoItemsCollector getErrors() { + final List errors = new ArrayList<>(super.getErrors()); + errors.addAll(streamCollector.getErrors()); + errors.addAll(userCollector.getErrors()); + errors.addAll(playlistCollector.getErrors()); + + return Collections.unmodifiableList(errors); + } + + @Override + public void reset() { + super.reset(); + streamCollector.reset(); + userCollector.reset(); + playlistCollector.reset(); + } + @Override public InfoItem extract(InfoItemExtractor extractor) throws ParsingException { // Use the corresponding collector for each item extractor type From d72130edae12ff4a85ad18b9957adb8bdef68930 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 02:27:31 -0300 Subject: [PATCH 26/66] Always return a new item collector in the search extractor --- .../org/schabi/newpipe/extractor/search/SearchExtractor.java | 5 +---- .../media_ccc/extractors/MediaCCCSearchExtractor.java | 3 +-- .../services/soundcloud/SoundcloudSearchExtractor.java | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index 0acc640e6..8b900ab7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -17,11 +17,8 @@ public abstract class SearchExtractor extends ListExtractor { } } - private final InfoItemsSearchCollector collector; - public SearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); - collector = new InfoItemsSearchCollector(service.getServiceId()); } public String getSearchString() { @@ -31,7 +28,7 @@ public abstract class SearchExtractor extends ListExtractor { public abstract String getSearchSuggestion() throws ParsingException; protected InfoItemsSearchCollector getInfoItemSearchCollector() { - return collector; + return new InfoItemsSearchCollector(getService().getServiceId()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 6d740337a..241d6d1e4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -47,8 +47,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); - searchItems.reset(); + final InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index 42cd8ae0f..de02af6bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -79,7 +79,6 @@ public class SoundcloudSearchExtractor extends SearchExtractor { private InfoItemsCollector collectItems(JsonArray searchCollection) { final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - collector.reset(); for (Object result : searchCollection) { if (!(result instanceof JsonObject)) continue; From 9704fc99521959c7abea90d6600dfa1d40de3bca Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:13:11 -0300 Subject: [PATCH 27/66] Improve search extractor tests for services --- .../services/BaseSearchExtractorTest.java | 7 + .../services/DefaultExtractorTest.java | 59 ++++++ .../services/DefaultListExtractorTest.java | 50 +++++ .../services/DefaultSearchExtractorTest.java | 34 ++++ .../extractor/services/DefaultTests.java | 33 ++- .../MediaCCCSearchExtractorAllTest.java | 58 ------ ...ediaCCCSearchExtractorConferencesTest.java | 48 ----- .../MediaCCCSearchExtractorEventsTest.java | 72 ------- .../search/MediaCCCSearchExtractorTest.java | 90 +++++++++ .../PeertubeSearchExtractorBaseTest.java | 28 --- .../PeertubeSearchExtractorDefaultTest.java | 89 -------- .../search/PeertubeSearchExtractorTest.java | 59 ++++++ .../SoundcloudSearchExtractorBaseTest.java | 55 ----- ...ndcloudSearchExtractorChannelOnlyTest.java | 66 ------ .../SoundcloudSearchExtractorDefaultTest.java | 104 ---------- .../search/SoundcloudSearchExtractorTest.java | 135 +++++++++++++ .../YoutubeSearchExtractorBaseTest.java | 50 ----- ...YoutubeSearchExtractorChannelOnlyTest.java | 115 ----------- .../YoutubeSearchExtractorDefaultTest.java | 142 ------------- .../search/YoutubeSearchExtractorTest.java | 190 ++++++++++++++++++ .../search/YoutubeSearchPagingTest.java | 69 ------- 21 files changed, 653 insertions(+), 900 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorBaseTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java new file mode 100644 index 000000000..e82ad7d0a --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java @@ -0,0 +1,7 @@ +package org.schabi.newpipe.extractor.services; + +@SuppressWarnings("unused") +public interface BaseSearchExtractorTest extends BaseListExtractorTest { + void testSearchString() throws Exception; + void testSearchSuggestion() throws Exception; +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java new file mode 100644 index 000000000..40891d494 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java @@ -0,0 +1,59 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.Test; +import org.schabi.newpipe.extractor.Extractor; +import org.schabi.newpipe.extractor.StreamingService; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; + +public abstract class DefaultExtractorTest implements BaseExtractorTest { + public abstract T extractor() throws Exception; + + public abstract StreamingService expectedService() throws Exception; + public abstract String expectedName() throws Exception; + public abstract String expectedId() throws Exception; + public abstract String expectedUrlContains() throws Exception; + public abstract String expectedOriginalUrlContains() throws Exception; + + @Test + @Override + public void testServiceId() throws Exception { + assertEquals(expectedService().getServiceId(), extractor().getServiceId()); + } + + @Test + @Override + public void testName() throws Exception { + assertEquals(expectedName(), extractor().getName()); + } + + @Test + @Override + public void testId() throws Exception { + assertEquals(expectedId(), extractor().getId()); + } + + @Test + @Override + public void testUrl() throws Exception { + final String url = extractor().getUrl(); + final String expectedContains = expectedUrlContains(); + + assertIsSecureUrl(url); + assertTrue("Url \"" + url + "\" doesn't contains \"" + expectedContains + "\"", + url.contains(expectedContains)); + } + + @Test + @Override + public void testOriginalUrl() throws Exception { + final String originalUrl = extractor().getOriginalUrl(); + final String expectedContains = expectedOriginalUrlContains(); + + assertIsSecureUrl(originalUrl); + assertTrue("Original url \"" + originalUrl + "\" doesn't contains \"" + expectedContains + "\"", + originalUrl.contains(expectedContains)); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java new file mode 100644 index 000000000..462eba1a1 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java @@ -0,0 +1,50 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.Test; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; + +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + +public abstract class DefaultListExtractorTest> extends DefaultExtractorTest + implements BaseListExtractorTest { + + @Nullable + public InfoItem.InfoType expectedInfoItemType() { + return null; + } + + public boolean expectedHasMoreItems() { + return true; + } + + @Test + @Override + public void testRelatedItems() throws Exception { + final ListExtractor extractor = extractor(); + + final InfoItem.InfoType expectedType = expectedInfoItemType(); + final ListExtractor.InfoItemsPage items = defaultTestRelatedItems(extractor); + if (expectedType != null) { + assertOnlyContainsType(items, expectedType); + } + } + + @Test + @Override + public void testMoreRelatedItems() throws Exception { + final ListExtractor extractor = extractor(); + + if (expectedHasMoreItems()) { + final InfoItem.InfoType expectedType = expectedInfoItemType(); + final ListExtractor.InfoItemsPage items = defaultTestMoreItems(extractor); + if (expectedType != null) { + assertOnlyContainsType(items, expectedType); + } + } else { + assertNoMoreItems(extractor); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java new file mode 100644 index 000000000..93c4eac70 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java @@ -0,0 +1,34 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.Test; +import org.schabi.newpipe.extractor.search.SearchExtractor; + + +import javax.annotation.Nullable; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; + +public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTest + implements BaseSearchExtractorTest { + + public abstract String expectedSearchString(); + @Nullable public abstract String expectedSearchSuggestion(); + + @Test + @Override + public void testSearchString() throws Exception { + assertEquals(expectedSearchString(), extractor().getSearchString()); + } + + @Test + @Override + public void testSearchSuggestion() throws Exception { + final String expectedSearchSuggestion = expectedSearchSuggestion(); + if (expectedSearchSuggestion == null || expectedSearchSuggestion.isEmpty()) { + assertEmpty("Suggestion was expected to be empty", extractor().getSearchSuggestion()); + } else { + assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion()); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index 6c2dd9ef9..b34bbbff9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -2,23 +2,22 @@ package org.schabi.newpipe.extractor.services; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.util.Calendar; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.*; -import static org.schabi.newpipe.extractor.StreamingService.*; +import static org.schabi.newpipe.extractor.StreamingService.LinkType; public final class DefaultTests { public static void defaultTestListOfItems(StreamingService expectedService, List itemsList, List errors) throws ParsingException { @@ -71,12 +70,38 @@ public final class DefaultTests { expectedLinkType, linkTypeByUrl); } + public static void assertOnlyContainsType(ListExtractor.InfoItemsPage items, InfoItem.InfoType expectedType) { + for (InfoItem item : items.getItems()) { + assertEquals("Item list contains unexpected info types", + expectedType, item.getInfoType()); + } + } + public static void assertNoMoreItems(ListExtractor extractor) throws Exception { assertFalse("More items available when it shouldn't", extractor.hasNextPage()); final String nextPageUrl = extractor.getNextPageUrl(); assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty()); } + public static void assertNoDuplicatedItems(StreamingService expectedService, + ListExtractor.InfoItemsPage page1, + ListExtractor.InfoItemsPage page2) throws Exception { + defaultTestListOfItems(expectedService, page1.getItems(), page1.getErrors()); + defaultTestListOfItems(expectedService, page2.getItems(), page2.getErrors()); + + final Set urlsSet = new HashSet<>(); + for (InfoItem item : page1.getItems()) { + urlsSet.add(item.getUrl()); + } + + for (InfoItem item : page2.getItems()) { + final boolean wasAdded = urlsSet.add(item.getUrl()); + if (!wasAdded) { + fail("Same item was on the first and second page item list"); + } + } + } + public static ListExtractor.InfoItemsPage defaultTestRelatedItems(ListExtractor extractor) throws Exception { final ListExtractor.InfoItemsPage page = extractor.getInitialPage(); final List itemsList = page.getItems(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java deleted file mode 100644 index cd33c562a..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.schabi.newpipe.extractor.services.media_ccc; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.util.Arrays; - -import static junit.framework.TestCase.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; - -/** - * Test for {@link MediaCCCSearchExtractor} - */ -public class MediaCCCSearchExtractorAllTest { - - private static SearchExtractor extractor; - private static ListExtractor.InfoItemsPage itemsPage; - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() - .fromQuery("c3", Arrays.asList(new String[0]), "")); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testIfChannelInfoItemsAvailable() { - boolean isAvialable = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof ChannelInfoItem) { - isAvialable = true; - } - } - assertTrue("ChannelInfoItem not in all list", isAvialable); - } - - @Test - public void testIfStreamInfoitemsAvailable() { - boolean isAvialable = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof StreamInfoItem) { - isAvialable = true; - } - } - assertTrue("ChannelInfoItem not in all list", isAvialable); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java deleted file mode 100644 index f9ce2334f..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.schabi.newpipe.extractor.services.media_ccc; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; - -import java.util.Arrays; - -import static junit.framework.TestCase.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; - -/** - * Test for {@link MediaCCCSearchExtractor} - */ -public class MediaCCCSearchExtractorConferencesTest { - - private static SearchExtractor extractor; - private static ListExtractor.InfoItemsPage itemsPage; - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() - .fromQuery("c3", Arrays.asList(new String[]{"conferences"}), "")); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testReturnTypeChannel() { - for (InfoItem item : itemsPage.getItems()) { - assertTrue("Item is not of type channel", item instanceof ChannelInfoItem); - } - } - - @Test - public void testItemCount() { - assertTrue("Count is to hight: " + itemsPage.getItems().size(), itemsPage.getItems().size() < 127); - assertTrue("Countis to low: " + itemsPage.getItems().size(), itemsPage.getItems().size() >= 29); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java deleted file mode 100644 index 29f90caf3..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.schabi.newpipe.extractor.services.media_ccc; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.util.Arrays; - -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; - -/** - * Test for {@link MediaCCCSearchExtractor} - */ -public class MediaCCCSearchExtractorEventsTest { - private static SearchExtractor extractor; - private static ListExtractor.InfoItemsPage itemsPage; - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() - .fromQuery("linux", Arrays.asList(new String[]{"events"}), "")); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testCount() throws Exception { - assertTrue(Integer.toString(itemsPage.getItems().size()), - itemsPage.getItems().size() >= 25); - } - - @Test - public void testServiceId() throws Exception { - assertEquals(2, extractor.getServiceId()); - } - - @Test - public void testName() throws Exception { - assertFalse(itemsPage.getItems().get(0).getName(), itemsPage.getItems().get(0).getName().isEmpty()); - } - - @Test - public void testUrl() throws Exception { - assertTrue("Url should start with: https://api.media.ccc.de/public/events/", - itemsPage.getItems().get(0).getUrl().startsWith("https://api.media.ccc.de/public/events/")); - } - - @Test - public void testThumbnailUrl() throws Exception { - assertTrue(itemsPage.getItems().get(0).getThumbnailUrl(), - itemsPage.getItems().get(0).getThumbnailUrl().startsWith("https://static.media.ccc.de/media/") - && itemsPage.getItems().get(0).getThumbnailUrl().endsWith(".jpg")); - } - - @Test - public void testReturnTypeStream() throws Exception { - for (InfoItem item : itemsPage.getItems()) { - assertTrue("Item is not of type StreamInfoItem", item instanceof StreamInfoItem); - } - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java new file mode 100644 index 000000000..812d6367a --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java @@ -0,0 +1,90 @@ +package org.schabi.newpipe.extractor.services.media_ccc.search; + +import org.junit.BeforeClass; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; + +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES; +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS; + +public class MediaCCCSearchExtractorTest { + + public static class All extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "kde"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = MediaCCC.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return MediaCCC; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public boolean expectedHasMoreItems() { return false; } + } + + public static class Conferences extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "c3"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = MediaCCC.getSearchExtractor(QUERY, singletonList(CONFERENCES), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return MediaCCC; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Nullable @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + @Override public boolean expectedHasMoreItems() { return false; } + } + + public static class Events extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "linux"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = MediaCCC.getSearchExtractor(QUERY, singletonList(EVENTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return MediaCCC; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Nullable @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean expectedHasMoreItems() { return false; } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java deleted file mode 100644 index 030bd6c53..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.schabi.newpipe.extractor.services.peertube.search; - -import org.junit.Test; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; - -import static org.junit.Assert.assertTrue; - -/** - * Test for {@link PeertubeSearchExtractor} - */ -public abstract class PeertubeSearchExtractorBaseTest { - - protected static PeertubeSearchExtractor extractor; - protected static ListExtractor.InfoItemsPage itemsPage; - - @Test - public void testResultListElementsLength() { - assertTrue(Integer.toString(itemsPage.getItems().size()), - itemsPage.getItems().size() >= 3); - } - - @Test - public void testUrl() throws Exception { - assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://peertube.mastodon.host/api/v1/search/videos")); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java deleted file mode 100644 index eeebf526b..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.schabi.newpipe.extractor.services.peertube.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -/** - * Test for {@link PeertubeSearchExtractor} - */ -public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde"); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=kde&start=12&count=12", extractor.getNextPageUrl()); - } - - @Test - public void testResultList_FirstElement() { - InfoItem firstInfoItem = itemsPage.getItems().get(0); - - assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde")); - } - - @Test - public void testResultListCheckIfContainsStreamItems() { - boolean hasStreams = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof StreamInfoItem) { - hasStreams = true; - } - } - assertTrue("Has no InfoItemStreams", hasStreams); - } - - @Test - public void testGetSecondPage() throws Exception { - extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); - itemsPage = extractor.getInitialPage(); - PeertubeSearchExtractor secondExtractor = - (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() >= 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - - assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12", - secondPage.getNextPageUrl()); - } - - - @Test - public void testId() throws Exception { - assertEquals("kde", extractor.getId()); - } - - @Test - public void testName() { - assertEquals("kde", extractor.getName()); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java new file mode 100644 index 000000000..5ea116333 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java @@ -0,0 +1,59 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; +import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; + +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; +import static org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory.VIDEOS; + +public class PeertubeSearchExtractorTest { + + public static class All extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "kde"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = PeerTube.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return PeerTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "/search/videos?search=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "/search/videos?search=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } + + public static class PagingTest { + @Test + public void duplicatedItemsCheck() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + final SearchExtractor extractor = PeerTube.getSearchExtractor("internet", singletonList(VIDEOS), ""); + extractor.fetchPage(); + + final InfoItemsPage page1 = extractor.getInitialPage(); + final InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + + assertNoDuplicatedItems(PeerTube, page1, page2); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorBaseTest.java deleted file mode 100644 index db38c8d4a..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorBaseTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.schabi.newpipe.extractor.services.soundcloud.search; - -import org.junit.Test; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor; - -import static org.junit.Assert.assertTrue; - - -/* - * Created by Christian Schabesberger on 17.06.18 - * - * Copyright (C) Christian Schabesberger 2018 - * SoundcloudSearchExtractorBaseTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** - * Test for {@link SoundcloudSearchExtractor} - */ -public abstract class SoundcloudSearchExtractorBaseTest { - - protected static SoundcloudSearchExtractor extractor; - protected static ListExtractor.InfoItemsPage itemsPage; - - - protected static String removeClientId(String url) { - String[] splitUrl = url.split("client_id=[a-zA-Z0-9]*&"); - return splitUrl[0] + splitUrl[1]; - } - - @Test - public void testResultListElementsLength() { - assertTrue(Integer.toString(itemsPage.getItems().size()), - itemsPage.getItems().size() >= 3); - } - - @Test - public void testUrl() throws Exception { - assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://api-v2.soundcloud.com/search")); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java deleted file mode 100644 index 04e7e0aba..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.schabi.newpipe.extractor.services.soundcloud.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.localization.Localization; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory; - -import static java.util.Arrays.asList; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; - -public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance(), new Localization("de", "DE")); - extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert", - asList(SoundcloudSearchQueryHandlerFactory.USERS), null); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetSecondPage() throws Exception { - SoundcloudSearchExtractor secondExtractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert", - asList(SoundcloudSearchQueryHandlerFactory.USERS), null); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() >= 3); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - - assertEquals("https://api-v2.soundcloud.com/search/users?q=lill+uzi+vert&limit=10&offset=20", - removeClientId(secondPage.getNextPageUrl())); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - assertEquals("https://api-v2.soundcloud.com/search/users?q=lill+uzi+vert&limit=10&offset=10", - removeClientId(extractor.getNextPageUrl())); - } - - @Test - public void testOnlyContainChannels() { - for (InfoItem item : itemsPage.getItems()) { - if (!(item instanceof ChannelInfoItem)) { - fail("The following item is no channel item: " + item.toString()); - } - } - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java deleted file mode 100644 index ca0d7d590..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.schabi.newpipe.extractor.services.soundcloud.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.util.Arrays; - -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; - -/* - * Created by Christian Schabesberger on 27.05.18 - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeSearchExtractorStreamTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** - * Test for {@link YoutubeSearchExtractor} - */ -public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor( - new SoundcloudSearchQueryHandlerFactory().fromQuery("lill uzi vert", - Arrays.asList(new String[]{"tracks"}), "")); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - assertEquals("https://api-v2.soundcloud.com/search/tracks?q=lill+uzi+vert&limit=10&offset=10", - removeClientId(extractor.getNextPageUrl())); - } - - @Test - public void testResultListCheckIfContainsStreamItems() { - boolean hasStreams = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof StreamInfoItem) { - hasStreams = true; - } - } - assertTrue("Has no InfoItemStreams", hasStreams); - } - - @Test - public void testGetSecondPage() throws Exception { - SoundcloudSearchExtractor secondExtractor = - (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert"); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() >= 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - - assertEquals("https://api-v2.soundcloud.com/search/tracks?q=lill+uzi+vert&limit=10&offset=20", - removeClientId(secondPage.getNextPageUrl())); - } - - - @Test - public void testId() throws Exception { - assertEquals("lill uzi vert", extractor.getId()); - } - - @Test - public void testName() { - assertEquals("lill uzi vert", extractor.getName()); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java new file mode 100644 index 000000000..8503f5733 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java @@ -0,0 +1,135 @@ +package org.schabi.newpipe.extractor.services.soundcloud.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; + +import javax.annotation.Nullable; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.*; + +public class SoundcloudSearchExtractorTest { + + public static class All extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "lill uzi vert"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } + + public static class Tracks extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "lill uzi vert"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY, singletonList(TRACKS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class Users extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "lill uzi vert"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY, singletonList(USERS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + } + + public static class Playlists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "lill uzi vert"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY, singletonList(PLAYLISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + public static class PagingTest { + @Test + public void duplicatedItemsCheck() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + final SearchExtractor extractor = SoundCloud.getSearchExtractor("cirque du soleil", singletonList(TRACKS), ""); + extractor.fetchPage(); + + final InfoItemsPage page1 = extractor.getInitialPage(); + final InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + + assertNoDuplicatedItems(SoundCloud, page1, page2); + } + } + + private static String urlEncode(String value) { + try { + return URLEncoder.encode(value, CHARSET_UTF_8); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java deleted file mode 100644 index 729254ad9..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.junit.Test; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; - -import static org.junit.Assert.assertTrue; - - -/* - * Created by Christian Schabesberger on 27.05.18 - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeSearchExtractorBaseTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** - * Test for {@link YoutubeSearchExtractor} - */ -public abstract class YoutubeSearchExtractorBaseTest { - - protected static YoutubeSearchExtractor extractor; - protected static ListExtractor.InfoItemsPage itemsPage; - - - @Test - public void testResultListElementsLength() { - assertTrue(Integer.toString(itemsPage.getItems().size()), - itemsPage.getItems().size() > 10); - } - - @Test - public void testUrl() throws Exception { - assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://www.youtube.com")); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java deleted file mode 100644 index cb72622b4..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.hamcrest.CoreMatchers; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; - -import java.net.URL; -import java.net.URLDecoder; -import java.util.LinkedHashMap; -import java.util.Map; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetSecondPage() throws Exception { - YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() > 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - URL url = new URL(extractor.getNextPageUrl()); - - assertEquals(url.getHost(), "www.youtube.com"); - assertEquals(url.getPath(), "/results"); - - Map queryPairs = new LinkedHashMap<>(); - for (String queryPair : url.getQuery().split("&")) { - int index = queryPair.indexOf("="); - queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), - URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); - } - - assertEquals("pewdiepie", queryPairs.get("search_query")); - assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); - assertTrue(queryPairs.get("continuation").length() > 5); - assertTrue(queryPairs.get("itct").length() > 5); - } - - @Ignore - @Test - public void testOnlyContainChannels() { - for (InfoItem item : itemsPage.getItems()) { - if (!(item instanceof ChannelInfoItem)) { - fail("The following item is no channel item: " + item.toString()); - } - } - } - - @Test - public void testChannelUrl() { - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof ChannelInfoItem) { - ChannelInfoItem channel = (ChannelInfoItem) item; - - if (channel.getSubscriberCount() > 1e8) { // the real PewDiePie - assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); - break; - } - } - } - - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof ChannelInfoItem) { - assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); - } - } - } - - @Test - public void testStreamCount() { - ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0); - assertTrue("Stream count does not fit: " + ci.getStreamCount(), - 4000 < ci.getStreamCount() && ci.getStreamCount() < 5500); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java deleted file mode 100644 index 1f905bdec..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ /dev/null @@ -1,142 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.net.URL; -import java.net.URLDecoder; -import java.util.LinkedHashMap; -import java.util.Map; - -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -/* - * Created by Christian Schabesberger on 27.05.18 - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeSearchExtractorStreamTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** - * Test for {@link YoutubeSearchExtractor} - */ -public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie"); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetUrl() throws Exception { - assertEquals("https://www.youtube.com/results?search_query=pewdiepie&gl=GB", extractor.getUrl()); - } - - - @Test - public void testGetSecondPageUrl() throws Exception { - URL url = new URL(extractor.getNextPageUrl()); - - assertEquals(url.getHost(), "www.youtube.com"); - assertEquals(url.getPath(), "/results"); - - Map queryPairs = new LinkedHashMap<>(); - for (String queryPair : url.getQuery().split("&")) { - int index = queryPair.indexOf("="); - queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), - URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); - } - - assertEquals("pewdiepie", queryPairs.get("search_query")); - assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); - assertTrue(queryPairs.get("continuation").length() > 5); - assertTrue(queryPairs.get("itct").length() > 5); - } - - @Test - public void testResultList_FirstElement() { - InfoItem firstInfoItem = itemsPage.getItems().get(0); - InfoItem secondInfoItem = itemsPage.getItems().get(1); - - InfoItem channelItem = firstInfoItem instanceof ChannelInfoItem ? firstInfoItem - : secondInfoItem; - - // The channel should be the first item - assertTrue((firstInfoItem instanceof ChannelInfoItem) - || (secondInfoItem instanceof ChannelInfoItem)); - assertEquals("name", "PewDiePie", channelItem.getName()); - assertEquals("url", "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", channelItem.getUrl()); - } - - @Test - public void testResultListCheckIfContainsStreamItems() { - boolean hasStreams = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof StreamInfoItem) { - hasStreams = true; - } - } - assertTrue("Has no InfoItemStreams", hasStreams); - } - - @Test - public void testGetSecondPage() throws Exception { - YoutubeSearchExtractor secondExtractor = - (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie"); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() > 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - } - - @Test - public void testSuggestionNotNull() throws Exception { - //todo write a real test - assertNotNull(extractor.getSearchSuggestion()); - } - - - @Test - public void testId() throws Exception { - assertEquals("pewdiepie", extractor.getId()); - } - - @Test - public void testName() { - assertEquals("pewdiepie", extractor.getName()); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java new file mode 100644 index 000000000..d5f2f1af2 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -0,0 +1,190 @@ +package org.schabi.newpipe.extractor.services.youtube.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; + +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; + +public class YoutubeSearchExtractorTest { + public static class All extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } + + public static class Channel extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + } + + public static class Playlists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(PLAYLISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + public static class Videos extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class Suggestion extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "pewdeipie"; + private static final String EXPECTED_SUGGESTION = "pewdiepie"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "UCO6AK"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + /*////////////////////////////////////////////////////////////////////////// + // Test Overrides + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testMoreRelatedItems() throws Exception { + // YouTube actually gives us an empty next page, but after that, no more pages. + assertTrue(extractor.hasNextPage()); + final ListExtractor.InfoItemsPage nextEmptyPage = extractor.getPage(extractor.getNextPageUrl()); + assertEquals(0, nextEmptyPage.getItems().size()); + assertEmptyErrors("Empty page has errors", nextEmptyPage.getErrors()); + + assertFalse("More items available when it shouldn't", nextEmptyPage.hasNextPage()); + final String nextPageUrl = nextEmptyPage.getNextPageUrl(); + assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty()); + } + } + + public static class PagingTest { + @Test + public void duplicatedItemsCheck() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + final SearchExtractor extractor = YouTube.getSearchExtractor("cirque du soleil", singletonList(VIDEOS), ""); + extractor.fetchPage(); + + final ListExtractor.InfoItemsPage page1 = extractor.getInitialPage(); + final ListExtractor.InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + + assertNoDuplicatedItems(YouTube, page1, page2); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java deleted file mode 100644 index 656e197ad..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static java.util.Collections.singletonList; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -public class YoutubeSearchPagingTest { - private static ListExtractor.InfoItemsPage page1; - private static ListExtractor.InfoItemsPage page2; - private static Set urlList1; - private static Set urlList2; - private static int page1Size; - private static int page2Size; - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - - YoutubeSearchExtractor extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("cirque du soleil", - singletonList(YoutubeSearchQueryHandlerFactory.VIDEOS), null); - - extractor.fetchPage(); - page1 = extractor.getInitialPage(); - urlList1 = extractUrls(page1.getItems()); - assertTrue("failed to load search result page one: too few items", 15 < page1.getItems().size()); - page1Size = page1.getItems().size(); - assertEquals("duplicated items in search result on page one", page1Size, urlList1.size()); - - assertTrue("search result has no second page", page1.hasNextPage()); - assertNotNull("next page url is null", page1.getNextPageUrl()); - page2 = extractor.getPage(page1.getNextPageUrl()); - urlList2 = extractUrls(page2.getItems()); - page2Size = page2.getItems().size(); - } - - private static Set extractUrls(List list) { - Set result = new HashSet<>(); - for (InfoItem item : list) { - result.add(item.getUrl()); - } - return result; - } - - @Test - public void secondPageUniqueVideos() { - assertEquals("Second search result page has duplicated items", page2Size, urlList2.size()); - } - - @Test - public void noRepeatingVideosInPages() { - Set intersection = new HashSet<>(urlList2); - intersection.retainAll(urlList1); - assertEquals("Found the same item on first AND second search page", 0, intersection.size()); - } - -} \ No newline at end of file From 9b7999fe54c30d77e710c38ca64cea80ba1cbf5c Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:15:23 -0300 Subject: [PATCH 28/66] [YouTube] Check if channel item has subscription count in search --- .../extractors/YoutubeChannelInfoItemExtractor.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 29aa045b7..09e984085 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -70,8 +70,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getSubscriberCount() throws ParsingException { try { - String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText")); - return Utils.mixedNumberWordToLong(subscribers); + final JsonObject subscriberCountObject = channelInfoItem.getObject("subscriberCountText"); + + if (subscriberCountObject == null) { + // Subscription count is not available for this channel item. + return -1; + } + + return Utils.mixedNumberWordToLong(getTextFromObject(subscriberCountObject)); } catch (Exception e) { throw new ParsingException("Could not get subscriber count", e); } From b7f8001a49fd963d008a02e64f7bd8ff1239548b Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:15:51 -0300 Subject: [PATCH 29/66] [YouTube] Add check for channel items without description in search --- .../extractors/YoutubeChannelInfoItemExtractor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 09e984085..15211da7e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -95,7 +95,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getDescription() throws ParsingException { try { - return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); + final JsonObject descriptionObject = channelInfoItem.getObject("descriptionSnippet"); + + if (descriptionObject == null) { + // Channel have no description. + return null; + } + + return getTextFromObject(descriptionObject); } catch (Exception e) { throw new ParsingException("Could not get description", e); } From 921bf30bb7eedbd0c92843d3471cb2c092ef4754 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:16:33 -0300 Subject: [PATCH 30/66] [YouTube] Add check for channel items with no video count in search --- .../extractors/YoutubeChannelInfoItemExtractor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 15211da7e..27c082359 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -86,7 +86,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getStreamCount() throws ParsingException { try { - return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText")))); + final JsonObject videoCountObject = channelInfoItem.getObject("videoCountText"); + + if (videoCountObject == null) { + // Video count is not available, channel probably has no public uploads. + return -1; + } + + return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(videoCountObject))); } catch (Exception e) { throw new ParsingException("Could not get stream count", e); } From 04bfa0ec6a01228a6cb0f91177855e94a88f4981 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:23:01 -0300 Subject: [PATCH 31/66] [PeerTube] Update video used in age limit test (was 404 not found) --- .../services/peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index e103e1a81..f5202f39d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -145,7 +145,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetAgeLimit() throws ExtractionException, IOException { assertEquals(0, extractor.getAgeLimit()); - PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/3c0da7fb-e4d9-442e-84e3-a8c47004ee28"); + PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/0d501633-f2d9-4476-87c6-71f1c02402a4"); ageLimit.fetchPage(); assertEquals(18, ageLimit.getAgeLimit()); } From c921e5d6a5651186af57ac3bd645c1ee53a3d9f5 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 23 Mar 2020 18:05:02 -0300 Subject: [PATCH 32/66] Remove unnecessary collector creation function from search extractor --- .../newpipe/extractor/search/InfoItemsSearchCollector.java | 2 +- .../org/schabi/newpipe/extractor/search/SearchExtractor.java | 4 ---- .../media_ccc/extractors/MediaCCCSearchExtractor.java | 2 +- .../services/peertube/extractors/PeertubeSearchExtractor.java | 3 +-- .../services/soundcloud/SoundcloudSearchExtractor.java | 2 +- .../services/youtube/extractors/YoutubeSearchExtractor.java | 4 ++-- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java index bfa8201a1..fd87b853d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java @@ -52,7 +52,7 @@ public class InfoItemsSearchCollector extends InfoItemsCollector { public abstract String getSearchSuggestion() throws ParsingException; - protected InfoItemsSearchCollector getInfoItemSearchCollector() { - return new InfoItemsSearchCollector(getService().getServiceId()); - } - @Override public SearchQueryHandler getLinkHandler() { return (SearchQueryHandler) super.getLinkHandler(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 241d6d1e4..96618fd94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -47,7 +47,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); + final InfoItemsSearchCollector searchItems = new InfoItemsSearchCollector(getServiceId()); if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 3840bf765..61fcdd6df 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -47,8 +47,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { } private InfoItemsCollector collectStreamsFrom(JsonObject json) throws ParsingException { - - final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); JsonArray contents; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index de02af6bb..780e7e2b1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -78,7 +78,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor { } private InfoItemsCollector collectItems(JsonArray searchCollection) { - final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); for (Object result : searchCollection) { if (!(result instanceof JsonObject)) continue; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 6fec11643..27a247096 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -79,7 +79,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); @@ -103,7 +103,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } - InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") From 0a20c53f1a5bad1c35686d19fabce855e4cd747b Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 23 Mar 2020 18:08:59 -0300 Subject: [PATCH 33/66] Use easier-to-read asserts statements in tests --- .../extractor/services/DefaultExtractorTest.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java index 40891d494..f25b9cf04 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java @@ -4,8 +4,8 @@ import org.junit.Test; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.StreamingService; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; public abstract class DefaultExtractorTest implements BaseExtractorTest { @@ -39,21 +39,15 @@ public abstract class DefaultExtractorTest implements BaseE @Override public void testUrl() throws Exception { final String url = extractor().getUrl(); - final String expectedContains = expectedUrlContains(); - assertIsSecureUrl(url); - assertTrue("Url \"" + url + "\" doesn't contains \"" + expectedContains + "\"", - url.contains(expectedContains)); + assertThat(url, containsString(expectedUrlContains())); } @Test @Override public void testOriginalUrl() throws Exception { final String originalUrl = extractor().getOriginalUrl(); - final String expectedContains = expectedOriginalUrlContains(); - assertIsSecureUrl(originalUrl); - assertTrue("Original url \"" + originalUrl + "\" doesn't contains \"" + expectedContains + "\"", - originalUrl.contains(expectedContains)); + assertThat(originalUrl, containsString(expectedOriginalUrlContains())); } } From bcd2a1092bcf22b182743243ea98ceec6b8c63ea Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 2 Mar 2020 19:56:43 +0100 Subject: [PATCH 34/66] Don't accept YouTube Mix playlists --- .../youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 29bbdfdfd..aef4626a7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -54,10 +54,9 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { try { - getId(url); + return !getId(url).startsWith("RD"); // Don't accept auto-generated "Mix" playlists } catch (ParsingException e) { return false; } - return true; } } From 647e7cd45054921712d6c6e592f39f70a1289e3b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 25 Mar 2020 22:23:53 +0100 Subject: [PATCH 35/66] Accept YouTube Music playlists but not YouTube Mixes --- .../linkHandler/YoutubePlaylistLinkHandlerFactory.java | 8 +++++++- .../youtube/YoutubePlaylistLinkHandlerFactoryTest.java | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index aef4626a7..cf51281b3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -45,6 +45,11 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { throw new ParsingException("the list-ID given in the URL does not match the list pattern"); } + // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists + if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) { + throw new ParsingException("YouTube Mix playlists are not yet supported"); + } + return listID; } catch (final Exception exception) { throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); @@ -54,9 +59,10 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { try { - return !getId(url).startsWith("RD"); // Don't accept auto-generated "Mix" playlists + getId(url); } catch (ParsingException e) { return false; } + return true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 037c09040..636a646f8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -55,6 +55,8 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM")); + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=RDCLAK5uy_ly6s4irLuZAcjEDwJmqcA_UtSipMyGgbQ")); // YouTube Music playlist + assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix } @Test From bc43f8977d8859d6eb290041dce91b76685fcaef Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 28 Mar 2020 10:22:47 +0100 Subject: [PATCH 36/66] [peertube] add feed url in channels --- .../services/peertube/extractors/PeertubeChannelExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index d53fb7ad0..e5acf1ee0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -57,7 +57,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getFeedUrl() throws ParsingException { - return null; + return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id"); } @Override From 0ae53c1e506b3b57ffa81eeea09b0a471c2127d1 Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 28 Mar 2020 11:48:24 +0100 Subject: [PATCH 37/66] [peertube] adapt test for channel feed url --- .../services/peertube/PeertubeChannelExtractorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 7c7a9b805..461095598 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -98,7 +98,7 @@ public class PeertubeChannelExtractorTest { @Test public void testFeedUrl() throws ParsingException { - assertEmpty(extractor.getFeedUrl()); + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl()); } @Test @@ -195,7 +195,7 @@ public class PeertubeChannelExtractorTest { @Test public void testFeedUrl() throws ParsingException { - assertEmpty(extractor.getFeedUrl()); + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl()); } @Test From c8abc1abd520fd43fc5b63aa6e7f01e90d8fa080 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 29 Mar 2020 23:47:26 +0200 Subject: [PATCH 38/66] v0.19.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8e9045652..a62598060 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.18.7' + version 'v0.19.0' group 'com.github.TeamNewPipe' repositories { From 1762a527c9c9d6d552bbee6035d5a3f2397f3894 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 17 Mar 2020 11:33:39 +0100 Subject: [PATCH 39/66] Add support for YouTube Music search --- .../extractors/YoutubeSearchExtractor.java | 423 +++++++++++++++++- .../linkHandler/YoutubeParsingHelper.java | 18 + .../YoutubeSearchQueryHandlerFactory.java | 41 +- 3 files changed, 450 insertions(+), 32 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 27a247096..dbb95f04f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,23 +2,43 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; /* * Created by Christian Schabesberger on 22.07.2018 @@ -49,21 +69,110 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + "&pbj=1"; + if (isMusicSearch()) { + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; - initialData = ajaxJson.getObject(1).getObject("response"); + String params = null; + + switch (getLinkHandler().getContentFilters().get(0)) { + case MUSIC_SONGS: + params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_VIDEOS: + params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ALBUMS: + params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_PLAYLISTS: + params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ARTISTS: + params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; + break; + } + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", getSearchString()) + .value("params", params) + .end().done().getBytes("UTF-8"); + // @formatter:on + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + Response response = getDownloader().post(url, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + try { + initialData = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } else { + final String url = getUrl() + "&pbj=1"; + + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + + initialData = ajaxJson.getObject(1).getObject("response"); + } } @Nonnull @Override public String getUrl() throws ParsingException { + if (isMusicSearch()) return super.getUrl(); return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } @Override public String getSearchSuggestion() throws ParsingException { + if (isMusicSearch()) return ""; + JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) @@ -78,23 +187,36 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException { + public InfoItemsPage getInitialPage() throws ExtractionException, IOException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - for (Object section : sections) { - collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); + if (isMusicSearch()) { + JsonArray sections = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("musicShelfRenderer").getArray("contents"); + + collectMusicStreamsFrom(collector, sections); + } else { + JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); + + for (Object section : sections) { + collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); + } } return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override - public String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); + public String getNextPageUrl() throws ExtractionException, IOException { + if (isMusicSearch()) { + return getNextPageUrlFrom(initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("musicShelfRenderer").getArray("continuations")); + } else { + return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); + } } @Override @@ -104,19 +226,97 @@ public class YoutubeSearchExtractor extends SearchExtractor { } final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("itemSectionContinuation"); + JsonArray continuations; - collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + if (isMusicSearch()) { + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations"))); + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .end().done().getBytes("UTF-8"); + // @formatter:on + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + Response response = getDownloader().post(pageUrl, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + final JsonObject ajaxJson; + try { + ajaxJson = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + + if (ajaxJson.getObject("continuationContents") == null) return new InfoItemsPage<>(collector, null); + + JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); + + collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); + continuations = musicShelfContinuation.getArray("continuations"); + } else { + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + + JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("itemSectionContinuation"); + + collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + continuations = itemSectionRenderer.getArray("continuations"); + } + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); + } + + private boolean isMusicSearch() { + final List contentFilters = getLinkHandler().getContentFilters(); + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) return true; + return false; } private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { - collector.reset(); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { @@ -133,7 +333,180 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException { + private void collectMusicStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) { + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object item : videos) { + final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); + if (info != null) { + final String searchType = getLinkHandler().getContentFilters().get(0); + if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { + collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { + @Override + public String getUrl() throws ParsingException { + String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) return url; + throw new ParsingException("Could not get url"); + } + + @Override + public String getName() throws ParsingException { + String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get name"); + } + + @Override + public long getDuration() throws ParsingException { + String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (duration != null && !duration.isEmpty()) + return YoutubeParsingHelper.parseDurationString(duration); + throw new ParsingException("Could not get duration"); + } + + @Override + public String getUploaderName() throws ParsingException { + String name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get uploader name"); + } + + @Override + public String getTextualUploadDate() { + return null; + } + + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + if (searchType.equals(MUSIC_SONGS)) return -1; + String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); + throw new ParsingException("Could not get view count"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } else if (searchType.equals(MUSIC_ARTISTS)) { + collector.commit(new YoutubeChannelInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) return url; + throw new ParsingException("Could not get url"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); + throw new ParsingException("Could not get subscriber count"); + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public String getDescription() { + return null; + } + }); + } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { + collector.commit(new YoutubePlaylistInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) return url; + throw new ParsingException("Could not get url"); + } + + @Override + public String getUploaderName() throws ParsingException { + String name; + if (searchType.equals(MUSIC_ALBUMS)) { + name = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } else { + name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get uploader name"); + } + + @Override + public long getStreamCount() throws ParsingException { + if (searchType.equals(MUSIC_ALBUMS)) return -1; + String count = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (count != null && !count.isEmpty()) return Long.parseLong(Utils.removeNonDigitCharacters(count)); + throw new ParsingException("Could not get count"); + } + }); + } + } + } + } + + private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { if (continuations == null) { return ""; } @@ -141,7 +514,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams; + + if (isMusicSearch()) { + return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; + } else { + return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 965325550..8847466fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -62,6 +62,8 @@ public class YoutubeParsingHelper { private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; + private static String[] youtubeMusicKeys; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -259,6 +261,19 @@ public class YoutubeParsingHelper { throw new ParsingException("Could not get client version"); } + public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException { + if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys; + + final String url = "https://music.youtube.com/"; + final String html = getDownloader().get(url).responseBody(); + + final String key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); + final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html); + final String clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + + return youtubeMusicKeys = new String[]{key, clientName, clientVersion}; + } + public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { if (navigationEndpoint.getObject("urlEndpoint") != null) { String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); @@ -303,6 +318,9 @@ public class YoutubeParsingHelper { if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); return url.toString(); + } else if (navigationEndpoint.getObject("watchPlaylistEndpoint") != null) { + return "https://www.youtube.com/playlist?list=" + + navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId"); } return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 13481b345..57829eab0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -8,13 +8,21 @@ import java.net.URLEncoder; import java.util.List; public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - public static final String CHARSET_UTF_8 = "UTF-8"; + public static final String ALL = "all"; public static final String VIDEOS = "videos"; public static final String CHANNELS = "channels"; public static final String PLAYLISTS = "playlists"; - public static final String ALL = "all"; + + public static final String MUSIC_SONGS = "music_songs"; + public static final String MUSIC_VIDEOS = "music_videos"; + public static final String MUSIC_ALBUMS = "music_albums"; + public static final String MUSIC_PLAYLISTS = "music_playlists"; + public static final String MUSIC_ARTISTS = "music_artists"; + + private static final String SEARCH_URL = "https://www.youtube.com/results?search_query="; + private static final String MUSIC_SEARCH_URL = "https://music.youtube.com/search?q="; public static YoutubeSearchQueryHandlerFactory getInstance() { return new YoutubeSearchQueryHandlerFactory(); @@ -23,20 +31,27 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { try { - final String url = "https://www.youtube.com/results" - + "?search_query=" + URLEncoder.encode(searchString, CHARSET_UTF_8); - if (contentFilters.size() > 0) { switch (contentFilters.get(0)) { - case VIDEOS: return url + "&sp=EgIQAQ%253D%253D"; - case CHANNELS: return url + "&sp=EgIQAg%253D%253D"; - case PLAYLISTS: return url + "&sp=EgIQAw%253D%253D"; case ALL: default: + break; + case VIDEOS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D"; + case CHANNELS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D"; + case PLAYLISTS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D"; + case MUSIC_SONGS: + case MUSIC_VIDEOS: + case MUSIC_ALBUMS: + case MUSIC_PLAYLISTS: + case MUSIC_ARTISTS: + return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); } } - return url; + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } @@ -48,6 +63,12 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory ALL, VIDEOS, CHANNELS, - PLAYLISTS}; + PLAYLISTS, + MUSIC_SONGS, + MUSIC_VIDEOS, + MUSIC_ALBUMS, + MUSIC_PLAYLISTS, + MUSIC_ARTISTS + }; } } From 4ddbdf0aee23b67fb5dd2e806c174d960dbd7ffc Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 17 Mar 2020 13:06:25 +0100 Subject: [PATCH 40/66] Disable artist search for now --- .../youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 57829eab0..7acf7d714 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -68,7 +68,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory MUSIC_VIDEOS, MUSIC_ALBUMS, MUSIC_PLAYLISTS, - MUSIC_ARTISTS +// MUSIC_ARTISTS }; } } From fb9b9691b76ddabd4cc3d5c02231fce28e6b8f17 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 11:05:19 +0100 Subject: [PATCH 41/66] Improve getYoutubeMusicKeys() --- .../linkHandler/YoutubeParsingHelper.java | 83 ++++++++++++++++--- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 8847466fc..7fd814b9d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -5,6 +5,8 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; @@ -62,6 +64,7 @@ public class YoutubeParsingHelper { private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; + private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"}; private static String[] youtubeMusicKeys; private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; @@ -198,11 +201,7 @@ public class YoutubeParsingHelper { */ public static String getClientVersion() throws IOException, ExtractionException { if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - - if (isHardcodedClientVersionValid()) { - clientVersion = HARDCODED_CLIENT_VERSION; - return clientVersion; - } + if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION; final String url = "https://www.youtube.com/results?search_query=test"; final String html = getDownloader().get(url).responseBody(); @@ -219,8 +218,7 @@ public class YoutubeParsingHelper { JsonObject p = (JsonObject) param; String key = p.getString("key"); if (key != null && key.equals("cver")) { - clientVersion = p.getString("value"); - return clientVersion; + return clientVersion = p.getString("value"); } } } else if (s.getString("service").equals("ECATCHER")) { @@ -246,30 +244,89 @@ public class YoutubeParsingHelper { try { contextClientVersion = Parser.matchGroup1(pattern, html); if (contextClientVersion != null && !contextClientVersion.isEmpty()) { - clientVersion = contextClientVersion; - return clientVersion; + return clientVersion = contextClientVersion; } } catch (Exception ignored) { } } if (shortClientVersion != null) { - clientVersion = shortClientVersion; - return clientVersion; + return clientVersion = shortClientVersion; } throw new ParsingException("Could not get client version"); } + public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException { + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0]; + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEYS[2]) + .value("hl", "en") + .value("gl", "GB") + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", "test") + .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") + .end().done().getBytes("UTF-8"); + // @formatter:on + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + String response = getDownloader().post(url, headers, json).responseBody(); + + return response.length() > 50; // ensure to have a valid response + } + public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException { if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys; + if (areHardcodedYoutubeMusicKeysValid()) return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS; final String url = "https://music.youtube.com/"; final String html = getDownloader().get(url).responseBody(); - final String key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); + String key; + try { + key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); + } catch (Parser.RegexException e) { + key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html); + } + final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html); - final String clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + + String clientVersion; + try { + clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + } catch (Parser.RegexException e) { + try { + clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + } catch (Parser.RegexException ee) { + clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html); + } + } return youtubeMusicKeys = new String[]{key, clientName, clientVersion}; } From 2b9b2a78e8128f1730456c83f3168a60580de4e9 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 11:17:12 +0100 Subject: [PATCH 42/66] Handle 100+ items in playlist --- .../extractor/playlist/PlaylistExtractor.java | 3 +++ .../youtube/extractors/YoutubeSearchExtractor.java | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 2a901dbf0..06330de05 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -7,6 +7,9 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; public abstract class PlaylistExtractor extends ListExtractor { + public final static long UNKNOWN_ITEMS = -1; + public final static long INFINITE_ITEMS = -2; + public final static long MORE_THAN_100_ITEMS = -3; public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index dbb95f04f..0108e3303 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -30,6 +30,8 @@ import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.playlist.PlaylistExtractor.MORE_THAN_100_ITEMS; +import static org.schabi.newpipe.extractor.playlist.PlaylistExtractor.UNKNOWN_ITEMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; @@ -494,10 +496,16 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) return -1; + if (searchType.equals(MUSIC_ALBUMS)) return UNKNOWN_ITEMS; String count = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (count != null && !count.isEmpty()) return Long.parseLong(Utils.removeNonDigitCharacters(count)); + if (count != null && !count.isEmpty()) { + if (count.contains("100+")) { + return MORE_THAN_100_ITEMS; + } else { + return Long.parseLong(Utils.removeNonDigitCharacters(count)); + } + } throw new ParsingException("Could not get count"); } }); From eb485244119c4338d32dc1468ae9dcf98b81058a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 12:23:58 +0100 Subject: [PATCH 43/66] Add tests for YouTube Music search --- .../youtube/YoutubeParsingHelperTest.java | 6 ++ .../YoutubeSearchExtractorMusicTest.java | 79 +++++++++++++++++++ .../youtube/search/YoutubeSearchQHTest.java | 18 ++++- 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 87dbbd750..669bc3292 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -22,4 +22,10 @@ public class YoutubeParsingHelperTest { assertTrue("Hardcoded client version is not valid anymore", YoutubeParsingHelper.isHardcodedClientVersionValid()); } + + @Test + public void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException { + assertTrue("Hardcoded YouTube Music keys are not valid anymore", + YoutubeParsingHelper.areHardcodedYoutubeMusicKeysValid()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java new file mode 100644 index 000000000..aaf6105c4 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java @@ -0,0 +1,79 @@ +package org.schabi.newpipe.extractor.services.youtube.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; + +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseTest { + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", + asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testGetSecondPage() throws Exception { + YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", + asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); + ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); + assertTrue(Integer.toString(secondPage.getItems().size()), + secondPage.getItems().size() > 10); + + // check if its the same result + boolean equals = true; + for (int i = 0; i < secondPage.getItems().size() + && i < itemsPage.getItems().size(); i++) { + if (!secondPage.getItems().get(i).getUrl().equals( + itemsPage.getItems().get(i).getUrl())) { + equals = false; + } + } + assertFalse("First and second page are equal", equals); + } + + @Override + @Test + public void testUrl() throws Exception { + assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://music.youtube.com/search?q=")); + } + + @Test + public void testGetSecondPageUrl() throws Exception { + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "music.youtube.com"); + assertEquals(url.getPath(), "/youtubei/v1/search"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); + assertEquals("json", queryPairs.get("alt")); + assertTrue(queryPairs.get("key").length() > 5); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index fc6af4c4d..b558ad88d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -16,6 +16,12 @@ public class YoutubeSearchQHTest { assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + + assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl()); } @Test @@ -24,6 +30,9 @@ public class YoutubeSearchQHTest { .fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0)); assertEquals(CHANNELS, YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0)); + + assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0)); } @Test @@ -36,16 +45,23 @@ public class YoutubeSearchQHTest { .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl()); assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); + + assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); } @Test public void testGetAvailableContentFilter() { final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableContentFilter(); - assertEquals(4, contentFilter.length); + assertEquals(8, contentFilter.length); assertEquals("all", contentFilter[0]); assertEquals("videos", contentFilter[1]); assertEquals("channels", contentFilter[2]); assertEquals("playlists", contentFilter[3]); + assertEquals("music_songs", contentFilter[4]); + assertEquals("music_videos", contentFilter[5]); + assertEquals("music_albums", contentFilter[6]); + assertEquals("music_playlists", contentFilter[7]); } @Test From c852b13d5a395f696b6d5e93a07dcc3dd26b5497 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 14:14:02 +0100 Subject: [PATCH 44/66] Add Referer header so that it also works with HttpsUrlConnection --- .../services/youtube/extractors/YoutubeSearchExtractor.java | 2 ++ .../services/youtube/linkHandler/YoutubeParsingHelper.java | 1 + 2 files changed, 3 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 0108e3303..85882e1f4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -130,6 +130,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); Response response = getDownloader().post(url, headers, json); @@ -266,6 +267,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); Response response = getDownloader().post(pageUrl, headers, json); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 7fd814b9d..4ea6e0967 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -294,6 +294,7 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1])); headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2])); headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); String response = getDownloader().post(url, headers, json).responseBody(); From dc29d87962f0f9acd97d7d0bcad6afcc67b127fd Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 16:11:28 +0100 Subject: [PATCH 45/66] Extract YouTube search suggestions --- .../extractors/YoutubeSearchExtractor.java | 45 ++++++++++++------- .../YoutubeSearchExtractorMusicTest.java | 10 +++++ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 85882e1f4..6dd574873 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -174,16 +174,19 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { - if (isMusicSearch()) return ""; - - JsonObject showingResultsForRenderer = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("showingResultsForRenderer"); - if (showingResultsForRenderer == null) { - return ""; + if (isMusicSearch()) { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) return ""; + return getTextFromObject(itemSectionRenderer.getArray("contents").getObject(0) + .getObject("didYouMeanRenderer").getObject("correctedQuery")); } else { + JsonObject showingResultsForRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + if (showingResultsForRenderer == null) return ""; return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } } @@ -194,10 +197,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); if (isMusicSearch()) { - JsonArray sections = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("musicShelfRenderer").getArray("contents"); + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); - collectMusicStreamsFrom(collector, sections); + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); + } + } } else { JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); @@ -213,8 +219,15 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getNextPageUrl() throws ExtractionException, IOException { if (isMusicSearch()) { - return getNextPageUrlFrom(initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("musicShelfRenderer").getArray("continuations")); + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); + } + } + + return ""; } else { return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") @@ -517,9 +530,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { - if (continuations == null) { - return ""; - } + if (continuations == null) return ""; JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java index aaf6105c4..61c8e04b7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java @@ -76,4 +76,14 @@ public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseT assertEquals("json", queryPairs.get("alt")); assertTrue(queryPairs.get("key").length() > 5); } + + @Test + public void testSuggestions() throws Exception { + YoutubeSearchExtractor newExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("megaman x3", + asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); + newExtractor.fetchPage(); + + assertTrue(newExtractor.getInitialPage().getItems().size() > 10); + assertEquals("mega man x3", newExtractor.getSearchSuggestion()); + } } From d58c0f230d27cf34c2bf044597c908c519f94a84 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 19:14:53 +0100 Subject: [PATCH 46/66] Improve code for YouTube Music search --- .../extractors/YoutubeSearchExtractor.java | 44 ++++++++++++++----- .../YoutubeSearchExtractorMusicTest.java | 6 +++ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 6dd574873..7c5adca5d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -308,7 +308,9 @@ public class YoutubeSearchExtractor extends SearchExtractor { throw new ParsingException("Could not parse JSON", e); } - if (ajaxJson.getObject("continuationContents") == null) return new InfoItemsPage<>(collector, null); + if (ajaxJson.getObject("continuationContents") == null) { + return InfoItemsPage.emptyPage(); + } JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); @@ -329,8 +331,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { private boolean isMusicSearch() { final List contentFilters = getLinkHandler().getContentFilters(); - if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) return true; - return false; + return contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_"); } private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { @@ -391,6 +392,16 @@ public class YoutubeSearchExtractor extends SearchExtractor { throw new ParsingException("Could not get uploader name"); } + @Override + public String getUploaderUrl() throws ParsingException { + if (searchType.equals(MUSIC_VIDEOS)) return null; + String url = getUrlFromNavigationEndpoint(info.getArray("flexColumns") + .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) return url; + throw new ParsingException("Could not get uploader url"); + } + @Override public String getTextualUploadDate() { return null; @@ -413,9 +424,12 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -426,9 +440,12 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -472,9 +489,12 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java index 61c8e04b7..6b2f472bd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseTest { @BeforeClass @@ -30,6 +31,11 @@ public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseT itemsPage = extractor.getInitialPage(); } + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + @Test public void testGetSecondPage() throws Exception { YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", From 5a775a4bbe6fc1e06a29a3665c8e8c23234a0536 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 21 Mar 2020 19:56:17 +0100 Subject: [PATCH 47/66] Use new way of specifying stream count --- .../newpipe/extractor/ListExtractor.java | 19 ++++++++++++++++++- .../extractor/playlist/PlaylistExtractor.java | 3 --- .../extractors/YoutubeSearchExtractor.java | 6 ++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index b254adbd8..0e21f04b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -3,16 +3,33 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + /** * Base class to extractors that have a list (e.g. playlists, users). */ public abstract class ListExtractor extends Extractor { + /** + * Constant that should be returned whenever + * a list has an unknown number of items. + */ + public static final long ITEM_COUNT_UNKNOWN = -1; + /** + * Constant that should be returned whenever a list has an + * infinite number of items. For example a YouTube mix. + */ + public static final long ITEM_COUNT_INFINITE = -2; + /** + * Constant that should be returned whenever a list + * has an unknown number of items bigger than 100. + */ + public static final long ITEM_COUNT_MORE_THAN_100 = -3; + public ListExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 06330de05..2a901dbf0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -7,9 +7,6 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; public abstract class PlaylistExtractor extends ListExtractor { - public final static long UNKNOWN_ITEMS = -1; - public final static long INFINITE_ITEMS = -2; - public final static long MORE_THAN_100_ITEMS = -3; public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 7c5adca5d..37c9515d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -30,8 +30,6 @@ import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.playlist.PlaylistExtractor.MORE_THAN_100_ITEMS; -import static org.schabi.newpipe.extractor.playlist.PlaylistExtractor.UNKNOWN_ITEMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; @@ -531,12 +529,12 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) return UNKNOWN_ITEMS; + if (searchType.equals(MUSIC_ALBUMS)) return ITEM_COUNT_UNKNOWN; String count = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); if (count != null && !count.isEmpty()) { if (count.contains("100+")) { - return MORE_THAN_100_ITEMS; + return ITEM_COUNT_MORE_THAN_100; } else { return Long.parseLong(Utils.removeNonDigitCharacters(count)); } From dd434cca01d401ffcb236a2e2f3f042fabe0e272 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 21 Mar 2020 20:11:06 +0100 Subject: [PATCH 48/66] Fix issue when there is no didYouMeanRenderer in itemSectionRenderer --- .../services/youtube/extractors/YoutubeSearchExtractor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 37c9515d0..7e640a1b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -176,8 +176,10 @@ public class YoutubeSearchExtractor extends SearchExtractor { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); if (itemSectionRenderer == null) return ""; - return getTextFromObject(itemSectionRenderer.getArray("contents").getObject(0) - .getObject("didYouMeanRenderer").getObject("correctedQuery")); + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + if (didYouMeanRenderer == null) return ""; + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } else { JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") From aa8cea47f3d6276ab592d18ff550ad9d6e63d81b Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 25 Mar 2020 10:22:32 +0100 Subject: [PATCH 49/66] Refactor YouTube Music search tests --- .../extractors/YoutubeSearchExtractor.java | 6 +- .../extractor/services/DefaultTests.java | 9 +- .../YoutubeSearchExtractorMusicTest.java | 120 +++++++----------- 3 files changed, 54 insertions(+), 81 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 7e640a1b8..52df8c449 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -395,9 +395,11 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getUploaderUrl() throws ParsingException { if (searchType.equals(MUSIC_VIDEOS)) return null; - String url = getUrlFromNavigationEndpoint(info.getArray("flexColumns") + JsonObject navigationEndpoint = info.getArray("flexColumns") .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint")); + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) return null; + String url = getUrlFromNavigationEndpoint(navigationEndpoint); if (url != null && !url.isEmpty()) return url; throw new ParsingException("Could not get uploader url"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index b34bbbff9..a52698415 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -37,11 +37,14 @@ public final class DefaultTests { if (item instanceof StreamInfoItem) { StreamInfoItem streamInfoItem = (StreamInfoItem) item; assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName()); - assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); - assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + +// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); + if (streamInfoItem.getUploaderUrl() != null && !streamInfoItem.getUploaderUrl().isEmpty()) { + assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); + } assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM); - assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); final String textualUploadDate = streamInfoItem.getTextualUploadDate(); if (textualUploadDate != null && !textualUploadDate.isEmpty()) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java index 6b2f472bd..fe1743803 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java @@ -1,95 +1,63 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.BeforeClass; -import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import java.net.URL; -import java.net.URLDecoder; -import java.util.LinkedHashMap; -import java.util.Map; +import java.net.URLEncoder; -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; -public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseTest { - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", - asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } +public class YoutubeSearchExtractorMusicTest { + public static class MusicSongs extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "mocromaniac"; - @Test - public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor); - } - - @Test - public void testGetSecondPage() throws Exception { - YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", - asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() > 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - } - - @Override - @Test - public void testUrl() throws Exception { - assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://music.youtube.com/search?q=")); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - URL url = new URL(extractor.getNextPageUrl()); - - assertEquals(url.getHost(), "music.youtube.com"); - assertEquals(url.getPath(), "/youtubei/v1/search"); - - Map queryPairs = new LinkedHashMap<>(); - for (String queryPair : url.getQuery().split("&")) { - int index = queryPair.indexOf("="); - queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), - URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); } - assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); - assertTrue(queryPairs.get("continuation").length() > 5); - assertTrue(queryPairs.get("itct").length() > 5); - assertEquals("json", queryPairs.get("alt")); - assertTrue(queryPairs.get("key").length() > 5); + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } - @Test - public void testSuggestions() throws Exception { - YoutubeSearchExtractor newExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("megaman x3", - asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); - newExtractor.fetchPage(); + public static class Suggestion extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "megaman x3"; - assertTrue(newExtractor.getInitialPage().getItems().size() > 10); - assertEquals("mega man x3", newExtractor.getSearchSuggestion()); + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } } From cf0f2aff3e8d7dc539ff662a006329342dd6d55a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 25 Mar 2020 10:42:34 +0100 Subject: [PATCH 50/66] Extract uploader url from certain YouTube Music videos --- .../extractors/YoutubeSearchExtractor.java | 25 ++++++++++++++----- ...a => YoutubeMusicSearchExtractorTest.java} | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/{YoutubeSearchExtractorMusicTest.java => YoutubeMusicSearchExtractorTest.java} (98%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 52df8c449..ca56283d9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -394,12 +394,25 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getUploaderUrl() throws ParsingException { - if (searchType.equals(MUSIC_VIDEOS)) return null; - JsonObject navigationEndpoint = info.getArray("flexColumns") - .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); - if (navigationEndpoint == null) return null; - String url = getUrlFromNavigationEndpoint(navigationEndpoint); + String url = null; + + if (searchType.equals(MUSIC_VIDEOS)) { + JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); + for (Object item : items) { + JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { + url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); + break; + } + } + } else { + JsonObject navigationEndpoint = info.getArray("flexColumns") + .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) return null; + url = getUrlFromNavigationEndpoint(navigationEndpoint); + } + if (url != null && !url.isEmpty()) return url; throw new ParsingException("Could not get uploader url"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java similarity index 98% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index fe1743803..689e6127c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -16,7 +16,7 @@ import javax.annotation.Nullable; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -public class YoutubeSearchExtractorMusicTest { +public class YoutubeMusicSearchExtractorTest { public static class MusicSongs extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "mocromaniac"; From ac15df45489178002c95db436a717319020a517a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 10:03:54 +0200 Subject: [PATCH 51/66] Move YouTube Music search to its own class --- .../services/youtube/YoutubeService.java | 41 +- .../YoutubeMusicSearchExtractor.java | 526 ++++++++++++++++++ .../extractors/YoutubeSearchExtractor.java | 508 ++--------------- 3 files changed, 598 insertions(+), 477 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index eae3bcb9b..519672141 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -7,22 +7,45 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.*; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -import javax.annotation.Nonnull; import java.util.List; +import javax.annotation.Nonnull; + import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; /* * Created by Christian Schabesberger on 23.08.15. @@ -92,7 +115,13 @@ public class YoutubeService extends StreamingService { @Override public SearchExtractor getSearchExtractor(SearchQueryHandler query) { - return new YoutubeSearchExtractor(this, query); + final List contentFilters = query.getContentFilters(); + + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) { + return new YoutubeMusicSearchExtractor(this, query); + } else { + return new YoutubeSearchExtractor(this, query); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java new file mode 100644 index 000000000..46e9d93c0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -0,0 +1,526 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; + +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; + +public class YoutubeMusicSearchExtractor extends SearchExtractor { + private JsonObject initialData; + + public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; + + final String params; + + switch (getLinkHandler().getContentFilters().get(0)) { + case MUSIC_SONGS: + params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_VIDEOS: + params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ALBUMS: + params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_PLAYLISTS: + params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ARTISTS: + params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; + break; + default: + params = null; + break; + } + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", getSearchString()) + .value("params", params) + .end().done().getBytes("UTF-8"); + // @formatter:on + + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + final Response response = getDownloader().post(url, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + try { + initialData = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return super.getUrl(); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) { + return ""; + } + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + if (didYouMeanRenderer == null) { + return ""; + } + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); + } + } + + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws ExtractionException, IOException { + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); + } + } + + return ""; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } + + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .end().done().getBytes("UTF-8"); + // @formatter:on + + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + final Response response = getDownloader().post(pageUrl, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + final JsonObject ajaxJson; + try { + ajaxJson = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + + if (ajaxJson.getObject("continuationContents") == null) { + return InfoItemsPage.emptyPage(); + } + + final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); + + collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); + final JsonArray continuations = musicShelfContinuation.getArray("continuations"); + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); + } + + private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) { + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object item : videos) { + final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); + if (info != null) { + final String searchType = getLinkHandler().getContentFilters().get(0); + if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { + collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public long getDuration() throws ParsingException { + final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (duration != null && !duration.isEmpty()) { + return YoutubeParsingHelper.parseDurationString(duration); + } + throw new ParsingException("Could not get duration"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String url = null; + + if (searchType.equals(MUSIC_VIDEOS)) { + JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); + for (Object item : items) { + final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { + url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); + break; + } + } + } else { + final JsonObject navigationEndpoint = info.getArray("flexColumns") + .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) { + return null; + } + url = getUrlFromNavigationEndpoint(navigationEndpoint); + } + + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get uploader url"); + } + + @Override + public String getTextualUploadDate() { + return null; + } + + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + if (searchType.equals(MUSIC_SONGS)) { + return -1; + } + final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) { + return Utils.mixedNumberWordToLong(viewCount); + } + throw new ParsingException("Could not get view count"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } else if (searchType.equals(MUSIC_ARTISTS)) { + collector.commit(new YoutubeChannelInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) { + return Utils.mixedNumberWordToLong(viewCount); + } + throw new ParsingException("Could not get subscriber count"); + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public String getDescription() { + return null; + } + }); + } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { + collector.commit(new YoutubePlaylistInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name; + if (searchType.equals(MUSIC_ALBUMS)) { + name = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } else { + name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public long getStreamCount() throws ParsingException { + if (searchType.equals(MUSIC_ALBUMS)) { + return ITEM_COUNT_UNKNOWN; + } + final String count = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (count != null && !count.isEmpty()) { + if (count.contains("100+")) { + return ITEM_COUNT_MORE_THAN_100; + } else { + return Long.parseLong(Utils.removeNonDigitCharacters(count)); + } + } + throw new ParsingException("Could not get count"); + } + }); + } + } + } + } + + private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { + if (continuations == null) { + return ""; + } + + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + + return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index ca56283d9..06d83b243 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,43 +2,23 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; /* * Created by Christian Schabesberger on 22.07.2018 @@ -63,278 +43,79 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeS public class YoutubeSearchExtractor extends SearchExtractor { private JsonObject initialData; - public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - if (isMusicSearch()) { - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String url = getUrl() + "&pbj=1"; - final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - String params = null; - - switch (getLinkHandler().getContentFilters().get(0)) { - case MUSIC_SONGS: - params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_VIDEOS: - params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_ALBUMS: - params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_PLAYLISTS: - params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_ARTISTS: - params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; - break; - } - - // @formatter:off - byte[] json = JsonWriter.string() - .object() - .object("context") - .object("client") - .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") - .value("gl", getExtractorContentCountry().getCountryCode()) - .array("experimentIds").end() - .value("experimentsToken", "") - .value("utcOffsetMinutes", 0) - .object("locationInfo").end() - .object("musicAppInfo").end() - .end() - .object("capabilities").end() - .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() - .end() - .object("activePlayers").end() - .object("user") - .value("enableSafetyMode", false) - .end() - .end() - .value("query", getSearchString()) - .value("params", params) - .end().done().getBytes("UTF-8"); - // @formatter:on - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - Response response = getDownloader().post(url, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } - - try { - initialData = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } - } else { - final String url = getUrl() + "&pbj=1"; - - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - - initialData = ajaxJson.getObject(1).getObject("response"); - } + initialData = ajaxJson.getObject(1).getObject("response"); } @Nonnull @Override public String getUrl() throws ParsingException { - if (isMusicSearch()) return super.getUrl(); return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } @Override public String getSearchSuggestion() throws ParsingException { - if (isMusicSearch()) { - final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer == null) return ""; - final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") - .getObject(0).getObject("didYouMeanRenderer"); - if (didYouMeanRenderer == null) return ""; - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); - } else { - JsonObject showingResultsForRenderer = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("showingResultsForRenderer"); - if (showingResultsForRenderer == null) return ""; - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + final JsonObject showingResultsForRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + if (showingResultsForRenderer == null) { + return ""; } + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + public InfoItemsPage getInitialPage() throws ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - if (isMusicSearch()) { - final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - for (Object content : contents) { - if (((JsonObject) content).getObject("musicShelfRenderer") != null) { - collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); - } - } - } else { - JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - - for (Object section : sections) { - collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); - } + for (Object section : sections) { + collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); } return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override - public String getNextPageUrl() throws ExtractionException, IOException { - if (isMusicSearch()) { - final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); - - for (Object content : contents) { - if (((JsonObject) content).getObject("musicShelfRenderer") != null) { - return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); - } - } - - return ""; - } else { - return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); - } + public String getNextPageUrl() throws ExtractionException { + return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonArray continuations; + final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("itemSectionContinuation"); - if (isMusicSearch()) { - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); - - // @formatter:off - byte[] json = JsonWriter.string() - .object() - .object("context") - .object("client") - .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") - .value("gl", getExtractorContentCountry().getCountryCode()) - .array("experimentIds").end() - .value("experimentsToken", "") - .value("utcOffsetMinutes", 0) - .object("locationInfo").end() - .object("musicAppInfo").end() - .end() - .object("capabilities").end() - .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() - .end() - .object("activePlayers").end() - .object("user") - .value("enableSafetyMode", false) - .end() - .end() - .end().done().getBytes("UTF-8"); - // @formatter:on - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - Response response = getDownloader().post(pageUrl, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } - - final JsonObject ajaxJson; - try { - ajaxJson = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } - - if (ajaxJson.getObject("continuationContents") == null) { - return InfoItemsPage.emptyPage(); - } - - JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); - - collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); - continuations = musicShelfContinuation.getArray("continuations"); - } else { - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - - JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("itemSectionContinuation"); - - collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); - continuations = itemSectionRenderer.getArray("continuations"); - } + collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + final JsonArray continuations = itemSectionRenderer.getArray("continuations"); return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); } - private boolean isMusicSearch() { - final List contentFilters = getLinkHandler().getContentFilters(); - return contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_"); - } - - private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { + private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { @@ -351,232 +132,17 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private void collectMusicStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) { - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Object item : videos) { - final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); - if (info != null) { - final String searchType = getLinkHandler().getContentFilters().get(0); - if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { - collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public long getDuration() throws ParsingException { - String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (duration != null && !duration.isEmpty()) - return YoutubeParsingHelper.parseDurationString(duration); - throw new ParsingException("Could not get duration"); - } - - @Override - public String getUploaderName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get uploader name"); - } - - @Override - public String getUploaderUrl() throws ParsingException { - String url = null; - - if (searchType.equals(MUSIC_VIDEOS)) { - JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); - for (Object item : items) { - JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { - url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); - break; - } - } - } else { - JsonObject navigationEndpoint = info.getArray("flexColumns") - .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); - if (navigationEndpoint == null) return null; - url = getUrlFromNavigationEndpoint(navigationEndpoint); - } - - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get uploader url"); - } - - @Override - public String getTextualUploadDate() { - return null; - } - - @Override - public DateWrapper getUploadDate() { - return null; - } - - @Override - public long getViewCount() throws ParsingException { - if (searchType.equals(MUSIC_SONGS)) return -1; - String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); - throw new ParsingException("Could not get view count"); - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); - } else if (searchType.equals(MUSIC_ARTISTS)) { - collector.commit(new YoutubeChannelInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public long getSubscriberCount() throws ParsingException { - String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); - throw new ParsingException("Could not get subscriber count"); - } - - @Override - public long getStreamCount() { - return -1; - } - - @Override - public String getDescription() { - return null; - } - }); - } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { - collector.commit(new YoutubePlaylistInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public String getUploaderName() throws ParsingException { - String name; - if (searchType.equals(MUSIC_ALBUMS)) { - name = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - } else { - name = getTextFromObject(info.getArray("flexColumns").getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - } - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get uploader name"); - } - - @Override - public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) return ITEM_COUNT_UNKNOWN; - String count = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (count != null && !count.isEmpty()) { - if (count.contains("100+")) { - return ITEM_COUNT_MORE_THAN_100; - } else { - return Long.parseLong(Utils.removeNonDigitCharacters(count)); - } - } - throw new ParsingException("Could not get count"); - } - }); - } - } + private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { + if (continuations == null) { + return ""; } - } - private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { - if (continuations == null) return ""; + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); - String continuation = nextContinuationData.getString("continuation"); - String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - - if (isMusicSearch()) { - return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; - } else { - return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams; - } + return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; } } From c7f7bd244278d52779b392500d148ccda9c1508c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 10:17:34 +0200 Subject: [PATCH 52/66] Fix error when YT Music videos has no uploader URL --- .../YoutubeMusicSearchExtractor.java | 19 ++++++++++--------- .../extractors/YoutubeSearchExtractor.java | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 46e9d93c0..da4dc2b52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -319,31 +319,32 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUploaderUrl() throws ParsingException { - String url = null; - if (searchType.equals(MUSIC_VIDEOS)) { JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); for (Object item : items) { final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { - url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); - break; + return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); } } + + return null; } else { final JsonObject navigationEndpoint = info.getArray("flexColumns") .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) { return null; } - url = getUrlFromNavigationEndpoint(navigationEndpoint); - } - if (url != null && !url.isEmpty()) { - return url; + final String url = getUrlFromNavigationEndpoint(navigationEndpoint); + + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get uploader url"); } - throw new ParsingException("Could not get uploader url"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 06d83b243..70c47f6b1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -132,7 +132,6 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { if (continuations == null) { return ""; From 2af610e3e7942b1d28e304fa53cb4d263c8d6403 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 10:28:55 +0200 Subject: [PATCH 53/66] Add tests for other YT Music search types --- .../YoutubeSearchQueryHandlerFactory.java | 2 +- .../YoutubeMusicSearchExtractorTest.java | 90 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 7acf7d714..866c18569 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -67,7 +67,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory MUSIC_SONGS, MUSIC_VIDEOS, MUSIC_ALBUMS, - MUSIC_PLAYLISTS, + MUSIC_PLAYLISTS // MUSIC_ARTISTS }; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 689e6127c..420db0adb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.BeforeClass; +import org.junit.Ignore; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; @@ -39,6 +40,95 @@ public class YoutubeMusicSearchExtractorTest { @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + public static class MusicVideos extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "fresku"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class MusicAlbums extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "johnny sellah"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + public static class MusicPlaylists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "louivos"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + @Ignore + public static class MusicArtists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "kevin"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + } + public static class Suggestion extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "megaman x3"; From 2d0c301440b6e8a8c1e1ca75321e1318b2fe07de Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 15:43:11 +0200 Subject: [PATCH 54/66] Add another way to test changes locally to README.md --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 66515055d..5d62b7a97 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,21 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `compile 'com.github.TeamNewPipe:NewPipeExtractor:v0.11.0'`the `dependencies` in your `build.gradle`. Replace `v0.11.0` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.19.0'`the `dependencies` in your `build.gradle`. Replace `v0.19.0` with the latest release. ### Testing changes -To test changes quickly you can build the library locally. Using the local Maven repository is a good approach, here's a gist of how to use it: +To test changes quickly you can build the library locally. A good approach would be to add something like the following to your `settings.gradle`: + +```groovy +includeBuild('../NewPipeExtractor') { + dependencySubstitution { + substitute module('com.github.TeamNewPipe:NewPipeExtractor') with project(':extractor') + } +} +``` + +Another approach would be to use the local Maven repository, here's a gist of how to use it: 1. Add `mavenLocal()` in your project `repositories` list (usually as the first entry to give priority above the others). 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`). From 8a9e137385c5e8dac62219ff40088ef5fac6ef95 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 16:01:21 +0200 Subject: [PATCH 55/66] Extract some code to getValidResponseBody() --- .../YoutubeMusicSearchExtractor.java | 39 ++----------------- .../linkHandler/YoutubeParsingHelper.java | 25 ++++++++---- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index da4dc2b52..e4251ffc7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -9,8 +9,6 @@ import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -31,6 +29,7 @@ import java.util.Map; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidResponseBody; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; @@ -112,23 +111,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final Response response = getDownloader().post(url, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } + final String responseBody = getValidResponseBody(getDownloader().post(url, headers, json)); try { initialData = JsonParser.object().from(responseBody); @@ -232,23 +215,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final Response response = getDownloader().post(pageUrl, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } + final String responseBody = getValidResponseBody(getDownloader().post(pageUrl, headers, json)); final JsonObject ajaxJson; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 4ea6e0967..c639261bc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.text.ParseException; @@ -427,12 +428,8 @@ public class YoutubeParsingHelper { return thumbnailUrl; } - public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); - final Response response = getDownloader().get(url, headers, localization); - + public static String getValidResponseBody(final Response response) + throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { throw new ContentNotAvailableException("Not found" + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); @@ -453,11 +450,24 @@ public class YoutubeParsingHelper { } final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + if (responseContentType != null + && responseContentType.toLowerCase().contains("text/html")) { throw new ParsingException("Got HTML document, expected JSON response" + " (latest url was: \"" + response.latestUrl() + "\")"); } + return responseBody; + } + + public static JsonArray getJsonResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + final Response response = getDownloader().get(url, headers, localization); + + final String responseBody = getValidResponseBody(response); + try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { @@ -469,6 +479,7 @@ public class YoutubeParsingHelper { * Shared alert detection function, multiple endpoints return the error similarly structured. *

* Will check if the object has an alert of the type "ERROR". + *

* * @param initialData the object which will be checked if an alert is present * @throws ContentNotAvailableException if an alert is detected From 1eb3deb7fd89d60f6e36e016c0c30291a485fb21 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 29 Mar 2020 11:13:39 +0200 Subject: [PATCH 56/66] add supported countries for soundcloud --- .../services/soundcloud/SoundcloudChartsExtractor.java | 10 +++++----- .../services/soundcloud/SoundcloudService.java | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index a08502d70..9a877faba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -11,6 +11,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + public class SoundcloudChartsExtractor extends KioskExtractor { private StreamInfoItemsCollector collector = null; private String nextPageUrl = null; @@ -57,11 +59,9 @@ public class SoundcloudChartsExtractor extends KioskExtractor { apiUrl += "&kind=trending"; } - /*List supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US"); - String contentCountry = getContentCountry(); - if (supportedCountries.contains(contentCountry)) { - apiUrl += "®ion=soundcloud:regions:" + contentCountry; - }*/ + + String contentCountry = SoundCloud.getContentCountry().getCountryCode(); + apiUrl += "®ion=soundcloud:regions:" + contentCountry; nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 30f2fc87c..1c2998791 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -7,11 +7,14 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import java.util.List; + import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; @@ -46,6 +49,13 @@ public class SoundcloudService extends StreamingService { return SoundcloudPlaylistLinkHandlerFactory.getInstance(); } + @Override + public List getSupportedCountries() { + //Country selector here https://soundcloud.com/charts/top?genre=all-music + return ContentCountry.listFrom( + "AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US" + ); + } @Override public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) { From 4a1d8c804c65d0df96084874aff94ee2b5cfd95a Mon Sep 17 00:00:00 2001 From: opusforlife2 <53176348+opusforlife2@users.noreply.github.com> Date: Thu, 2 Apr 2020 10:42:54 +0000 Subject: [PATCH 57/66] Corrected the pull request template It was a bit weird before. --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4bf58ad9c..76ca39a5e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,3 @@ - [ ] I carefully read the [contribution guidelines](https://github.com/TeamNewPipe/NewPipe/blob/HEAD/.github/CONTRIBUTING.md) and agree to them. -- [ ] I did test the API against [NewPipe](https://github.com/TeamNewPipe/NewPipe). -- [ ] I agree to ASAP create a PULL request for [NewPipe](https://github.com/TeamNewPipe/NewPipe) for making in compatible when I changed the api. +- [ ] I have tested the API against [NewPipe](https://github.com/TeamNewPipe/NewPipe). +- [ ] I agree to create a pull request for [NewPipe](https://github.com/TeamNewPipe/NewPipe) as soon as possible to make it compatible with the changed API. From bce27a0e22ef124fc4f426c7b3512c40d79f43bb Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 3 Apr 2020 17:23:18 +0200 Subject: [PATCH 58/66] Rename getValidResponseBody() to getValidJsonResponseBody() --- .../youtube/extractors/YoutubeMusicSearchExtractor.java | 6 +++--- .../services/youtube/linkHandler/YoutubeParsingHelper.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index e4251ffc7..51918e9ce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -29,7 +29,7 @@ import java.util.Map; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; @@ -111,7 +111,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidResponseBody(getDownloader().post(url, headers, json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, json)); try { initialData = JsonParser.object().from(responseBody); @@ -215,7 +215,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidResponseBody(getDownloader().post(pageUrl, headers, json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(pageUrl, headers, json)); final JsonObject ajaxJson; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index c639261bc..54bfe95c5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -428,7 +428,7 @@ public class YoutubeParsingHelper { return thumbnailUrl; } - public static String getValidResponseBody(final Response response) + public static String getValidJsonResponseBody(final Response response) throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { throw new ContentNotAvailableException("Not found" + @@ -466,7 +466,7 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); final Response response = getDownloader().get(url, headers, localization); - final String responseBody = getValidResponseBody(response); + final String responseBody = getValidJsonResponseBody(response); try { return JsonParser.array().from(responseBody); From a7d3d149109a5625a3ec40ebf4353246a612f551 Mon Sep 17 00:00:00 2001 From: Tobias Groza Date: Fri, 3 Apr 2020 18:14:14 +0200 Subject: [PATCH 59/66] Add link to JDoc to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d62b7a97..835b640a0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # NewPipe Extractor -[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [Documentation](https://teamnewpipe.github.io/documentation/) +[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [JDoc](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/) • [Documentation](https://teamnewpipe.github.io/documentation/) NewPipe Extractor is a library for extracting things from streaming sites. It is a core component of [NewPipe](https://github.com/TeamNewPipe/NewPipe), but could be used independently. From 093762e793810bcdded4167fd615ade6639a0d59 Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 30 Mar 2020 11:48:00 +0200 Subject: [PATCH 60/66] throw ContentNotSupportedException when content is know to be unsupported --- .../exceptions/ContentNotSupportedException.java | 11 +++++++++++ .../soundcloud/SoundcloudStreamExtractor.java | 5 +++++ .../YoutubePlaylistLinkHandlerFactory.java | 3 ++- .../schabi/newpipe/extractor/stream/StreamInfo.java | 5 ++++- 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ContentNotSupportedException.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ContentNotSupportedException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ContentNotSupportedException.java new file mode 100644 index 000000000..c32575b45 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ContentNotSupportedException.java @@ -0,0 +1,11 @@ +package org.schabi.newpipe.extractor.exceptions; + +public class ContentNotSupportedException extends ParsingException { + public ContentNotSupportedException(String message) { + super(message); + } + + public ContentNotSupportedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 16eaad012..eb7d57429 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -196,6 +197,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { throw new ExtractionException("Could not get SoundCloud's track audio url", e); } + if (audioStreams.isEmpty()) { + throw new ContentNotSupportedException("HLS audio streams / opus streams are not yet supported"); + } + return audioStreams; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index cf51281b3..1950f697b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; @@ -47,7 +48,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) { - throw new ParsingException("YouTube Mix playlists are not yet supported"); + throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported"); } return listID; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 580f9c213..0e5ff0808 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.utils.DashMpdParser; @@ -47,7 +48,7 @@ public class StreamInfo extends Info { } public StreamInfo(int serviceId, String url, String originalUrl, StreamType streamType, String id, String name, - int ageLimit) { + int ageLimit) { super(serviceId, id, url, originalUrl, name); this.streamType = streamType; this.ageLimit = ageLimit; @@ -131,6 +132,8 @@ public class StreamInfo extends Info { /* Load and extract audio */ try { streamInfo.setAudioStreams(extractor.getAudioStreams()); + } catch (ContentNotSupportedException e) { + throw e; } catch (Exception e) { streamInfo.addError(new ExtractionException("Couldn't get audio streams", e)); } From a22104cbda19fa98b8238addd0425613dcddb665 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 1 Apr 2020 21:50:28 +0200 Subject: [PATCH 61/66] add tests for ContentNotSupportedException --- .../SoundcloudStreamExtractorDefaultTest.java | 222 ++++++++++-------- 1 file changed, 125 insertions(+), 97 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 86203f4c3..75178c4c9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -5,6 +5,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -25,107 +26,134 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; * Test for {@link StreamExtractor} */ public class SoundcloudStreamExtractorDefaultTest { - private static SoundcloudStreamExtractor extractor; - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (SoundcloudStreamExtractor) SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); - extractor.fetchPage(); + public static class LilUziVertDoWhatIWant { + private static SoundcloudStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (SoundcloudStreamExtractor) SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetValidTimeStamp() throws IOException, ExtractionException { + StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69"); + assertEquals("69", extractor.getTimeStamp() + ""); + } + + @Test + public void testGetTitle() throws ParsingException { + assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertEquals("LIL UZI VERT", extractor.getUploaderName()); + } + + @Test + public void testGetLength() throws ParsingException { + assertEquals(175, extractor.getLength()); + } + + @Test + public void testGetViewCount() throws ParsingException { + assertTrue(Long.toString(extractor.getViewCount()), + extractor.getViewCount() > 44227978); + } + + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse("2016/07/31 18:18:07 +0000")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetAudioStreams() throws IOException, ExtractionException { + assertFalse(extractor.getAudioStreams().isEmpty()); + } + + @Test + public void testStreamType() throws ParsingException { + assertTrue(extractor.getStreamType() == StreamType.AUDIO_STREAM); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } } - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", - extractor.getTimeStamp() <= 0); - } + public static class ContentNotSupported { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } - @Test - public void testGetValidTimeStamp() throws IOException, ExtractionException { - StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69"); - assertEquals("69", extractor.getTimeStamp() + ""); - } + @Test(expected = ContentNotSupportedException.class) + public void hlsAudioStream() throws Exception { + final StreamExtractor extractor = + SoundCloud.getStreamExtractor("https://soundcloud.com/dualipa/cool"); + extractor.fetchPage(); + extractor.getAudioStreams(); + } - @Test - public void testGetTitle() throws ParsingException { - assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName()); - } - - @Test - public void testGetDescription() throws ParsingException { - assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertEquals("LIL UZI VERT", extractor.getUploaderName()); - } - - @Test - public void testGetLength() throws ParsingException { - assertEquals(175, extractor.getLength()); - } - - @Test - public void testGetViewCount() throws ParsingException { - assertTrue(Long.toString(extractor.getViewCount()), - extractor.getViewCount() > 44227978); - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse("2016/07/31 18:18:07 +0000")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - } - - @Test - public void testGetUploaderUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl()); - } - - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } - - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetAudioStreams() throws IOException, ExtractionException { - assertFalse(extractor.getAudioStreams().isEmpty()); - } - - @Test - public void testStreamType() throws ParsingException { - assertTrue(extractor.getStreamType() == StreamType.AUDIO_STREAM); - } - - @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); - assertFalse(relatedVideos.getItems().isEmpty()); - assertTrue(relatedVideos.getErrors().isEmpty()); - } - - @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + @Test(expected = ContentNotSupportedException.class) + public void bothHlsAndOpusAudioStreams() throws Exception { + final StreamExtractor extractor = + SoundCloud.getStreamExtractor("https://soundcloud.com/lil-baby-4pf/no-sucker"); + extractor.fetchPage(); + extractor.getAudioStreams(); + } } } + From 9a7c6b7ab00c0f5b8337232fc66d3d9b538c229f Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 1 Apr 2020 22:31:53 +0200 Subject: [PATCH 62/66] use ContentNotSupportedException for channels without tabs --- .../extractors/YoutubeChannelExtractor.java | 3 ++- .../youtube/YoutubeChannelExtractorTest.java | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 385980745..6a570da53 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -310,7 +311,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } if (videoTab == null) { - throw new ParsingException("Could not find Videos tab"); + throw new ContentNotSupportedException("This channel has no Videos tab"); } try { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 3615bc63c..a916c5602 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -5,13 +5,14 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; -import org.schabi.newpipe.extractor.channel.ChannelInfo; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; -import java.util.List; +import java.io.IOException; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; @@ -45,6 +46,20 @@ public class YoutubeChannelExtractorTest { } } + public static class NotSupported { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotSupportedException.class) + public void noVideoTab() throws Exception { + final ChannelExtractor extractor = YouTube.getChannelExtractor("https://invidio.us/channel/UC-9-kyTW8ZkZNDHQJ6FgpwQ"); + extractor.fetchPage(); + extractor.getInitialPage(); + } + } + public static class Gronkh implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; From a20d53dd98b8bfd55818fc5cb8f611608d27f8e4 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 8 Apr 2020 15:30:39 +0200 Subject: [PATCH 63/66] remove the mention of Opus on wb9688 advise when ContentNotSupportedException is thrown --- .../services/soundcloud/SoundcloudStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index eb7d57429..9ebb3dd6e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -198,7 +198,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } if (audioStreams.isEmpty()) { - throw new ContentNotSupportedException("HLS audio streams / opus streams are not yet supported"); + throw new ContentNotSupportedException("HLS audio streams are not yet supported"); } return audioStreams; From b6e6f403a80fe4ae691f81bfe53cfc1723de7345 Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Thu, 9 Apr 2020 20:37:49 +0300 Subject: [PATCH 64/66] add support for PeerTube channels extraction --- .../services/peertube/PeertubeService.java | 5 +- .../extractors/PeertubeChannelExtractor.java | 4 +- .../PeertubeCommentsInfoItemExtractor.java | 2 +- .../extractors/PeertubeStreamExtractor.java | 2 +- .../PeertubeStreamInfoItemExtractor.java | 3 +- .../extractors/PeertubeUserExtractor.java | 187 ++++++++++++++++ .../PeertubeChannelLinkHandlerFactory.java | 10 +- .../PeertubeChannelExtractorTest.java | 33 ++- ...PeertubeChannelLinkHandlerFactoryTest.java | 4 +- .../peertube/PeertubeUserExtractorTest.java | 205 ++++++++++++++++++ 10 files changed, 425 insertions(+), 30 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 29cc2b1de..53e0f5463 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -77,7 +77,10 @@ public class PeertubeService extends StreamingService { @Override public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return new PeertubeChannelExtractor(this, linkHandler); + + return linkHandler.getUrl().matches("^.*\\/accounts\\/[^\\/]*$") ? + new PeertubeUserExtractor(this, linkHandler) : + new PeertubeChannelExtractor(this, linkHandler); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index e5acf1ee0..dc27be80c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -57,7 +57,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getFeedUrl() throws ParsingException { - return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id"); + return getBaseUrl() + "/feeds/videos.xml?videoChannelId=" + json.get("id"); } @Override @@ -181,7 +181,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getOriginalUrl() throws ParsingException { - return baseUrl + "/accounts/" + getId(); + return baseUrl + "/" + getId(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index b7cbcec7e..d3b43fae2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -97,7 +97,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac public String getAuthorEndpoint() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 76ead5d61..2b7fa0945 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -128,7 +128,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(json, "account.name"); String host = JsonUtils.getString(json, "account.host"); - return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index c4ed77305..df8b8a609 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -51,7 +51,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + + return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java new file mode 100644 index 000000000..2ada58550 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java @@ -0,0 +1,187 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import java.io.IOException; + +public class PeertubeUserExtractor extends ChannelExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + private JsonObject json; + private final String baseUrl; + + public PeertubeUserExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { + super(service, linkHandler); + this.baseUrl = getBaseUrl(); + } + + @Override + public String getAvatarUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(json, "avatar.path"); + } catch (Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + + @Override + public String getBannerUrl() throws ParsingException { + return null; + } + + @Override + public String getFeedUrl() throws ParsingException { + return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + Number number = JsonUtils.getNumber(json, "followersCount"); + return number.longValue(); + } + + @Override + public String getDescription() throws ParsingException { + try { + return JsonUtils.getString(json, "description"); + } catch (ParsingException e) { + return "No description"; + } + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + } catch (Exception e) { + throw new ParsingException("unable to extract channel streams", e); + } + + for (Object c : contents) { + if (c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Response response = getDownloader().get(pageUrl); + JsonObject json = null; + if (null != response && !StringUtil.isBlank(response.responseBody())) { + try { + json = JsonParser.object().from(response.responseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if (json != null) { + PeertubeParsingHelper.validate(json); + Number number = JsonUtils.getNumber(json, "total"); + if (number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get PeerTube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if (StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if (nextStart >= total) { + return ""; + } else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + Response response = downloader.get(getUrl()); + if (null != response && null != response.responseBody()) { + setInitialData(response.responseBody()); + } else { + throw new ExtractionException("Unable to extract PeerTube channel data"); + } + + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract peertube channel data", e); + } + if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "displayName"); + } + + @Override + public String getOriginalUrl() throws ParsingException { + return baseUrl + "/" + getId(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index c1e3f570f..27e0fa593 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -10,8 +10,8 @@ import java.util.List; public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); - private static final String ID_PATTERN = "/accounts/([^/?&#]*)"; - private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/"; + private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)"; + private static final String API_ENDPOINT = "/api/v1/"; public static PeertubeChannelLinkHandlerFactory getInstance() { return instance; @@ -19,7 +19,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { - return Parser.matchGroup1(ID_PATTERN, url); + return Parser.matchGroup(ID_PATTERN, url, 0); } @Override @@ -31,11 +31,11 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - return baseUrl + ACCOUNTS_ENDPOINT + id; + return baseUrl + API_ENDPOINT + id; } @Override public boolean onAcceptUrl(String url) { - return url.contains("/accounts/"); + return url.contains("/accounts/") || url.contains("/video-channels/"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 461095598..9dc4b013e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.services.DefaultTests.*; @@ -20,7 +19,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link PeertubeChannelExtractor} */ public class PeertubeChannelExtractorTest { - public static class KDE implements BaseChannelExtractorTest { + public static class DanDAugeTutoriels implements BaseChannelExtractorTest { private static PeertubeChannelExtractor extractor; @BeforeClass @@ -29,7 +28,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); + .getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"); extractor.fetchPage(); } @@ -44,22 +43,22 @@ public class PeertubeChannelExtractorTest { @Test public void testName() throws ParsingException { - assertEquals("The KDE Community", extractor.getName()); + assertEquals("Dan d'Auge tutoriels", extractor.getName()); } @Test public void testId() throws ParsingException { - assertEquals("kde", extractor.getId()); + assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -98,16 +97,16 @@ public class PeertubeChannelExtractorTest { @Test public void testFeedUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl()); + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1361", extractor.getFeedUrl()); } @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 4); } } - public static class Booteille implements BaseChannelExtractorTest { + public static class Divers implements BaseChannelExtractorTest { private static PeertubeChannelExtractor extractor; @BeforeClass @@ -116,7 +115,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); + .getChannelExtractor("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457"); extractor.fetchPage(); } @@ -141,22 +140,22 @@ public class PeertubeChannelExtractorTest { @Test public void testName() throws ParsingException { - assertEquals("booteille", extractor.getName()); + assertEquals("Divers", extractor.getName()); } @Test public void testId() throws ParsingException { - assertEquals("booteille", extractor.getId()); + assertEquals("video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -195,12 +194,12 @@ public class PeertubeChannelExtractorTest { @Test public void testFeedUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl()); + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1227", extractor.getFeedUrl()); } @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index f9b0c69d1..c44c7548f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -30,7 +30,7 @@ public class PeertubeChannelLinkHandlerFactoryTest { @Test public void getIdFromUrl() throws ParsingException { - assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); - assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); + assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java new file mode 100644 index 000000000..a1084454a --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java @@ -0,0 +1,205 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeUserExtractor; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + +/** + * Test for {@link PeertubeUserExtractor} + */ +public class PeertubeUserExtractorTest { + public static class KDE implements BaseChannelExtractorTest { + private static PeertubeUserExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeUserExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("The KDE Community", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("accounts/kde", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + } + } + + public static class Booteille implements BaseChannelExtractorTest { + private static PeertubeUserExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeUserExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Additional Testing + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testGetPageInNewExtractor() throws Exception { + final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("booteille", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("accounts/booteille", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); + } + } +} From 2c9f1260eb969fd98d147d5f9684370b855e0764 Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Fri, 10 Apr 2020 16:40:53 +0300 Subject: [PATCH 65/66] Fix wb9688 review comments * Rename PeertubeUserExtractor to PeertubeAccountExtractor * Add test for video-channels in PeertubeChannelLinkHandlerFactoryTest * Compatibility support for older versions (use "accounts/" as default) --- .../services/peertube/PeertubeService.java | 8 +++++--- ...xtractor.java => PeertubeAccountExtractor.java} | 4 ++-- .../PeertubeChannelLinkHandlerFactory.java | 8 +++++++- ...Test.java => PeertubeAccountExtractorTest.java} | 14 +++++++------- .../PeertubeChannelLinkHandlerFactoryTest.java | 1 + 5 files changed, 22 insertions(+), 13 deletions(-) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/{PeertubeUserExtractor.java => PeertubeAccountExtractor.java} (97%) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/{PeertubeUserExtractorTest.java => PeertubeAccountExtractorTest.java} (95%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 53e0f5463..e025c2be0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -78,9 +78,11 @@ public class PeertubeService extends StreamingService { public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return linkHandler.getUrl().matches("^.*\\/accounts\\/[^\\/]*$") ? - new PeertubeUserExtractor(this, linkHandler) : - new PeertubeChannelExtractor(this, linkHandler); + if (linkHandler.getUrl().contains("/video-channels/")) { + return new PeertubeChannelExtractor(this, linkHandler); + } else { + return new PeertubeAccountExtractor(this, linkHandler); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 2ada58550..ecde0d43c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeUserExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -21,7 +21,7 @@ import org.schabi.newpipe.extractor.utils.Parser.RegexException; import java.io.IOException; -public class PeertubeUserExtractor extends ChannelExtractor { +public class PeertubeAccountExtractor extends ChannelExtractor { private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; @@ -34,7 +34,7 @@ public class PeertubeUserExtractor extends ChannelExtractor { private JsonObject json; private final String baseUrl; - public PeertubeUserExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { + public PeertubeAccountExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { super(service, linkHandler); this.baseUrl = getBaseUrl(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index 27e0fa593..ef81ca4b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -31,7 +31,13 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - return baseUrl + API_ENDPOINT + id; + + if (id.matches(ID_PATTERN)) { + return baseUrl + API_ENDPOINT + id; + } else { + // This is needed for compatibility with older versions were we didn't support video channels yet + return baseUrl + API_ENDPOINT + "accounts/" + id; + } } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java similarity index 95% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index a1084454a..fd944f49d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeUserExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeUserExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; @@ -16,18 +16,18 @@ import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.services.DefaultTests.*; /** - * Test for {@link PeertubeUserExtractor} + * Test for {@link PeertubeAccountExtractor} */ -public class PeertubeUserExtractorTest { +public class PeertubeAccountExtractorTest { public static class KDE implements BaseChannelExtractorTest { - private static PeertubeUserExtractor extractor; + private static PeertubeAccountExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeUserExtractor) PeerTube + extractor = (PeertubeAccountExtractor) PeerTube .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); extractor.fetchPage(); } @@ -107,14 +107,14 @@ public class PeertubeUserExtractorTest { } public static class Booteille implements BaseChannelExtractorTest { - private static PeertubeUserExtractor extractor; + private static PeertubeAccountExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeUserExtractor) PeerTube + extractor = (PeertubeAccountExtractor) PeerTube .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); extractor.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index c44c7548f..28fe1981b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -32,5 +32,6 @@ public class PeertubeChannelLinkHandlerFactoryTest { public void getIdFromUrl() throws ParsingException { assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId()); } } From 4afe657f6ff97860460dec391adabc0bc62afbb6 Mon Sep 17 00:00:00 2001 From: Roy Yosef Date: Fri, 10 Apr 2020 19:05:38 +0300 Subject: [PATCH 66/66] Add tests for getUrlFromId and test for video-channels in acceptUrlTest --- .../PeertubeChannelLinkHandlerFactoryTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index 28fe1981b..d47dc6f64 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChanne import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; /** * Test for {@link PeertubeChannelLinkHandlerFactory} @@ -19,6 +20,7 @@ public class PeertubeChannelLinkHandlerFactoryTest { @BeforeClass public static void setUp() { + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); linkHandler = PeertubeChannelLinkHandlerFactory.getInstance(); NewPipe.init(DownloaderTestImpl.getInstance()); } @@ -26,6 +28,7 @@ public class PeertubeChannelLinkHandlerFactoryTest { @Test public void acceptUrlTest() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa")); } @Test @@ -34,4 +37,11 @@ public class PeertubeChannelLinkHandlerFactoryTest { assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId()); } + + @Test + public void getUrlFromId() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("kranti@videos.squat.net").getUrl()); + } }