diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 10caa5091..24edef600 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -214,7 +214,7 @@ public class YoutubeParsingHelper { } } } else if (s.getString("service").equals("ECATCHER")) { - // fallback to get a shortened client version which does not contain the last do digits + // fallback to get a shortened client version which does not contain the last two digits JsonArray params = s.getArray("params"); for (Object param : params) { JsonObject p = (JsonObject) param; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 7add41262..eb56bb5e8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -12,10 +12,17 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import java.util.regex.Pattern; +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; import static java.util.Arrays.asList; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest { @@ -47,18 +54,26 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto } } assertFalse("First and second page are equal", equals); - - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - // check that ctoken, continuation and itct are longer than 5 characters - Pattern pattern = Pattern.compile( - "https:\\/\\/www.youtube.com\\/results\\?search_query=pewdiepie&sp=EgIQAg%253D%253D&gl=GB&pbj=1" - + "&ctoken=[\\w%]{5,}?&continuation=[\\w%]{5,}?&itct=[\\w]{5,}?" - ); - assertTrue(pattern.matcher(extractor.getNextPageUrl()).find()); + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "www.youtube.com"); + assertEquals(url.getPath(), "/results"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals("pewdiepie", queryPairs.get("search_query")); + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); } @Ignore @@ -77,7 +92,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto if (item instanceof ChannelInfoItem) { ChannelInfoItem channel = (ChannelInfoItem) item; - if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie + if (channel.getSubscriberCount() > 5e7 && channel.getName().equals("PewDiePie")) { // the real PewDiePie assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); } else { assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 3078d8736..65ffe839d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -10,6 +10,11 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; + import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -48,13 +53,28 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testGetUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB", extractor.getUrl()); + assertEquals("https://www.youtube.com/results?search_query=pewdiepie&gl=GB", extractor.getUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=2", extractor.getNextPageUrl()); + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "www.youtube.com"); + assertEquals(url.getPath(), "/results"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals("pewdiepie", queryPairs.get("search_query")); + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); } @Test @@ -101,14 +121,12 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas } } assertFalse("First and second page are equal", equals); - - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test - public void testSuggestionNotNull() throws Exception { + public void testSuggestionNotNull() { //todo write a real test - assertTrue(extractor.getSearchSuggestion() != null); + assertNotNull(extractor.getSearchSuggestion()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index 8777cc701..fc6af4c4d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -11,11 +11,11 @@ public class YoutubeSearchQHTest { @Test public void testRegularValues() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://www.youtube.com/results?q=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); - assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); - assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); - assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 2798e8db5..333d0ee05 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -10,17 +10,24 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; import static java.util.Objects.requireNonNull; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -89,7 +96,6 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetFullLinksInDescription() throws ParsingException { assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); - assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @Test @@ -142,12 +148,12 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetAudioStreams() throws IOException, ExtractionException { + public void testGetAudioStreams() throws ExtractionException { assertFalse(extractor.getAudioStreams().isEmpty()); } @Test - public void testGetVideoStreams() throws IOException, ExtractionException { + public void testGetVideoStreams() throws ExtractionException { for (VideoStream s : extractor.getVideoStreams()) { assertIsSecureUrl(s.url); assertTrue(s.resolution.length() > 0); @@ -169,7 +175,7 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { + public void testGetRelatedVideos() throws ExtractionException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); @@ -177,13 +183,13 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + public void testGetSubtitlesListDefault() { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { + public void testGetSubtitlesList() { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } @@ -223,10 +229,6 @@ public class YoutubeStreamExtractorDefaultTest { assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); - - assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub...")); - assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8...")); - assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/...")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index b1d6f53a9..f5c7fe104 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -13,9 +13,12 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -26,7 +29,7 @@ public class YoutubeStreamExtractorLivestreamTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY"); + .getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A"); extractor.fetchPage(); } @@ -49,8 +52,7 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/")); - assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba...")); + assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists")); } @Test @@ -119,7 +121,7 @@ public class YoutubeStreamExtractorLivestreamTest { } @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { + public void testGetRelatedVideos() throws ExtractionException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); @@ -127,12 +129,12 @@ public class YoutubeStreamExtractorLivestreamTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + public void testGetSubtitlesListDefault() { assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { + public void testGetSubtitlesList() { assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } }