From 0c27198ba17bc8721fb712fd187dab8153743fdb Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 02:21:23 -0300 Subject: [PATCH 1/9] Include errors from child collectors in the search collector --- .../search/InfoItemsSearchCollector.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java index 206fb68fb..bfa8201a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java @@ -11,6 +11,10 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + /* * Created by Christian Schabesberger on 12.02.17. * @@ -55,6 +59,24 @@ public class InfoItemsSearchCollector extends InfoItemsCollector getErrors() { + final List errors = new ArrayList<>(super.getErrors()); + errors.addAll(streamCollector.getErrors()); + errors.addAll(userCollector.getErrors()); + errors.addAll(playlistCollector.getErrors()); + + return Collections.unmodifiableList(errors); + } + + @Override + public void reset() { + super.reset(); + streamCollector.reset(); + userCollector.reset(); + playlistCollector.reset(); + } + @Override public InfoItem extract(InfoItemExtractor extractor) throws ParsingException { // Use the corresponding collector for each item extractor type From d72130edae12ff4a85ad18b9957adb8bdef68930 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 02:27:31 -0300 Subject: [PATCH 2/9] Always return a new item collector in the search extractor --- .../org/schabi/newpipe/extractor/search/SearchExtractor.java | 5 +---- .../media_ccc/extractors/MediaCCCSearchExtractor.java | 3 +-- .../services/soundcloud/SoundcloudSearchExtractor.java | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index 0acc640e6..8b900ab7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -17,11 +17,8 @@ public abstract class SearchExtractor extends ListExtractor { } } - private final InfoItemsSearchCollector collector; - public SearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); - collector = new InfoItemsSearchCollector(service.getServiceId()); } public String getSearchString() { @@ -31,7 +28,7 @@ public abstract class SearchExtractor extends ListExtractor { public abstract String getSearchSuggestion() throws ParsingException; protected InfoItemsSearchCollector getInfoItemSearchCollector() { - return collector; + return new InfoItemsSearchCollector(getService().getServiceId()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 6d740337a..241d6d1e4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -47,8 +47,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); - searchItems.reset(); + final InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index 42cd8ae0f..de02af6bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -79,7 +79,6 @@ public class SoundcloudSearchExtractor extends SearchExtractor { private InfoItemsCollector collectItems(JsonArray searchCollection) { final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - collector.reset(); for (Object result : searchCollection) { if (!(result instanceof JsonObject)) continue; From 9704fc99521959c7abea90d6600dfa1d40de3bca Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:13:11 -0300 Subject: [PATCH 3/9] Improve search extractor tests for services --- .../services/BaseSearchExtractorTest.java | 7 + .../services/DefaultExtractorTest.java | 59 ++++++ .../services/DefaultListExtractorTest.java | 50 +++++ .../services/DefaultSearchExtractorTest.java | 34 ++++ .../extractor/services/DefaultTests.java | 33 ++- .../MediaCCCSearchExtractorAllTest.java | 58 ------ ...ediaCCCSearchExtractorConferencesTest.java | 48 ----- .../MediaCCCSearchExtractorEventsTest.java | 72 ------- .../search/MediaCCCSearchExtractorTest.java | 90 +++++++++ .../PeertubeSearchExtractorBaseTest.java | 28 --- .../PeertubeSearchExtractorDefaultTest.java | 89 -------- .../search/PeertubeSearchExtractorTest.java | 59 ++++++ .../SoundcloudSearchExtractorBaseTest.java | 55 ----- ...ndcloudSearchExtractorChannelOnlyTest.java | 66 ------ .../SoundcloudSearchExtractorDefaultTest.java | 104 ---------- .../search/SoundcloudSearchExtractorTest.java | 135 +++++++++++++ .../YoutubeSearchExtractorBaseTest.java | 50 ----- ...YoutubeSearchExtractorChannelOnlyTest.java | 115 ----------- .../YoutubeSearchExtractorDefaultTest.java | 142 ------------- .../search/YoutubeSearchExtractorTest.java | 190 ++++++++++++++++++ .../search/YoutubeSearchPagingTest.java | 69 ------- 21 files changed, 653 insertions(+), 900 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorBaseTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java new file mode 100644 index 000000000..e82ad7d0a --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java @@ -0,0 +1,7 @@ +package org.schabi.newpipe.extractor.services; + +@SuppressWarnings("unused") +public interface BaseSearchExtractorTest extends BaseListExtractorTest { + void testSearchString() throws Exception; + void testSearchSuggestion() throws Exception; +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java new file mode 100644 index 000000000..40891d494 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java @@ -0,0 +1,59 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.Test; +import org.schabi.newpipe.extractor.Extractor; +import org.schabi.newpipe.extractor.StreamingService; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; + +public abstract class DefaultExtractorTest implements BaseExtractorTest { + public abstract T extractor() throws Exception; + + public abstract StreamingService expectedService() throws Exception; + public abstract String expectedName() throws Exception; + public abstract String expectedId() throws Exception; + public abstract String expectedUrlContains() throws Exception; + public abstract String expectedOriginalUrlContains() throws Exception; + + @Test + @Override + public void testServiceId() throws Exception { + assertEquals(expectedService().getServiceId(), extractor().getServiceId()); + } + + @Test + @Override + public void testName() throws Exception { + assertEquals(expectedName(), extractor().getName()); + } + + @Test + @Override + public void testId() throws Exception { + assertEquals(expectedId(), extractor().getId()); + } + + @Test + @Override + public void testUrl() throws Exception { + final String url = extractor().getUrl(); + final String expectedContains = expectedUrlContains(); + + assertIsSecureUrl(url); + assertTrue("Url \"" + url + "\" doesn't contains \"" + expectedContains + "\"", + url.contains(expectedContains)); + } + + @Test + @Override + public void testOriginalUrl() throws Exception { + final String originalUrl = extractor().getOriginalUrl(); + final String expectedContains = expectedOriginalUrlContains(); + + assertIsSecureUrl(originalUrl); + assertTrue("Original url \"" + originalUrl + "\" doesn't contains \"" + expectedContains + "\"", + originalUrl.contains(expectedContains)); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java new file mode 100644 index 000000000..462eba1a1 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java @@ -0,0 +1,50 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.Test; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; + +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + +public abstract class DefaultListExtractorTest> extends DefaultExtractorTest + implements BaseListExtractorTest { + + @Nullable + public InfoItem.InfoType expectedInfoItemType() { + return null; + } + + public boolean expectedHasMoreItems() { + return true; + } + + @Test + @Override + public void testRelatedItems() throws Exception { + final ListExtractor extractor = extractor(); + + final InfoItem.InfoType expectedType = expectedInfoItemType(); + final ListExtractor.InfoItemsPage items = defaultTestRelatedItems(extractor); + if (expectedType != null) { + assertOnlyContainsType(items, expectedType); + } + } + + @Test + @Override + public void testMoreRelatedItems() throws Exception { + final ListExtractor extractor = extractor(); + + if (expectedHasMoreItems()) { + final InfoItem.InfoType expectedType = expectedInfoItemType(); + final ListExtractor.InfoItemsPage items = defaultTestMoreItems(extractor); + if (expectedType != null) { + assertOnlyContainsType(items, expectedType); + } + } else { + assertNoMoreItems(extractor); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java new file mode 100644 index 000000000..93c4eac70 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java @@ -0,0 +1,34 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.Test; +import org.schabi.newpipe.extractor.search.SearchExtractor; + + +import javax.annotation.Nullable; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; + +public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTest + implements BaseSearchExtractorTest { + + public abstract String expectedSearchString(); + @Nullable public abstract String expectedSearchSuggestion(); + + @Test + @Override + public void testSearchString() throws Exception { + assertEquals(expectedSearchString(), extractor().getSearchString()); + } + + @Test + @Override + public void testSearchSuggestion() throws Exception { + final String expectedSearchSuggestion = expectedSearchSuggestion(); + if (expectedSearchSuggestion == null || expectedSearchSuggestion.isEmpty()) { + assertEmpty("Suggestion was expected to be empty", extractor().getSearchSuggestion()); + } else { + assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion()); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index 6c2dd9ef9..b34bbbff9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -2,23 +2,22 @@ package org.schabi.newpipe.extractor.services; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.util.Calendar; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.*; -import static org.schabi.newpipe.extractor.StreamingService.*; +import static org.schabi.newpipe.extractor.StreamingService.LinkType; public final class DefaultTests { public static void defaultTestListOfItems(StreamingService expectedService, List itemsList, List errors) throws ParsingException { @@ -71,12 +70,38 @@ public final class DefaultTests { expectedLinkType, linkTypeByUrl); } + public static void assertOnlyContainsType(ListExtractor.InfoItemsPage items, InfoItem.InfoType expectedType) { + for (InfoItem item : items.getItems()) { + assertEquals("Item list contains unexpected info types", + expectedType, item.getInfoType()); + } + } + public static void assertNoMoreItems(ListExtractor extractor) throws Exception { assertFalse("More items available when it shouldn't", extractor.hasNextPage()); final String nextPageUrl = extractor.getNextPageUrl(); assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty()); } + public static void assertNoDuplicatedItems(StreamingService expectedService, + ListExtractor.InfoItemsPage page1, + ListExtractor.InfoItemsPage page2) throws Exception { + defaultTestListOfItems(expectedService, page1.getItems(), page1.getErrors()); + defaultTestListOfItems(expectedService, page2.getItems(), page2.getErrors()); + + final Set urlsSet = new HashSet<>(); + for (InfoItem item : page1.getItems()) { + urlsSet.add(item.getUrl()); + } + + for (InfoItem item : page2.getItems()) { + final boolean wasAdded = urlsSet.add(item.getUrl()); + if (!wasAdded) { + fail("Same item was on the first and second page item list"); + } + } + } + public static ListExtractor.InfoItemsPage defaultTestRelatedItems(ListExtractor extractor) throws Exception { final ListExtractor.InfoItemsPage page = extractor.getInitialPage(); final List itemsList = page.getItems(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java deleted file mode 100644 index cd33c562a..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.schabi.newpipe.extractor.services.media_ccc; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.util.Arrays; - -import static junit.framework.TestCase.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; - -/** - * Test for {@link MediaCCCSearchExtractor} - */ -public class MediaCCCSearchExtractorAllTest { - - private static SearchExtractor extractor; - private static ListExtractor.InfoItemsPage itemsPage; - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() - .fromQuery("c3", Arrays.asList(new String[0]), "")); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testIfChannelInfoItemsAvailable() { - boolean isAvialable = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof ChannelInfoItem) { - isAvialable = true; - } - } - assertTrue("ChannelInfoItem not in all list", isAvialable); - } - - @Test - public void testIfStreamInfoitemsAvailable() { - boolean isAvialable = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof StreamInfoItem) { - isAvialable = true; - } - } - assertTrue("ChannelInfoItem not in all list", isAvialable); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java deleted file mode 100644 index f9ce2334f..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.schabi.newpipe.extractor.services.media_ccc; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; - -import java.util.Arrays; - -import static junit.framework.TestCase.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; - -/** - * Test for {@link MediaCCCSearchExtractor} - */ -public class MediaCCCSearchExtractorConferencesTest { - - private static SearchExtractor extractor; - private static ListExtractor.InfoItemsPage itemsPage; - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() - .fromQuery("c3", Arrays.asList(new String[]{"conferences"}), "")); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testReturnTypeChannel() { - for (InfoItem item : itemsPage.getItems()) { - assertTrue("Item is not of type channel", item instanceof ChannelInfoItem); - } - } - - @Test - public void testItemCount() { - assertTrue("Count is to hight: " + itemsPage.getItems().size(), itemsPage.getItems().size() < 127); - assertTrue("Countis to low: " + itemsPage.getItems().size(), itemsPage.getItems().size() >= 29); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java deleted file mode 100644 index 29f90caf3..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.schabi.newpipe.extractor.services.media_ccc; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.util.Arrays; - -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; - -/** - * Test for {@link MediaCCCSearchExtractor} - */ -public class MediaCCCSearchExtractorEventsTest { - private static SearchExtractor extractor; - private static ListExtractor.InfoItemsPage itemsPage; - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() - .fromQuery("linux", Arrays.asList(new String[]{"events"}), "")); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testCount() throws Exception { - assertTrue(Integer.toString(itemsPage.getItems().size()), - itemsPage.getItems().size() >= 25); - } - - @Test - public void testServiceId() throws Exception { - assertEquals(2, extractor.getServiceId()); - } - - @Test - public void testName() throws Exception { - assertFalse(itemsPage.getItems().get(0).getName(), itemsPage.getItems().get(0).getName().isEmpty()); - } - - @Test - public void testUrl() throws Exception { - assertTrue("Url should start with: https://api.media.ccc.de/public/events/", - itemsPage.getItems().get(0).getUrl().startsWith("https://api.media.ccc.de/public/events/")); - } - - @Test - public void testThumbnailUrl() throws Exception { - assertTrue(itemsPage.getItems().get(0).getThumbnailUrl(), - itemsPage.getItems().get(0).getThumbnailUrl().startsWith("https://static.media.ccc.de/media/") - && itemsPage.getItems().get(0).getThumbnailUrl().endsWith(".jpg")); - } - - @Test - public void testReturnTypeStream() throws Exception { - for (InfoItem item : itemsPage.getItems()) { - assertTrue("Item is not of type StreamInfoItem", item instanceof StreamInfoItem); - } - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java new file mode 100644 index 000000000..812d6367a --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java @@ -0,0 +1,90 @@ +package org.schabi.newpipe.extractor.services.media_ccc.search; + +import org.junit.BeforeClass; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; + +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES; +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS; + +public class MediaCCCSearchExtractorTest { + + public static class All extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "kde"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = MediaCCC.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return MediaCCC; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public boolean expectedHasMoreItems() { return false; } + } + + public static class Conferences extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "c3"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = MediaCCC.getSearchExtractor(QUERY, singletonList(CONFERENCES), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return MediaCCC; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Nullable @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + @Override public boolean expectedHasMoreItems() { return false; } + } + + public static class Events extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "linux"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = MediaCCC.getSearchExtractor(QUERY, singletonList(EVENTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return MediaCCC; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Nullable @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean expectedHasMoreItems() { return false; } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java deleted file mode 100644 index 030bd6c53..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.schabi.newpipe.extractor.services.peertube.search; - -import org.junit.Test; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; - -import static org.junit.Assert.assertTrue; - -/** - * Test for {@link PeertubeSearchExtractor} - */ -public abstract class PeertubeSearchExtractorBaseTest { - - protected static PeertubeSearchExtractor extractor; - protected static ListExtractor.InfoItemsPage itemsPage; - - @Test - public void testResultListElementsLength() { - assertTrue(Integer.toString(itemsPage.getItems().size()), - itemsPage.getItems().size() >= 3); - } - - @Test - public void testUrl() throws Exception { - assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://peertube.mastodon.host/api/v1/search/videos")); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java deleted file mode 100644 index eeebf526b..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.schabi.newpipe.extractor.services.peertube.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -/** - * Test for {@link PeertubeSearchExtractor} - */ -public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde"); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=kde&start=12&count=12", extractor.getNextPageUrl()); - } - - @Test - public void testResultList_FirstElement() { - InfoItem firstInfoItem = itemsPage.getItems().get(0); - - assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde")); - } - - @Test - public void testResultListCheckIfContainsStreamItems() { - boolean hasStreams = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof StreamInfoItem) { - hasStreams = true; - } - } - assertTrue("Has no InfoItemStreams", hasStreams); - } - - @Test - public void testGetSecondPage() throws Exception { - extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); - itemsPage = extractor.getInitialPage(); - PeertubeSearchExtractor secondExtractor = - (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() >= 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - - assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12", - secondPage.getNextPageUrl()); - } - - - @Test - public void testId() throws Exception { - assertEquals("kde", extractor.getId()); - } - - @Test - public void testName() { - assertEquals("kde", extractor.getName()); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java new file mode 100644 index 000000000..5ea116333 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java @@ -0,0 +1,59 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; +import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; + +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; +import static org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory.VIDEOS; + +public class PeertubeSearchExtractorTest { + + public static class All extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "kde"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = PeerTube.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return PeerTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "/search/videos?search=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "/search/videos?search=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } + + public static class PagingTest { + @Test + public void duplicatedItemsCheck() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + final SearchExtractor extractor = PeerTube.getSearchExtractor("internet", singletonList(VIDEOS), ""); + extractor.fetchPage(); + + final InfoItemsPage page1 = extractor.getInitialPage(); + final InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + + assertNoDuplicatedItems(PeerTube, page1, page2); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorBaseTest.java deleted file mode 100644 index db38c8d4a..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorBaseTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.schabi.newpipe.extractor.services.soundcloud.search; - -import org.junit.Test; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor; - -import static org.junit.Assert.assertTrue; - - -/* - * Created by Christian Schabesberger on 17.06.18 - * - * Copyright (C) Christian Schabesberger 2018 - * SoundcloudSearchExtractorBaseTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** - * Test for {@link SoundcloudSearchExtractor} - */ -public abstract class SoundcloudSearchExtractorBaseTest { - - protected static SoundcloudSearchExtractor extractor; - protected static ListExtractor.InfoItemsPage itemsPage; - - - protected static String removeClientId(String url) { - String[] splitUrl = url.split("client_id=[a-zA-Z0-9]*&"); - return splitUrl[0] + splitUrl[1]; - } - - @Test - public void testResultListElementsLength() { - assertTrue(Integer.toString(itemsPage.getItems().size()), - itemsPage.getItems().size() >= 3); - } - - @Test - public void testUrl() throws Exception { - assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://api-v2.soundcloud.com/search")); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java deleted file mode 100644 index 04e7e0aba..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.schabi.newpipe.extractor.services.soundcloud.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.localization.Localization; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory; - -import static java.util.Arrays.asList; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; - -public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance(), new Localization("de", "DE")); - extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert", - asList(SoundcloudSearchQueryHandlerFactory.USERS), null); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetSecondPage() throws Exception { - SoundcloudSearchExtractor secondExtractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert", - asList(SoundcloudSearchQueryHandlerFactory.USERS), null); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() >= 3); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - - assertEquals("https://api-v2.soundcloud.com/search/users?q=lill+uzi+vert&limit=10&offset=20", - removeClientId(secondPage.getNextPageUrl())); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - assertEquals("https://api-v2.soundcloud.com/search/users?q=lill+uzi+vert&limit=10&offset=10", - removeClientId(extractor.getNextPageUrl())); - } - - @Test - public void testOnlyContainChannels() { - for (InfoItem item : itemsPage.getItems()) { - if (!(item instanceof ChannelInfoItem)) { - fail("The following item is no channel item: " + item.toString()); - } - } - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java deleted file mode 100644 index ca0d7d590..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.schabi.newpipe.extractor.services.soundcloud.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.util.Arrays; - -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; - -/* - * Created by Christian Schabesberger on 27.05.18 - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeSearchExtractorStreamTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** - * Test for {@link YoutubeSearchExtractor} - */ -public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor( - new SoundcloudSearchQueryHandlerFactory().fromQuery("lill uzi vert", - Arrays.asList(new String[]{"tracks"}), "")); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - assertEquals("https://api-v2.soundcloud.com/search/tracks?q=lill+uzi+vert&limit=10&offset=10", - removeClientId(extractor.getNextPageUrl())); - } - - @Test - public void testResultListCheckIfContainsStreamItems() { - boolean hasStreams = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof StreamInfoItem) { - hasStreams = true; - } - } - assertTrue("Has no InfoItemStreams", hasStreams); - } - - @Test - public void testGetSecondPage() throws Exception { - SoundcloudSearchExtractor secondExtractor = - (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert"); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() >= 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - - assertEquals("https://api-v2.soundcloud.com/search/tracks?q=lill+uzi+vert&limit=10&offset=20", - removeClientId(secondPage.getNextPageUrl())); - } - - - @Test - public void testId() throws Exception { - assertEquals("lill uzi vert", extractor.getId()); - } - - @Test - public void testName() { - assertEquals("lill uzi vert", extractor.getName()); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java new file mode 100644 index 000000000..8503f5733 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java @@ -0,0 +1,135 @@ +package org.schabi.newpipe.extractor.services.soundcloud.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; + +import javax.annotation.Nullable; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.*; + +public class SoundcloudSearchExtractorTest { + + public static class All extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "lill uzi vert"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } + + public static class Tracks extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "lill uzi vert"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY, singletonList(TRACKS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class Users extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "lill uzi vert"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY, singletonList(USERS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + } + + public static class Playlists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "lill uzi vert"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY, singletonList(PLAYLISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + public static class PagingTest { + @Test + public void duplicatedItemsCheck() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + final SearchExtractor extractor = SoundCloud.getSearchExtractor("cirque du soleil", singletonList(TRACKS), ""); + extractor.fetchPage(); + + final InfoItemsPage page1 = extractor.getInitialPage(); + final InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + + assertNoDuplicatedItems(SoundCloud, page1, page2); + } + } + + private static String urlEncode(String value) { + try { + return URLEncoder.encode(value, CHARSET_UTF_8); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java deleted file mode 100644 index 729254ad9..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.junit.Test; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; - -import static org.junit.Assert.assertTrue; - - -/* - * Created by Christian Schabesberger on 27.05.18 - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeSearchExtractorBaseTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** - * Test for {@link YoutubeSearchExtractor} - */ -public abstract class YoutubeSearchExtractorBaseTest { - - protected static YoutubeSearchExtractor extractor; - protected static ListExtractor.InfoItemsPage itemsPage; - - - @Test - public void testResultListElementsLength() { - assertTrue(Integer.toString(itemsPage.getItems().size()), - itemsPage.getItems().size() > 10); - } - - @Test - public void testUrl() throws Exception { - assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://www.youtube.com")); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java deleted file mode 100644 index cb72622b4..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.hamcrest.CoreMatchers; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; - -import java.net.URL; -import java.net.URLDecoder; -import java.util.LinkedHashMap; -import java.util.Map; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetSecondPage() throws Exception { - YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() > 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - URL url = new URL(extractor.getNextPageUrl()); - - assertEquals(url.getHost(), "www.youtube.com"); - assertEquals(url.getPath(), "/results"); - - Map queryPairs = new LinkedHashMap<>(); - for (String queryPair : url.getQuery().split("&")) { - int index = queryPair.indexOf("="); - queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), - URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); - } - - assertEquals("pewdiepie", queryPairs.get("search_query")); - assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); - assertTrue(queryPairs.get("continuation").length() > 5); - assertTrue(queryPairs.get("itct").length() > 5); - } - - @Ignore - @Test - public void testOnlyContainChannels() { - for (InfoItem item : itemsPage.getItems()) { - if (!(item instanceof ChannelInfoItem)) { - fail("The following item is no channel item: " + item.toString()); - } - } - } - - @Test - public void testChannelUrl() { - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof ChannelInfoItem) { - ChannelInfoItem channel = (ChannelInfoItem) item; - - if (channel.getSubscriberCount() > 1e8) { // the real PewDiePie - assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); - break; - } - } - } - - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof ChannelInfoItem) { - assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); - } - } - } - - @Test - public void testStreamCount() { - ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0); - assertTrue("Stream count does not fit: " + ci.getStreamCount(), - 4000 < ci.getStreamCount() && ci.getStreamCount() < 5500); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java deleted file mode 100644 index 1f905bdec..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ /dev/null @@ -1,142 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; - -import java.net.URL; -import java.net.URLDecoder; -import java.util.LinkedHashMap; -import java.util.Map; - -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -/* - * Created by Christian Schabesberger on 27.05.18 - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeSearchExtractorStreamTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** - * Test for {@link YoutubeSearchExtractor} - */ -public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBaseTest { - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie"); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testGetUrl() throws Exception { - assertEquals("https://www.youtube.com/results?search_query=pewdiepie&gl=GB", extractor.getUrl()); - } - - - @Test - public void testGetSecondPageUrl() throws Exception { - URL url = new URL(extractor.getNextPageUrl()); - - assertEquals(url.getHost(), "www.youtube.com"); - assertEquals(url.getPath(), "/results"); - - Map queryPairs = new LinkedHashMap<>(); - for (String queryPair : url.getQuery().split("&")) { - int index = queryPair.indexOf("="); - queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), - URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); - } - - assertEquals("pewdiepie", queryPairs.get("search_query")); - assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); - assertTrue(queryPairs.get("continuation").length() > 5); - assertTrue(queryPairs.get("itct").length() > 5); - } - - @Test - public void testResultList_FirstElement() { - InfoItem firstInfoItem = itemsPage.getItems().get(0); - InfoItem secondInfoItem = itemsPage.getItems().get(1); - - InfoItem channelItem = firstInfoItem instanceof ChannelInfoItem ? firstInfoItem - : secondInfoItem; - - // The channel should be the first item - assertTrue((firstInfoItem instanceof ChannelInfoItem) - || (secondInfoItem instanceof ChannelInfoItem)); - assertEquals("name", "PewDiePie", channelItem.getName()); - assertEquals("url", "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", channelItem.getUrl()); - } - - @Test - public void testResultListCheckIfContainsStreamItems() { - boolean hasStreams = false; - for (InfoItem item : itemsPage.getItems()) { - if (item instanceof StreamInfoItem) { - hasStreams = true; - } - } - assertTrue("Has no InfoItemStreams", hasStreams); - } - - @Test - public void testGetSecondPage() throws Exception { - YoutubeSearchExtractor secondExtractor = - (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie"); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() > 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - } - - @Test - public void testSuggestionNotNull() throws Exception { - //todo write a real test - assertNotNull(extractor.getSearchSuggestion()); - } - - - @Test - public void testId() throws Exception { - assertEquals("pewdiepie", extractor.getId()); - } - - @Test - public void testName() { - assertEquals("pewdiepie", extractor.getName()); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java new file mode 100644 index 000000000..d5f2f1af2 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -0,0 +1,190 @@ +package org.schabi.newpipe.extractor.services.youtube.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; + +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; + +public class YoutubeSearchExtractorTest { + public static class All extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } + + public static class Channel extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + } + + public static class Playlists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(PLAYLISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + public static class Videos extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class Suggestion extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "pewdeipie"; + private static final String EXPECTED_SUGGESTION = "pewdiepie"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } + + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "UCO6AK"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + + /*////////////////////////////////////////////////////////////////////////// + // Test Overrides + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testMoreRelatedItems() throws Exception { + // YouTube actually gives us an empty next page, but after that, no more pages. + assertTrue(extractor.hasNextPage()); + final ListExtractor.InfoItemsPage nextEmptyPage = extractor.getPage(extractor.getNextPageUrl()); + assertEquals(0, nextEmptyPage.getItems().size()); + assertEmptyErrors("Empty page has errors", nextEmptyPage.getErrors()); + + assertFalse("More items available when it shouldn't", nextEmptyPage.hasNextPage()); + final String nextPageUrl = nextEmptyPage.getNextPageUrl(); + assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty()); + } + } + + public static class PagingTest { + @Test + public void duplicatedItemsCheck() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + final SearchExtractor extractor = YouTube.getSearchExtractor("cirque du soleil", singletonList(VIDEOS), ""); + extractor.fetchPage(); + + final ListExtractor.InfoItemsPage page1 = extractor.getInitialPage(); + final ListExtractor.InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + + assertNoDuplicatedItems(YouTube, page1, page2); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java deleted file mode 100644 index 656e197ad..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static java.util.Collections.singletonList; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -public class YoutubeSearchPagingTest { - private static ListExtractor.InfoItemsPage page1; - private static ListExtractor.InfoItemsPage page2; - private static Set urlList1; - private static Set urlList2; - private static int page1Size; - private static int page2Size; - - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - - YoutubeSearchExtractor extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("cirque du soleil", - singletonList(YoutubeSearchQueryHandlerFactory.VIDEOS), null); - - extractor.fetchPage(); - page1 = extractor.getInitialPage(); - urlList1 = extractUrls(page1.getItems()); - assertTrue("failed to load search result page one: too few items", 15 < page1.getItems().size()); - page1Size = page1.getItems().size(); - assertEquals("duplicated items in search result on page one", page1Size, urlList1.size()); - - assertTrue("search result has no second page", page1.hasNextPage()); - assertNotNull("next page url is null", page1.getNextPageUrl()); - page2 = extractor.getPage(page1.getNextPageUrl()); - urlList2 = extractUrls(page2.getItems()); - page2Size = page2.getItems().size(); - } - - private static Set extractUrls(List list) { - Set result = new HashSet<>(); - for (InfoItem item : list) { - result.add(item.getUrl()); - } - return result; - } - - @Test - public void secondPageUniqueVideos() { - assertEquals("Second search result page has duplicated items", page2Size, urlList2.size()); - } - - @Test - public void noRepeatingVideosInPages() { - Set intersection = new HashSet<>(urlList2); - intersection.retainAll(urlList1); - assertEquals("Found the same item on first AND second search page", 0, intersection.size()); - } - -} \ No newline at end of file From 9b7999fe54c30d77e710c38ca64cea80ba1cbf5c Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:15:23 -0300 Subject: [PATCH 4/9] [YouTube] Check if channel item has subscription count in search --- .../extractors/YoutubeChannelInfoItemExtractor.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 29aa045b7..09e984085 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -70,8 +70,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getSubscriberCount() throws ParsingException { try { - String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText")); - return Utils.mixedNumberWordToLong(subscribers); + final JsonObject subscriberCountObject = channelInfoItem.getObject("subscriberCountText"); + + if (subscriberCountObject == null) { + // Subscription count is not available for this channel item. + return -1; + } + + return Utils.mixedNumberWordToLong(getTextFromObject(subscriberCountObject)); } catch (Exception e) { throw new ParsingException("Could not get subscriber count", e); } From b7f8001a49fd963d008a02e64f7bd8ff1239548b Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:15:51 -0300 Subject: [PATCH 5/9] [YouTube] Add check for channel items without description in search --- .../extractors/YoutubeChannelInfoItemExtractor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 09e984085..15211da7e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -95,7 +95,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getDescription() throws ParsingException { try { - return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); + final JsonObject descriptionObject = channelInfoItem.getObject("descriptionSnippet"); + + if (descriptionObject == null) { + // Channel have no description. + return null; + } + + return getTextFromObject(descriptionObject); } catch (Exception e) { throw new ParsingException("Could not get description", e); } From 921bf30bb7eedbd0c92843d3471cb2c092ef4754 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:16:33 -0300 Subject: [PATCH 6/9] [YouTube] Add check for channel items with no video count in search --- .../extractors/YoutubeChannelInfoItemExtractor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 15211da7e..27c082359 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -86,7 +86,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getStreamCount() throws ParsingException { try { - return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText")))); + final JsonObject videoCountObject = channelInfoItem.getObject("videoCountText"); + + if (videoCountObject == null) { + // Video count is not available, channel probably has no public uploads. + return -1; + } + + return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(videoCountObject))); } catch (Exception e) { throw new ParsingException("Could not get stream count", e); } From 04bfa0ec6a01228a6cb0f91177855e94a88f4981 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 21 Mar 2020 03:23:01 -0300 Subject: [PATCH 7/9] [PeerTube] Update video used in age limit test (was 404 not found) --- .../services/peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index e103e1a81..f5202f39d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -145,7 +145,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetAgeLimit() throws ExtractionException, IOException { assertEquals(0, extractor.getAgeLimit()); - PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/3c0da7fb-e4d9-442e-84e3-a8c47004ee28"); + PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/0d501633-f2d9-4476-87c6-71f1c02402a4"); ageLimit.fetchPage(); assertEquals(18, ageLimit.getAgeLimit()); } From c921e5d6a5651186af57ac3bd645c1ee53a3d9f5 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 23 Mar 2020 18:05:02 -0300 Subject: [PATCH 8/9] Remove unnecessary collector creation function from search extractor --- .../newpipe/extractor/search/InfoItemsSearchCollector.java | 2 +- .../org/schabi/newpipe/extractor/search/SearchExtractor.java | 4 ---- .../media_ccc/extractors/MediaCCCSearchExtractor.java | 2 +- .../services/peertube/extractors/PeertubeSearchExtractor.java | 3 +-- .../services/soundcloud/SoundcloudSearchExtractor.java | 2 +- .../services/youtube/extractors/YoutubeSearchExtractor.java | 4 ++-- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java index bfa8201a1..fd87b853d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java @@ -52,7 +52,7 @@ public class InfoItemsSearchCollector extends InfoItemsCollector { public abstract String getSearchSuggestion() throws ParsingException; - protected InfoItemsSearchCollector getInfoItemSearchCollector() { - return new InfoItemsSearchCollector(getService().getServiceId()); - } - @Override public SearchQueryHandler getLinkHandler() { return (SearchQueryHandler) super.getLinkHandler(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 241d6d1e4..96618fd94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -47,7 +47,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); + final InfoItemsSearchCollector searchItems = new InfoItemsSearchCollector(getServiceId()); if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 3840bf765..61fcdd6df 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -47,8 +47,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { } private InfoItemsCollector collectStreamsFrom(JsonObject json) throws ParsingException { - - final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); JsonArray contents; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index de02af6bb..780e7e2b1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -78,7 +78,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor { } private InfoItemsCollector collectItems(JsonArray searchCollection) { - final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); for (Object result : searchCollection) { if (!(result instanceof JsonObject)) continue; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 6fec11643..27a247096 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -79,7 +79,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); @@ -103,7 +103,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } - InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") From 0a20c53f1a5bad1c35686d19fabce855e4cd747b Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 23 Mar 2020 18:08:59 -0300 Subject: [PATCH 9/9] Use easier-to-read asserts statements in tests --- .../extractor/services/DefaultExtractorTest.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java index 40891d494..f25b9cf04 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java @@ -4,8 +4,8 @@ import org.junit.Test; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.StreamingService; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; public abstract class DefaultExtractorTest implements BaseExtractorTest { @@ -39,21 +39,15 @@ public abstract class DefaultExtractorTest implements BaseE @Override public void testUrl() throws Exception { final String url = extractor().getUrl(); - final String expectedContains = expectedUrlContains(); - assertIsSecureUrl(url); - assertTrue("Url \"" + url + "\" doesn't contains \"" + expectedContains + "\"", - url.contains(expectedContains)); + assertThat(url, containsString(expectedUrlContains())); } @Test @Override public void testOriginalUrl() throws Exception { final String originalUrl = extractor().getOriginalUrl(); - final String expectedContains = expectedOriginalUrlContains(); - assertIsSecureUrl(originalUrl); - assertTrue("Original url \"" + originalUrl + "\" doesn't contains \"" + expectedContains + "\"", - originalUrl.contains(expectedContains)); + assertThat(originalUrl, containsString(expectedOriginalUrlContains())); } }