diff --git a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index aeeff6251..662c5dca9 100644 --- a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -4,21 +4,45 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import javax.annotation.Nonnull; import java.io.IOException; +import java.util.Collections; import java.util.List; /** * Base class to extractors that have a list (e.g. playlists, users). */ -public abstract class ListExtractor extends Extractor { +public abstract class ListExtractor extends Extractor { public ListExtractor(StreamingService service, String url) { super(service, url); } + /** + * A {@link InfoItemsPage InfoItemsPage} corresponding to the initial page where the items are from the initial request and + * the nextPageUrl relative to it. + * + * @return a {@link InfoItemsPage} corresponding to the initial page + */ @Nonnull - public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException; + public abstract InfoItemsPage getInitialPage() throws IOException, ExtractionException; + + /** + * Returns an url that can be used to get the next page relative to the initial one.
+ *

Usually, these links will only work in the implementation itself.

+ * + * @return an url pointing to the next page relative to the initial page + * @see #getPage(String) + */ public abstract String getNextPageUrl() throws IOException, ExtractionException; - public abstract InfoItemPage getPage(final String nextPageUrl) throws IOException, ExtractionException; + + /** + * Get a list of items corresponding to the specific requested page. + * + * @param nextPageUrl any next page url got from the exclusive implementation of the list extractor + * @return a {@link InfoItemsPage} corresponding to the requested page + * @see #getNextPageUrl() + * @see InfoItemsPage#getNextPageUrl() + */ + public abstract InfoItemsPage getPage(final String nextPageUrl) throws IOException, ExtractionException; public boolean hasNextPage() throws IOException, ExtractionException { final String nextPageUrl = getNextPageUrl(); @@ -29,14 +53,34 @@ public abstract class ListExtractor extends Extractor { // Inner //////////////////////////////////////////////////////////////////////////*/ - public static class InfoItemPage { + /** + * A class that is used to wrap a list of gathered items and eventual errors, it + * also contains a field that points to the next available page ({@link #nextPageUrl}). + */ + public static class InfoItemsPage { + private static final InfoItemsPage EMPTY = + new InfoItemsPage<>(Collections.emptyList(), "", Collections.emptyList()); + /** - * The current list of items to this result + * A convenient method that returns a representation of an empty page. + * + * @return a type-safe page with the list of items and errors empty and the nextPageUrl set to an empty string. + */ + public static InfoItemsPage emptyPage() { + //noinspection unchecked + return (InfoItemsPage) EMPTY; + } + + + /** + * The current list of items of this page */ private final List itemsList; /** - * Next url to fetch more items + * Url pointing to the next page relative to this one + * + * @see ListExtractor#getPage(String) */ private final String nextPageUrl; @@ -45,11 +89,11 @@ public abstract class ListExtractor extends Extractor { */ private final List errors; - public InfoItemPage(InfoItemsCollector collector, String nextPageUrl) { + public InfoItemsPage(InfoItemsCollector collector, String nextPageUrl) { this(collector.getItems(), nextPageUrl, collector.getErrors()); } - public InfoItemPage(List itemsList, String nextPageUrl, List errors) { + public InfoItemsPage(List itemsList, String nextPageUrl, List errors) { this.itemsList = itemsList; this.nextPageUrl = nextPageUrl; this.errors = errors; @@ -59,7 +103,7 @@ public abstract class ListExtractor extends Extractor { return nextPageUrl != null && !nextPageUrl.isEmpty(); } - public List getItemsList() { + public List getItems() { return itemsList; } diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 49a6b5394..6a920ad94 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -1,16 +1,12 @@ package org.schabi.newpipe.extractor.channel; -import edu.umd.cs.findbugs.annotations.NonNull; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; -import java.io.IOException; /* * Created by Christian Schabesberger on 25.07.16. @@ -32,7 +28,7 @@ import java.io.IOException; * along with NewPipe. If not, see . */ -public abstract class ChannelExtractor extends ListExtractor { +public abstract class ChannelExtractor extends ListExtractor { public ChannelExtractor(StreamingService service, String url) { super(service, url); @@ -44,12 +40,6 @@ public abstract class ChannelExtractor extends ListExtractor { return getService().getChannelUrlIdHandler(); } - @NonNull - @Override - public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException; - @Override - public abstract InfoItemPage getPage(String nextPageUrl) throws IOException, ExtractionException; - public abstract String getAvatarUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException; public abstract String getFeedUrl() throws ParsingException; diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 0d9066c42..760e3efa1 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -1,6 +1,6 @@ package org.schabi.newpipe.extractor.channel; -import org.schabi.newpipe.extractor.ListExtractor.InfoItemPage; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -30,18 +30,12 @@ import java.io.IOException; * along with NewPipe. If not, see . */ -public class ChannelInfo extends ListInfo { +public class ChannelInfo extends ListInfo { public ChannelInfo(int serviceId, String url, String id, String name) { super(serviceId, id, url, name); } - - public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) - throws IOException, ExtractionException { - return service.getChannelExtractor(url).getPage(pageUrl); - } - public static ChannelInfo getInfo(String url) throws IOException, ExtractionException { return getInfo(NewPipe.getServiceByUrl(url), url); } @@ -52,6 +46,10 @@ public class ChannelInfo extends ListInfo { return getInfo(extractor); } + public static InfoItemsPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException { + return service.getChannelExtractor(url).getPage(pageUrl); + } + public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException { // important data @@ -79,7 +77,9 @@ public class ChannelInfo extends ListInfo { info.addError(e); } - info.setRelatedItems(ExtractorHelper.getInfoItemsOrLogError(info, extractor)); + final InfoItemsPage itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor); + info.setRelatedItems(itemsPage.getItems()); + info.setNextPageUrl(itemsPage.getNextPageUrl()); try { info.setSubscriberCount(extractor.getSubscriberCount()); @@ -92,7 +92,6 @@ public class ChannelInfo extends ListInfo { info.addError(e); } - info.setNextPageUrl(extractor.getNextPageUrl()); return info; } diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 98afe99e1..31b496b45 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -20,18 +20,15 @@ package org.schabi.newpipe.extractor.kiosk; * along with NewPipe. If not, see . */ -import edu.umd.cs.findbugs.annotations.NonNull; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; -import java.io.IOException; -public abstract class KioskExtractor extends ListExtractor { +public abstract class KioskExtractor extends ListExtractor { private String contentCountry = null; private final String id; @@ -43,12 +40,6 @@ public abstract class KioskExtractor extends ListExtractor { this.id = kioskId; } - @NonNull - @Override - public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException; - @Override - public abstract InfoItemPage getPage(String nextPageUrl) throws IOException, ExtractionException; - /** * For certain Websites the content of a kiosk will be different depending * on the country you want to poen the website in. Therefore you should diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index 6ae47ed9e..4f98f77cb 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -30,16 +30,16 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; -public class KioskInfo extends ListInfo { +public class KioskInfo extends ListInfo { private KioskInfo(int serviceId, String id, String url, String name) { super(serviceId, id, url, name); } - public static ListExtractor.InfoItemPage getMoreItems(StreamingService service, - String url, - String pageUrl, - String contentCountry) throws IOException, ExtractionException { + public static ListExtractor.InfoItemsPage getMoreItems(StreamingService service, + String url, + String pageUrl, + String contentCountry) throws IOException, ExtractionException { KioskList kl = service.getKioskList(); KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl); extractor.setContentCountry(contentCountry); @@ -75,7 +75,9 @@ public class KioskInfo extends ListInfo { KioskInfo info = new KioskInfo(serviceId, id, name, url); - info.setRelatedItems(ExtractorHelper.getInfoItemsOrLogError(info, extractor)); + final ListExtractor.InfoItemsPage itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor); + info.setRelatedItems(itemsPage.getItems()); + info.setNextPageUrl(itemsPage.getNextPageUrl()); return info; } diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 621004f86..4569e7b51 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -1,18 +1,14 @@ package org.schabi.newpipe.extractor.playlist; -import edu.umd.cs.findbugs.annotations.NonNull; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; -import java.io.IOException; -public abstract class PlaylistExtractor extends ListExtractor { +public abstract class PlaylistExtractor extends ListExtractor { public PlaylistExtractor(StreamingService service, String url) { super(service, url); @@ -24,12 +20,6 @@ public abstract class PlaylistExtractor extends ListExtractor { return getService().getPlaylistUrlIdHandler(); } - @NonNull - @Override - public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException; - @Override - public abstract InfoItemPage getPage(String nextPageUrl) throws IOException, ExtractionException; - public abstract String getThumbnailUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException; diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index c854233b9..24bd25b6f 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -1,26 +1,21 @@ package org.schabi.newpipe.extractor.playlist; -import org.schabi.newpipe.extractor.ListExtractor.InfoItemPage; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; -import static org.schabi.newpipe.extractor.utils.ExtractorHelper.getInfoItemsOrLogError; - -public class PlaylistInfo extends ListInfo { +public class PlaylistInfo extends ListInfo { public PlaylistInfo(int serviceId, String id, String url, String name) { super(serviceId, id, url, name); } - public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException { - return service.getPlaylistExtractor(url).getPage(pageUrl); - } - public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException { return getInfo(NewPipe.getServiceByUrl(url), url); } @@ -31,6 +26,10 @@ public class PlaylistInfo extends ListInfo { return getInfo(extractor); } + public static InfoItemsPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException { + return service.getPlaylistExtractor(url).getPage(pageUrl); + } + /** * Get PlaylistInfo from PlaylistExtractor * @@ -75,8 +74,10 @@ public class PlaylistInfo extends ListInfo { info.addError(e); } - info.setRelatedItems(getInfoItemsOrLogError(info, extractor)); - info.setNextPageUrl(extractor.getNextPageUrl()); + final InfoItemsPage itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor); + info.setRelatedItems(itemsPage.getItems()); + info.setNextPageUrl(itemsPage.getNextPageUrl()); + return info; } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index b71d2672e..0df15fa69 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -92,11 +92,11 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Nonnull @Override - public StreamInfoItemsCollector getInfoItems() throws ExtractionException { + public InfoItemsPage getInitialPage() throws ExtractionException { if(streamInfoItemsCollector == null) { computeNextPageAndGetStreams(); } - return streamInfoItemsCollector; + return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); } @Override @@ -123,7 +123,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } @Override - public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -131,6 +131,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); - return new InfoItemPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextPageUrl); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index c93120598..f6bcf47a2 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -42,7 +42,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } @Override - public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -50,7 +50,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true); - return new InfoItemPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextPageUrl); } @@ -86,10 +86,10 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Nonnull @Override - public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException { + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if(collector == null) { computNextPageAndStreams(); } - return collector; + return new InfoItemsPage<>(collector, getNextPageUrl()); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 6eb4e8ff6..84379b9e4 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -70,7 +70,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { // If the thumbnail is null, traverse the items list and get a valid one, // if it also fails, return null try { - final StreamInfoItemsCollector infoItems = getInfoItems(); + final InfoItemsPage infoItems = getInitialPage(); if (infoItems.getItems().isEmpty()) return null; for (StreamInfoItem item : infoItems.getItems()) { @@ -113,11 +113,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException { - if(streamInfoItemsCollector == null) { + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + if (streamInfoItemsCollector == null) { computeStreamsAndNextPageUrl(); } - return streamInfoItemsCollector; + return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); } private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException { @@ -134,14 +134,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public String getNextPageUrl() throws IOException, ExtractionException { - if(nextPageUrl == null) { + if (nextPageUrl == null) { computeStreamsAndNextPageUrl(); } return nextPageUrl; } @Override - public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -149,6 +149,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); - return new InfoItemPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextPageUrl); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index 4305af14e..de0cb625f 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -150,15 +150,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override - public StreamInfoItemsCollector getInfoItems() throws ExtractionException { + public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Element ul = doc.select("ul[id=\"browse-items-primary\"]").first(); collectStreamsFrom(collector, ul); - return collector; + return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override - public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -178,7 +178,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl); collectStreamsFrom(collector, ajaxHtml.select("body").first()); - return new InfoItemPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); + return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); } private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index 597b1d9c3..74f395024 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -129,15 +129,15 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException { + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first(); collectStreamsFrom(collector, tbody); - return collector; + return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override - public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -156,7 +156,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first()); - return new InfoItemPage<>(collector, getNextPageUrlFromAjax(pageJson, pageUrl)); + return new InfoItemsPage<>(collector, getNextPageUrlFromAjax(pageJson, pageUrl)); } private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index d5342724d..81c2c2680 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -25,7 +25,6 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -70,7 +69,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { } @Override - public ListExtractor.InfoItemPage getPage(String pageUrl) { + public InfoItemsPage getPage(String pageUrl) { return null; } @@ -89,7 +88,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Nonnull @Override - public StreamInfoItemsCollector getInfoItems() throws ParsingException { + public InfoItemsPage getInitialPage() throws ParsingException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]"); for(Element ul : uls) { @@ -165,6 +164,6 @@ public class YoutubeTrendingExtractor extends KioskExtractor { } } - return collector; + return new InfoItemsPage<>(collector, getNextPageUrl()); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index e50cdf07a..bad595933 100644 --- a/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -4,9 +4,9 @@ import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.util.Collections; import java.util.List; @@ -14,40 +14,29 @@ import java.util.List; public class ExtractorHelper { private ExtractorHelper() {} - public static List getInfoItemsOrLogError(Info info, ListExtractor extractor) { - InfoItemsCollector collector; + public static InfoItemsPage getItemsPageOrLogError(Info info, ListExtractor extractor) { try { - collector = extractor.getInfoItems(); + InfoItemsPage page = extractor.getInitialPage(); + info.addAllErrors(page.getErrors()); + + return page; } catch (Exception e) { info.addError(e); - return Collections.emptyList(); + return InfoItemsPage.emptyPage(); } - // Get from collector - return getInfoItems(info, collector); } public static List getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { - StreamInfoItemsCollector collector; try { - collector = extractor.getRelatedVideos(); - } catch (Exception e) { - info.addError(e); - return Collections.emptyList(); - } - // Get from collector - return getInfoItems(info, collector); - } - - private static List getInfoItems(Info info, InfoItemsCollector collector) { - List result; - try { - result = collector.getItems(); + InfoItemsCollector collector = extractor.getRelatedVideos(); info.addAllErrors(collector.getErrors()); + + //noinspection unchecked + return (List) collector.getItems(); } catch (Exception e) { info.addError(e); return Collections.emptyList(); } - return result; } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index 33c8d58cd..c2355a19f 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -32,28 +31,30 @@ public final class DefaultTests { } } - public static void defaultTestRelatedItems(ListExtractor extractor, int expectedServiceId) throws Exception { - final InfoItemsCollector itemsCollector = extractor.getInfoItems(); - final List itemsList = itemsCollector.getItems(); - List errors = itemsCollector.getErrors(); + public static ListExtractor.InfoItemsPage defaultTestRelatedItems(ListExtractor extractor, int expectedServiceId) throws Exception { + final ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + final List itemsList = page.getItems(); + List errors = page.getErrors(); defaultTestListOfItems(expectedServiceId, itemsList, errors); + return page; } - public static ListExtractor.InfoItemPage defaultTestMoreItems(ListExtractor extractor, int expectedServiceId) throws Exception { + public static ListExtractor.InfoItemsPage defaultTestMoreItems(ListExtractor extractor, int expectedServiceId) throws Exception { assertTrue("Doesn't have more items", extractor.hasNextPage()); - ListExtractor.InfoItemPage nextPage = extractor.getPage(extractor.getNextPageUrl()); - assertTrue("Next page is empty", !nextPage.getItemsList().isEmpty()); + ListExtractor.InfoItemsPage nextPage = extractor.getPage(extractor.getNextPageUrl()); + final List items = nextPage.getItems(); + assertTrue("Next page is empty", !items.isEmpty()); assertEmptyErrors("Next page have errors", nextPage.getErrors()); - defaultTestListOfItems(expectedServiceId, nextPage.getItemsList(), nextPage.getErrors()); + defaultTestListOfItems(expectedServiceId, nextPage.getItems(), nextPage.getErrors()); return nextPage; } - public static void defaultTestGetPageInNewExtractor(ListExtractor extractor, ListExtractor newExtractor, int expectedServiceId) throws Exception { + public static void defaultTestGetPageInNewExtractor(ListExtractor extractor, ListExtractor newExtractor, int expectedServiceId) throws Exception { final String nextPageUrl = extractor.getNextPageUrl(); - final ListExtractor.InfoItemPage page = newExtractor.getPage(nextPageUrl); - defaultTestListOfItems(expectedServiceId, page.getItemsList(), page.getErrors()); + final ListExtractor.InfoItemsPage page = newExtractor.getPage(nextPageUrl); + defaultTestListOfItems(expectedServiceId, page.getItems(), page.getErrors()); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index 3c725970a..f3e720c85 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -4,9 +4,10 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.util.List; @@ -47,29 +48,29 @@ public class SoundcloudChartsExtractorTest { @Test public void testGetStreams() throws Exception { - InfoItemsCollector collector = extractor.getInfoItems(); - if(!collector.getErrors().isEmpty()) { + ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + if(!page.getErrors().isEmpty()) { System.err.println("----------"); - List errors = collector.getErrors(); + List errors = page.getErrors(); for(Throwable e: errors) { e.printStackTrace(); System.err.println("----------"); } } assertTrue("no streams are received", - !collector.getItems().isEmpty() - && collector.getErrors().isEmpty()); + !page.getItems().isEmpty() + && page.getErrors().isEmpty()); } @Test public void testGetStreamsErrors() throws Exception { - assertTrue("errors during stream list extraction", extractor.getInfoItems().getErrors().isEmpty()); + assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty()); } @Test public void testHasMoreStreams() throws Exception { // Setup the streams - extractor.getInfoItems(); + extractor.getInitialPage(); assertTrue("has more streams", extractor.hasNextPage()); } @@ -80,9 +81,9 @@ public class SoundcloudChartsExtractorTest { @Test public void testGetNextPage() throws Exception { - extractor.getInfoItems().getItems(); + extractor.getInitialPage().getItems(); assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).getItemsList().isEmpty()); + || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 2a396f963..196744556 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -6,12 +6,12 @@ import org.junit.Test; import org.junit.experimental.runners.Enclosed; import org.junit.runner.RunWith; import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; @@ -274,11 +274,11 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testMoreRelatedItems() throws Exception { - ListExtractor.InfoItemPage currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId()); + ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId()); // Test for 2 more levels for (int i = 0; i < 2; i++) { currentPage = extractor.getPage(currentPage.getNextPageUrl()); - defaultTestListOfItems(SoundCloud.getServiceId(), currentPage.getItemsList(), currentPage.getErrors()); + defaultTestListOfItems(SoundCloud.getServiceId(), currentPage.getItems(), currentPage.getErrors()); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 874526223..62dbb2a71 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -5,12 +5,12 @@ import org.junit.Test; import org.junit.experimental.runners.Enclosed; import org.junit.runner.RunWith; import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -181,11 +181,11 @@ public class YoutubePlaylistExtractorTest { @Test public void testMoreRelatedItems() throws Exception { - ListExtractor.InfoItemPage currentPage = defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); // Test for 2 more levels for (int i = 0; i < 2; i++) { currentPage = extractor.getPage(currentPage.getNextPageUrl()); - defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItemsList(), currentPage.getErrors()); + defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors()); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index 0f3288bc1..ea8312a12 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -23,8 +23,9 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.Utils; import static junit.framework.TestCase.assertFalse; @@ -66,27 +67,27 @@ public class YoutubeTrendingExtractorTest { @Test public void testGetStreamsQuantity() throws Exception { - InfoItemsCollector collector = extractor.getInfoItems(); - Utils.printErrors(collector.getErrors()); - assertTrue("no streams are received", collector.getItems().size() >= 20); + ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + Utils.printErrors(page.getErrors()); + assertTrue("no streams are received", page.getItems().size() >= 20); } @Test public void testGetStreamsErrors() throws Exception { - assertEmptyErrors("errors during stream list extraction", extractor.getInfoItems().getErrors()); + assertEmptyErrors("errors during stream list extraction", extractor.getInitialPage().getErrors()); } @Test public void testHasMoreStreams() throws Exception { // Setup the streams - extractor.getInfoItems(); + extractor.getInitialPage(); assertFalse("has more streams", extractor.hasNextPage()); } @Test public void testGetNextPage() { assertTrue("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).getItemsList().isEmpty()); + || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); } @Test