kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2025-04-27 23:40:36 +05:30
Code Issues Projects Releases Wiki Activity

architecture refacturing

Browse Source
This commit is contained in:
Christian Schabesberger 2018-02-24 22:20:50 +01:00
parent 9dfcb3be06
commit 86db415b18
40 changed files with 298 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java → src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java
View File

@ -11,7 +11,7 @@ import java.util.List;
* Created by Christian Schabesberger on 12.02.17. * Created by Christian Schabesberger on 12.02.17.
* *
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* InfoItemCollector.java is part of NewPipe. * InfoItemsCollector.java is part of NewPipe.
* *
* NewPipe is free software: you can redistribute it and/or modify * NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -27,7 +27,7 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public abstract class InfoItemCollector<I extends InfoItem, E> implements Collector<I,E> { public abstract class InfoItemsCollector<I extends InfoItem, E> implements Collector<I,E> {
private final List<I> itemList = new ArrayList<>(); private final List<I> itemList = new ArrayList<>();
private final List<Throwable> errors = new ArrayList<>(); private final List<Throwable> errors = new ArrayList<>();
@ -37,7 +37,7 @@ public abstract class InfoItemCollector<I extends InfoItem, E> implements Collec
* Create a new collector * Create a new collector
* @param serviceId the service id * @param serviceId the service id
*/ */
public InfoItemCollector(int serviceId) { public InfoItemsCollector(int serviceId) {
this.serviceId = serviceId; this.serviceId = serviceId;
} }

51
src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
@ -11,73 +10,73 @@ import java.util.List;
* Base class to extractors that have a list (e.g. playlists, users). * Base class to extractors that have a list (e.g. playlists, users).
*/ */
public abstract class ListExtractor extends Extractor { public abstract class ListExtractor extends Extractor {
protected String nextStreamsUrl; protected String nextPageUrl;
/** /**
* Get a new ListExtractor with the given nextStreamsUrl set. * Get a new ListExtractor with the given nextPageUrl set.
*/ */
public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws ExtractionException { public ListExtractor(StreamingService service, String url, String nextPageUrl) throws ExtractionException {
super(service, url); super(service, url);
setNextStreamsUrl(nextStreamsUrl); setNextPageUrl(nextPageUrl);
} }
@Nonnull @Nonnull
public abstract StreamInfoItemCollector getStreams() throws IOException, ExtractionException; public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException;
public abstract NextItemsResult getNextStreams() throws IOException, ExtractionException; public abstract InfoItemPage getInfoItemPage() throws IOException, ExtractionException;
public boolean hasMoreStreams() { public boolean hasNextPage() {
return nextStreamsUrl != null && !nextStreamsUrl.isEmpty(); return nextPageUrl != null && !nextPageUrl.isEmpty();
} }
public String getNextStreamsUrl() { public String getNextPageUrl() {
return nextStreamsUrl; return nextPageUrl;
} }
public void setNextStreamsUrl(String nextStreamsUrl) { public void setNextPageUrl(String nextPageUrl) {
this.nextStreamsUrl = nextStreamsUrl; this.nextPageUrl = nextPageUrl;
} }
/*////////////////////////////////////////////////////////////////////////// /*//////////////////////////////////////////////////////////////////////////
// Inner // Inner
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
public static class NextItemsResult { public static class InfoItemPage {
/** /**
* The current list of items to this result * The current list of items to this result
*/ */
public final List<InfoItem> nextItemsList; public final List<InfoItem> infoItemList;
/** /**
* Next url to fetch more items * Next url to fetch more items
*/ */
public final String nextItemsUrl; public final String nextPageUrl;
/** /**
* Errors that happened during the extraction * Errors that happened during the extraction
*/ */
public final List<Throwable> errors; public final List<Throwable> errors;
public NextItemsResult(InfoItemCollector collector, String nextItemsUrl) { public InfoItemPage(InfoItemsCollector collector, String nextPageUrl) {
this(collector.getItemList(), nextItemsUrl, collector.getErrors()); this(collector.getItemList(), nextPageUrl, collector.getErrors());
} }
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl, List<Throwable> errors) { public InfoItemPage(List<InfoItem> infoItemList, String nextPageUrl, List<Throwable> errors) {
this.nextItemsList = nextItemsList; this.infoItemList = infoItemList;
this.nextItemsUrl = nextItemsUrl; this.nextPageUrl = nextPageUrl;
this.errors = errors; this.errors = errors;
} }
public boolean hasMoreStreams() { public boolean hasNextPage() {
return nextItemsUrl != null && !nextItemsUrl.isEmpty(); return nextPageUrl != null && !nextPageUrl.isEmpty();
} }
public List<InfoItem> getNextItemsList() { public List<InfoItem> getNextItemsList() {
return nextItemsList; return infoItemList;
} }
public String getNextItemsUrl() { public String getNextPageUrl() {
return nextItemsUrl; return nextPageUrl;
} }
public List<Throwable> getErrors() { public List<Throwable> getErrors() {

4
src/main/java/org/schabi/newpipe/extractor/ListInfo.java
View File

@ -19,7 +19,7 @@ public abstract class ListInfo extends Info {
this.related_streams = related_streams; this.related_streams = related_streams;
} }
public boolean hasMoreStreams() { public boolean hasNextPage() {
return has_more_streams; return has_more_streams;
} }
@ -27,7 +27,7 @@ public abstract class ListInfo extends Info {
this.has_more_streams = has_more_streams; this.has_more_streams = has_more_streams;
} }
public String getNextStreamsUrl() { public String getNextPageUrl() {
return next_streams_url; return next_streams_url;
} }

4
src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -69,8 +69,8 @@ public abstract class StreamingService {
public abstract SearchEngine getSearchEngine(); public abstract SearchEngine getSearchEngine();
public abstract SuggestionExtractor getSuggestionExtractor(); public abstract SuggestionExtractor getSuggestionExtractor();
public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException; public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException;
public abstract KioskList getKioskList() throws ExtractionException; public abstract KioskList getKioskList() throws ExtractionException;
public abstract SubscriptionExtractor getSubscriptionExtractor(); public abstract SubscriptionExtractor getSubscriptionExtractor();

21
src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor.channel; package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor; import edu.umd.cs.findbugs.annotations.NonNull;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
@ -31,16 +31,25 @@ import java.io.IOException;
public abstract class ChannelExtractor extends ListExtractor { public abstract class ChannelExtractor extends ListExtractor {
public ChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public ChannelExtractor(StreamingService service, String url, String nextPageUrl)
super(service, url, nextStreamsUrl); throws ExtractionException {
super(service, url, nextPageUrl);
} }
@Nonnull @Nonnull
@Override @Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException { protected UrlIdHandler getUrlIdHandler() {
return getService().getChannelUrlIdHandler(); return getService().getChannelUrlIdHandler();
} }
@NonNull
@Override
public InfoItemsCollector getInfoItems()
throws IOException, ExtractionException {
return getStreams();
}
public abstract StreamInfoItemsCollector getStreams() throws IOException, ExtractionException;
public abstract String getAvatarUrl() throws ParsingException; public abstract String getAvatarUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException;
public abstract String getFeedUrl() throws ParsingException; public abstract String getFeedUrl() throws ParsingException;

13
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -1,6 +1,6 @@
package org.schabi.newpipe.extractor.channel; package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult; import org.schabi.newpipe.extractor.ListExtractor.InfoItemPage;
import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
@ -37,8 +37,9 @@ public class ChannelInfo extends ListInfo {
} }
public static NextItemsResult getMoreItems(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl)
return service.getChannelExtractor(url, nextStreamsUrl).getNextStreams(); throws IOException, ExtractionException {
return service.getChannelExtractor(url, nextPageUrl).getInfoItemPage();
} }
public static ChannelInfo getInfo(String url) throws IOException, ExtractionException { public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
@ -78,7 +79,7 @@ public class ChannelInfo extends ListInfo {
info.addError(e); info.addError(e);
} }
info.setRelatedStreams(ExtractorHelper.getStreamsOrLogError(info, extractor)); info.setRelatedStreams(ExtractorHelper.getInfoItemsOrLogError(info, extractor));
try { try {
info.setSubscriberCount(extractor.getSubscriberCount()); info.setSubscriberCount(extractor.getSubscriberCount());
@ -91,8 +92,8 @@ public class ChannelInfo extends ListInfo {
info.addError(e); info.addError(e);
} }
info.setHasMoreStreams(extractor.hasMoreStreams()); info.setHasMoreStreams(extractor.hasNextPage());
info.setNextStreamsUrl(extractor.getNextStreamsUrl()); info.setNextStreamsUrl(extractor.getNextPageUrl());
return info; return info;
} }

8
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java → src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java
View File

@ -1,13 +1,13 @@
package org.schabi.newpipe.extractor.channel; package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
/* /*
* Created by Christian Schabesberger on 12.02.17. * Created by Christian Schabesberger on 12.02.17.
* *
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* ChannelInfoItemCollector.java is part of NewPipe. * ChannelInfoItemsCollector.java is part of NewPipe.
* *
* NewPipe is free software: you can redistribute it and/or modify * NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -23,8 +23,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class ChannelInfoItemCollector extends InfoItemCollector<ChannelInfoItem, ChannelInfoItemExtractor> { public class ChannelInfoItemsCollector extends InfoItemsCollector<ChannelInfoItem, ChannelInfoItemExtractor> {
public ChannelInfoItemCollector(int serviceId) { public ChannelInfoItemsCollector(int serviceId) {
super(serviceId); super(serviceId);
} }

10
src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java
View File

@ -34,10 +34,10 @@ public abstract class KioskExtractor extends ListExtractor {
public KioskExtractor(StreamingService streamingService, public KioskExtractor(StreamingService streamingService,
String url, String url,
String nextStreamsUrl, String nextPageUrl,
String kioskId) String kioskId)
throws IOException, ExtractionException { throws ExtractionException {
super(streamingService, url, nextStreamsUrl); super(streamingService, url, nextPageUrl);
this.id = kioskId; this.id = kioskId;
} }
@ -54,12 +54,12 @@ public abstract class KioskExtractor extends ListExtractor {
@Nonnull @Nonnull
@Override @Override
public String getId() throws ParsingException { public String getId() {
return id; return id;
} }
/** /**
* Id should be the name of the kiosk, tho Id is used for identifing it in the programm, * Id should be the name of the kiosk, tho Id is used for identifing it in the frontend,
* so id should be kept in english. * so id should be kept in english.
* In order to get the name of the kiosk in the desired language we have to * In order to get the name of the kiosk in the desired language we have to
* crawl if from the website. * crawl if from the website.

10
src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View File

@ -35,14 +35,14 @@ public class KioskInfo extends ListInfo {
super(serviceId, id, url, name); super(serviceId, id, url, name);
} }
public static ListExtractor.NextItemsResult getMoreItems(StreamingService service, public static ListExtractor.InfoItemPage getMoreItems(StreamingService service,
String url, String url,
String nextStreamsUrl, String nextPageUrl,
String contentCountry) throws IOException, ExtractionException { String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList(); KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, nextStreamsUrl); KioskExtractor extractor = kl.getExtractorByUrl(url, nextPageUrl);
extractor.setContentCountry(contentCountry); extractor.setContentCountry(contentCountry);
return extractor.getNextStreams(); return extractor.getInfoItemPage();
} }
public static KioskInfo getInfo(String url, public static KioskInfo getInfo(String url,
@ -74,7 +74,7 @@ public class KioskInfo extends ListInfo {
KioskInfo info = new KioskInfo(serviceId, id, name, url); KioskInfo info = new KioskInfo(serviceId, id, name, url);
info.related_streams = ExtractorHelper.getStreamsOrLogError(info, extractor); info.related_streams = ExtractorHelper.getInfoItemsOrLogError(info, extractor);
return info; return info;
} }

16
src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -14,7 +14,7 @@ public class KioskList {
public interface KioskExtractorFactory { public interface KioskExtractorFactory {
KioskExtractor createNewKiosk(final StreamingService streamingService, KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url, final String url,
final String nextStreamUrl, final String nextPageUrl,
final String kioskId) final String kioskId)
throws ExtractionException, IOException; throws ExtractionException, IOException;
} }
@ -48,15 +48,15 @@ public class KioskList {
defaultKiosk = kioskType; defaultKiosk = kioskType;
} }
public KioskExtractor getDefaultKioskExtractor(String nextStreamUrl) public KioskExtractor getDefaultKioskExtractor(String nextPageUrl)
throws ExtractionException, IOException { throws ExtractionException, IOException {
if(defaultKiosk != null && !defaultKiosk.equals("")) { if(defaultKiosk != null && !defaultKiosk.equals("")) {
return getExtractorById(defaultKiosk, nextStreamUrl); return getExtractorById(defaultKiosk, nextPageUrl);
} else { } else {
if(!kioskList.isEmpty()) { if(!kioskList.isEmpty()) {
// if not set get any entry // if not set get any entry
Object[] keySet = kioskList.keySet().toArray(); Object[] keySet = kioskList.keySet().toArray();
return getExtractorById(keySet[0].toString(), nextStreamUrl); return getExtractorById(keySet[0].toString(), nextPageUrl);
} else { } else {
return null; return null;
} }
@ -67,7 +67,7 @@ public class KioskList {
return defaultKiosk; return defaultKiosk;
} }
public KioskExtractor getExtractorById(String kioskId, String nextStreamsUrl) public KioskExtractor getExtractorById(String kioskId, String nextPageUrl)
throws ExtractionException, IOException { throws ExtractionException, IOException {
KioskEntry ke = kioskList.get(kioskId); KioskEntry ke = kioskList.get(kioskId);
if(ke == null) { if(ke == null) {
@ -75,7 +75,7 @@ public class KioskList {
} else { } else {
return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id), return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
ke.handler.getUrl(kioskId), ke.handler.getUrl(kioskId),
nextStreamsUrl, kioskId); nextPageUrl, kioskId);
} }
} }
@ -83,12 +83,12 @@ public class KioskList {
return kioskList.keySet(); return kioskList.keySet();
} }
public KioskExtractor getExtractorByUrl(String url, String nextStreamsUrl) public KioskExtractor getExtractorByUrl(String url, String nextPageUrl)
throws ExtractionException, IOException { throws ExtractionException, IOException {
for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) { for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
KioskEntry ke = e.getValue(); KioskEntry ke = e.getValue();
if(ke.handler.acceptUrl(url)) { if(ke.handler.acceptUrl(url)) {
return getExtractorById(e.getKey(), nextStreamsUrl); return getExtractorById(e.getKey(), nextPageUrl);
} }
} }
throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); throw new ExtractionException("Could not find a kiosk that fits to the url: " + url);

17
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java
View File

@ -1,26 +1,37 @@
package org.schabi.newpipe.extractor.playlist; package org.schabi.newpipe.extractor.playlist;
import edu.umd.cs.findbugs.annotations.NonNull;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
public abstract class PlaylistExtractor extends ListExtractor { public abstract class PlaylistExtractor extends ListExtractor {
public PlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public PlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl); super(service, url, nextPageUrl);
} }
@Nonnull @Nonnull
@Override @Override
protected UrlIdHandler getUrlIdHandler() throws ParsingException { protected UrlIdHandler getUrlIdHandler() {
return getService().getPlaylistUrlIdHandler(); return getService().getPlaylistUrlIdHandler();
} }
@NonNull
@Override
public InfoItemsCollector getInfoItems()
throws IOException, ExtractionException {
return getStreams();
}
public abstract StreamInfoItemsCollector getStreams() throws IOException, ExtractionException;
public abstract String getThumbnailUrl() throws ParsingException; public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException;

14
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -1,6 +1,6 @@
package org.schabi.newpipe.extractor.playlist; package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult; import org.schabi.newpipe.extractor.ListExtractor.InfoItemPage;
import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
@ -9,7 +9,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.IOException; import java.io.IOException;
import static org.schabi.newpipe.extractor.utils.ExtractorHelper.getStreamsOrLogError; import static org.schabi.newpipe.extractor.utils.ExtractorHelper.getInfoItemsOrLogError;
public class PlaylistInfo extends ListInfo { public class PlaylistInfo extends ListInfo {
@ -17,8 +17,8 @@ public class PlaylistInfo extends ListInfo {
super(serviceId, id, url, name); super(serviceId, id, url, name);
} }
public static NextItemsResult getMoreItems(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public static InfoItemPage getMoreItems(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
return service.getPlaylistExtractor(url, nextStreamsUrl).getNextStreams(); return service.getPlaylistExtractor(url, nextPageUrl).getInfoItemPage();
} }
public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException { public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
@ -75,9 +75,9 @@ public class PlaylistInfo extends ListInfo {
info.addError(e); info.addError(e);
} }
info.setRelatedStreams(getStreamsOrLogError(info, extractor)); info.setRelatedStreams(getInfoItemsOrLogError(info, extractor));
info.setHasMoreStreams(extractor.hasMoreStreams()); info.setHasMoreStreams(extractor.hasNextPage());
info.setNextStreamsUrl(extractor.getNextStreamsUrl()); info.setNextStreamsUrl(extractor.getNextPageUrl());
return info; return info;
} }

6
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemCollector.java → src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java
View File

@ -1,11 +1,11 @@
package org.schabi.newpipe.extractor.playlist; package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class PlaylistInfoItemCollector extends InfoItemCollector<PlaylistInfoItem, PlaylistInfoItemExtractor> { public class PlaylistInfoItemsCollector extends InfoItemsCollector<PlaylistInfoItem, PlaylistInfoItemExtractor> {
public PlaylistInfoItemCollector(int serviceId) { public PlaylistInfoItemsCollector(int serviceId) {
super(serviceId); super(serviceId);
} }

24
src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java → src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java
View File

@ -1,20 +1,20 @@
package org.schabi.newpipe.extractor.search; package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
/* /*
* Created by Christian Schabesberger on 12.02.17. * Created by Christian Schabesberger on 12.02.17.
* *
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* InfoItemSearchCollector.java is part of NewPipe. * InfoItemsSearchCollector.java is part of NewPipe.
* *
* NewPipe is free software: you can redistribute it and/or modify * NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -42,17 +42,17 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
* Calling {@link #extract(InfoItemExtractor)} or {@link #commit(Object)} with any * Calling {@link #extract(InfoItemExtractor)} or {@link #commit(Object)} with any
* other extractor type will raise an exception. * other extractor type will raise an exception.
*/ */
public class InfoItemSearchCollector extends InfoItemCollector<InfoItem, InfoItemExtractor> { public class InfoItemsSearchCollector extends InfoItemsCollector<InfoItem, InfoItemExtractor> {
private String suggestion = ""; private String suggestion = "";
private final StreamInfoItemCollector streamCollector; private final StreamInfoItemsCollector streamCollector;
private final ChannelInfoItemCollector userCollector; private final ChannelInfoItemsCollector userCollector;
private final PlaylistInfoItemCollector playlistCollector; private final PlaylistInfoItemsCollector playlistCollector;
InfoItemSearchCollector(int serviceId) { InfoItemsSearchCollector(int serviceId) {
super(serviceId); super(serviceId);
streamCollector = new StreamInfoItemCollector(serviceId); streamCollector = new StreamInfoItemsCollector(serviceId);
userCollector = new ChannelInfoItemCollector(serviceId); userCollector = new ChannelInfoItemsCollector(serviceId);
playlistCollector = new PlaylistInfoItemCollector(serviceId); playlistCollector = new PlaylistInfoItemsCollector(serviceId);
} }
public void setSuggestion(String suggestion) { public void setSuggestion(String suggestion) {

8
src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java
View File

@ -35,16 +35,16 @@ public abstract class SearchEngine {
} }
} }
private final InfoItemSearchCollector collector; private final InfoItemsSearchCollector collector;
public SearchEngine(int serviceId) { public SearchEngine(int serviceId) {
collector = new InfoItemSearchCollector(serviceId); collector = new InfoItemsSearchCollector(serviceId);
} }
protected InfoItemSearchCollector getInfoItemSearchCollector() { protected InfoItemsSearchCollector getInfoItemSearchCollector() {
return collector; return collector;
} }
public abstract InfoItemSearchCollector search(String query, int page, String contentCountry, Filter filter) public abstract InfoItemsSearchCollector search(String query, int page, String contentCountry, Filter filter)
throws IOException, ExtractionException; throws IOException, ExtractionException;
} }

23
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -4,12 +4,11 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
@ -19,8 +18,8 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId; private String userId;
private JsonObject user; private JsonObject user;
public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public SoundcloudChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl); super(service, url, nextPageUrl);
} }
@Override @Override
@ -87,27 +86,27 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Nonnull @Nonnull
@Override @Override
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks" String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId() + "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20" + "&limit=20"
+ "&linked_partitioning=1"; + "&linked_partitioning=1";
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector; return collector;
} }
@Override @Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException { public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
if (!hasMoreStreams()) { if (!hasNextPage()) {
throw new ExtractionException("Channel doesn't have more streams"); throw new ExtractionException("Channel doesn't have more streams");
} }
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl);
return new NextItemsResult(collector, nextStreamsUrl); return new InfoItemPage(collector, nextPageUrl);
} }
} }

27
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View File

@ -8,18 +8,17 @@ import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
public class SoundcloudChartsExtractor extends KioskExtractor { public class SoundcloudChartsExtractor extends KioskExtractor {
private String url; private String url;
public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl, String kioskId) public SoundcloudChartsExtractor(StreamingService service, String url, String nextPageUrl, String kioskId)
throws IOException, ExtractionException { throws ExtractionException {
super(service, url, nextStreamsUrl, kioskId); super(service, url, nextPageUrl, kioskId);
this.url = url; this.url = url;
} }
@ -29,7 +28,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
@Nonnull @Nonnull
@Override @Override
public String getName() throws ParsingException { public String getName() {
return "< Implement me (♥_♥) >"; return "< Implement me (♥_♥) >";
} }
@ -40,21 +39,21 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
} }
@Override @Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException { public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
if (!hasMoreStreams()) { if (!hasNextPage()) {
throw new ExtractionException("Chart doesn't have more streams"); throw new ExtractionException("Chart doesn't have more streams");
} }
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextPageUrl, true);
return new NextItemsResult(collector, nextStreamsUrl); return new InfoItemPage(collector, nextPageUrl);
} }
@Nonnull @Nonnull
@Override @Override
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" + String apiUrl = "https://api-v2.soundcloud.com/charts" +
"?genre=soundcloud:genres:all-music" + "?genre=soundcloud:genres:all-music" +
@ -72,7 +71,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
apiUrl += "&region=soundcloud:regions:" + contentCountry; apiUrl += "&region=soundcloud:regions:" + contentCountry;
} }
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
return collector; return collector;
} }
} }

54
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -9,10 +9,10 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Parser.RegexException;
@ -112,19 +112,19 @@ public class SoundcloudParsingHelper {
/** /**
* Fetch the users from the given api and commit each of them to the collector. * Fetch the users from the given api and commit each of them to the collector.
* <p> * <p>
* This differ from {@link #getUsersFromApi(ChannelInfoItemCollector, String)} in the sense that they will always * This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense that they will always
* get MIN_ITEMS or more. * get MIN_ITEMS or more.
* *
* @param minItems the method will return only when it have extracted that many items (equal or more) * @param minItems the method will return only when it have extracted that many items (equal or more)
*/ */
public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String nextStreamsUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl); String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) { while (!nextPageUrl.isEmpty() && collector.getItemList().size() < minItems) {
nextStreamsUrl = SoundcloudParsingHelper.getUsersFromApi(collector, nextStreamsUrl); nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, nextPageUrl);
} }
return nextStreamsUrl; return nextPageUrl;
} }
/** /**
@ -132,7 +132,7 @@ public class SoundcloudParsingHelper {
* *
* @return the next streams url, empty if don't have * @return the next streams url, empty if don't have
*/ */
public static String getUsersFromApi(ChannelInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl); String response = NewPipe.getDownloader().download(apiUrl);
JsonObject responseObject; JsonObject responseObject;
try { try {
@ -149,33 +149,33 @@ public class SoundcloudParsingHelper {
} }
} }
String nextStreamsUrl; String nextPageUrl;
try { try {
nextStreamsUrl = responseObject.getString("next_href"); nextPageUrl = responseObject.getString("next_href");
if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) { } catch (Exception ignored) {
nextStreamsUrl = ""; nextPageUrl = "";
} }
return nextStreamsUrl; return nextPageUrl;
} }
/** /**
* Fetch the streams from the given api and commit each of them to the collector. * Fetch the streams from the given api and commit each of them to the collector.
* <p> * <p>
* This differ from {@link #getStreamsFromApi(StreamInfoItemCollector, String)} in the sense that they will always * This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense that they will always
* get MIN_ITEMS or more items. * get MIN_ITEMS or more items.
* *
* @param minItems the method will return only when it have extracted that many items (equal or more) * @param minItems the method will return only when it have extracted that many items (equal or more)
*/ */
public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) { while (!nextPageUrl.isEmpty() && collector.getItemList().size() < minItems) {
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextPageUrl);
} }
return nextStreamsUrl; return nextPageUrl;
} }
/** /**
@ -183,7 +183,7 @@ public class SoundcloudParsingHelper {
* *
* @return the next streams url, empty if don't have * @return the next streams url, empty if don't have
*/ */
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException { public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl); String response = NewPipe.getDownloader().download(apiUrl);
JsonObject responseObject; JsonObject responseObject;
try { try {
@ -200,18 +200,18 @@ public class SoundcloudParsingHelper {
} }
} }
String nextStreamsUrl; String nextPageUrl;
try { try {
nextStreamsUrl = responseObject.getString("next_href"); nextPageUrl = responseObject.getString("next_href");
if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) { } catch (Exception ignored) {
nextStreamsUrl = ""; nextPageUrl = "";
} }
return nextStreamsUrl; return nextPageUrl;
} }
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException { public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
return getStreamsFromApi(collector, apiUrl, false); return getStreamsFromApi(collector, apiUrl, false);
} }

22
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
@ -18,8 +18,8 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId; private String playlistId;
private JsonObject playlist; private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl); super(service, url, nextPageUrl);
} }
@Override @Override
@ -88,8 +88,8 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Nonnull @Nonnull
@Override @Override
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
// Note the "api", NOT "api-v2" // Note the "api", NOT "api-v2"
String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks" String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks"
@ -97,19 +97,19 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
+ "&limit=20" + "&limit=20"
+ "&linked_partitioning=1"; + "&linked_partitioning=1";
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
return collector; return collector;
} }
@Override @Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException { public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
if (!hasMoreStreams()) { if (!hasNextPage()) {
throw new ExtractionException("Playlist doesn't have more streams"); throw new ExtractionException("Playlist doesn't have more streams");
} }
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextPageUrl);
return new NextItemsResult(collector, nextStreamsUrl); return new InfoItemPage(collector, nextPageUrl);
} }
} }

6
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java
View File

@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException; import java.io.IOException;
@ -22,8 +22,8 @@ public class SoundcloudSearchEngine extends SearchEngine {
} }
@Override @Override
public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException { public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector(); InfoItemsSearchCollector collector = getInfoItemSearchCollector();
Downloader dl = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();

12
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -50,13 +50,13 @@ public class SoundcloudService extends StreamingService {
} }
@Override @Override
public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException { public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new SoundcloudChannelExtractor(this, url, nextStreamsUrl); return new SoundcloudChannelExtractor(this, url, nextPageUrl);
} }
@Override @Override
public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException { public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new SoundcloudPlaylistExtractor(this, url, nextStreamsUrl); return new SoundcloudPlaylistExtractor(this, url, nextPageUrl);
} }
@Override @Override
@ -70,12 +70,12 @@ public class SoundcloudService extends StreamingService {
@Override @Override
public KioskExtractor createNewKiosk(StreamingService streamingService, public KioskExtractor createNewKiosk(StreamingService streamingService,
String url, String url,
String nextStreamUrl, String nextPageUrl,
String id) String id)
throws ExtractionException, IOException { throws ExtractionException, IOException {
return new SoundcloudChartsExtractor(SoundcloudService.this, return new SoundcloudChartsExtractor(SoundcloudService.this,
url, url,
nextStreamUrl, nextPageUrl,
id); id);
} }
}; };

5
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.stream.*;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
@ -189,8 +188,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException { public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related" String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related"
+ "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId()); + "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());

4
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem; import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
@ -39,7 +39,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings" String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings"
+ "?client_id=" + SoundcloudParsingHelper.clientId() + "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=200"; + "&limit=200";
ChannelInfoItemCollector collector = new ChannelInfoItemCollector(service.getServiceId()); ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId());
// ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough // ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough
SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl); SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl);

40
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View File

@ -14,7 +14,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
@ -48,7 +48,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private Document doc; private Document doc;
/** /**
* It's lazily initialized (when getNextStreams is called) * It's lazily initialized (when getInfoItemPage is called)
*/ */
private Document nextStreamsAjax; private Document nextStreamsAjax;
@ -60,10 +60,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
*/ */
private boolean fetchingNextStreams; private boolean fetchingNextStreams;
public YoutubeChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public YoutubeChannelExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl); super(service, url, nextPageUrl);
fetchingNextStreams = nextStreamsUrl != null && !nextStreamsUrl.isEmpty(); fetchingNextStreams = nextPageUrl != null && !nextPageUrl.isEmpty();
} }
@Override @Override
@ -73,7 +73,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
doc = Jsoup.parse(pageContent, channelUrl); doc = Jsoup.parse(pageContent, channelUrl);
if (!fetchingNextStreams) { if (!fetchingNextStreams) {
nextStreamsUrl = getNextStreamsUrlFrom(doc); nextPageUrl = getNextPageUrlFrom(doc);
} }
nextStreamsAjax = null; nextStreamsAjax = null;
} }
@ -163,49 +163,49 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull @Nonnull
@Override @Override
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { public StreamInfoItemsCollector getStreams() throws ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first(); Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
collectStreamsFrom(collector, ul); collectStreamsFrom(collector, ul);
return collector; return collector;
} }
@Override @Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException { public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
if (!hasMoreStreams()) { if (!hasNextPage()) {
throw new ExtractionException("Channel doesn't have more streams"); throw new ExtractionException("Channel doesn't have more streams");
} }
fetchPage(); fetchPage();
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader()); setupNextPageAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first()); collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
return new NextItemsResult(collector, nextStreamsUrl); return new InfoItemPage(collector, nextPageUrl);
} }
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { private void setupNextPageAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextStreamsUrl); String ajaxDataRaw = downloader.download(nextPageUrl);
try { try {
JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw); JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = ajaxData.getString("content_html"); String htmlDataRaw = ajaxData.getString("content_html");
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl); nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl);
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html"); String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) { if (!nextStreamsHtmlDataRaw.isEmpty()) {
nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl)); nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl));
} else { } else {
nextStreamsUrl = ""; nextPageUrl = "";
} }
} catch (JsonParserException e) { } catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e); throw new ParsingException("Could not parse json data for next streams", e);
} }
} }
private String getNextStreamsUrlFrom(Document d) throws ParsingException { private String getNextPageUrlFrom(Document d) throws ParsingException {
try { try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if (button != null) { if (button != null) {
@ -219,7 +219,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
} }
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException { private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) throws ParsingException {
collector.reset(); collector.reset();
final String uploaderName = getName(); final String uploaderName = getName();

34
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
View File

@ -14,7 +14,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
@ -27,12 +27,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc; private Document doc;
/** /**
* It's lazily initialized (when getNextStreams is called) * It's lazily initialized (when getInfoItemPage is called)
*/ */
private Document nextStreamsAjax; private Document nextStreamsAjax;
public YoutubePlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public YoutubePlaylistExtractor(StreamingService service, String url, String nextPageUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl); super(service, url, nextPageUrl);
} }
@Override @Override
@ -40,7 +40,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
String pageContent = downloader.download(getCleanUrl()); String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getCleanUrl()); doc = Jsoup.parse(pageContent, getCleanUrl());
nextStreamsUrl = getNextStreamsUrlFrom(doc); nextPageUrl = getNextPageUrlFrom(doc);
nextStreamsAjax = null; nextStreamsAjax = null;
} }
@ -143,46 +143,46 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull @Nonnull
@Override @Override
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first(); Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
collectStreamsFrom(collector, tbody); collectStreamsFrom(collector, tbody);
return collector; return collector;
} }
@Override @Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException { public InfoItemPage getInfoItemPage() throws IOException, ExtractionException {
if (!hasMoreStreams()) { if (!hasNextPage()) {
throw new ExtractionException("Playlist doesn't have more streams"); throw new ExtractionException("Playlist doesn't have more streams");
} }
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader()); setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first()); collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
return new NextItemsResult(collector, nextStreamsUrl); return new InfoItemPage(collector, nextPageUrl);
} }
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextStreamsUrl); String ajaxDataRaw = downloader.download(nextPageUrl);
try { try {
JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw); JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>"; String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>";
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl); nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextPageUrl);
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html"); String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) { if (!nextStreamsHtmlDataRaw.isEmpty()) {
nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl)); nextPageUrl = getNextPageUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextPageUrl));
} else { } else {
nextStreamsUrl = ""; nextPageUrl = "";
} }
} catch (JsonParserException e) { } catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e); throw new ParsingException("Could not parse json data for next streams", e);
} }
} }
private String getNextStreamsUrlFrom(Document d) throws ParsingException { private String getNextPageUrlFrom(Document d) throws ParsingException {
try { try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if (button != null) { if (button != null) {
@ -196,7 +196,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
} }
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException { private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) throws ParsingException {
collector.reset(); collector.reset();
final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler(); final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler();

6
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java
View File

@ -6,7 +6,7 @@ import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException; import java.io.IOException;
@ -43,9 +43,9 @@ public class YoutubeSearchEngine extends SearchEngine {
} }
@Override @Override
public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter) public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter)
throws IOException, ExtractionException { throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector(); InfoItemsSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader(); Downloader downloader = NewPipe.getDownloader();
String url = "https://www.youtube.com/results" String url = "https://www.youtube.com/results"

12
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -70,13 +70,13 @@ public class YoutubeService extends StreamingService {
} }
@Override @Override
public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException { public ChannelExtractor getChannelExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new YoutubeChannelExtractor(this, url, nextStreamsUrl); return new YoutubeChannelExtractor(this, url, nextPageUrl);
} }
@Override @Override
public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException { public PlaylistExtractor getPlaylistExtractor(String url, String nextPageUrl) throws IOException, ExtractionException {
return new YoutubePlaylistExtractor(this, url, nextStreamsUrl); return new YoutubePlaylistExtractor(this, url, nextPageUrl);
} }
@Override @Override
@ -92,9 +92,9 @@ public class YoutubeService extends StreamingService {
try { try {
list.addKioskEntry(new KioskList.KioskExtractorFactory() { list.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override @Override
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String nextStreamUrl, String id) public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String nextPageUrl, String id)
throws ExtractionException, IOException { throws ExtractionException, IOException {
return new YoutubeTrendingExtractor(YoutubeService.this, url, nextStreamUrl, id); return new YoutubeTrendingExtractor(YoutubeService.this, url, nextPageUrl, id);
} }
}, new YoutubeTrendingUrlIdHandler(), "Trending"); }, new YoutubeTrendingUrlIdHandler(), "Trending");
list.setDefaultKiosk("Trending"); list.setDefaultKiosk("Trending");

6
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -452,7 +452,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public StreamInfoItem getNextVideo() throws IOException, ExtractionException { public StreamInfoItem getNextVideo() throws IOException, ExtractionException {
assertPageFetched(); assertPageFetched();
try { try {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collector.commit(extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]") collector.commit(extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]")
.first().select("li").first())); .first().select("li").first()));
@ -463,10 +463,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException { public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException {
assertPageFetched(); assertPageFetched();
try { try {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Element ul = doc.select("ul[id=\"watch-related\"]").first(); Element ul = doc.select("ul[id=\"watch-related\"]").first();
if (ul != null) { if (ul != null) {
for (Element li : ul.children()) { for (Element li : ul.children()) {

14
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java
View File

@ -28,7 +28,7 @@ import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
@ -37,9 +37,9 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
private Document doc; private Document doc;
public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl, String kioskId) public YoutubeTrendingExtractor(StreamingService service, String url, String nextPageUrl, String kioskId)
throws IOException, ExtractionException { throws ExtractionException {
super(service, url, nextStreamsUrl, kioskId); super(service, url, nextPageUrl, kioskId);
} }
@Override @Override
@ -61,7 +61,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
} }
@Override @Override
public ListExtractor.NextItemsResult getNextStreams() { public ListExtractor.InfoItemPage getInfoItemPage() {
return null; return null;
} }
@ -80,8 +80,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
@Nonnull @Nonnull
@Override @Override
public StreamInfoItemCollector getStreams() throws ParsingException { public StreamInfoItemsCollector getInfoItems() throws ParsingException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]"); Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]");
for(Element ul : uls) { for(Element ul : uls) {
for(final Element li : ul.children()) { for(final Element li : ul.children()) {

3
src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -29,7 +29,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
@ -139,7 +138,7 @@ public abstract class StreamExtractor extends Extractor {
public abstract StreamType getStreamType() throws ParsingException; public abstract StreamType getStreamType() throws ParsingException;
public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException;
public abstract StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException; public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;
/** /**
* Analyses the webpage's document and extracts any error message there might be. * Analyses the webpage's document and extracts any error message there might be.

8
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java → src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.stream; package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -12,7 +12,7 @@ import java.util.Vector;
* Created by Christian Schabesberger on 28.02.16. * Created by Christian Schabesberger on 28.02.16.
* *
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamInfoItemCollector.java is part of NewPipe. * StreamInfoItemsCollector.java is part of NewPipe.
* *
* NewPipe is free software: you can redistribute it and/or modify * NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@ -28,9 +28,9 @@ import java.util.Vector;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class StreamInfoItemCollector extends InfoItemCollector<StreamInfoItem, StreamInfoItemExtractor> { public class StreamInfoItemsCollector extends InfoItemsCollector<StreamInfoItem, StreamInfoItemExtractor> {
public StreamInfoItemCollector(int serviceId) { public StreamInfoItemsCollector(int serviceId) {
super(serviceId); super(serviceId);
} }

14
src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java
View File

@ -2,11 +2,11 @@ package org.schabi.newpipe.extractor.utils;
import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -14,10 +14,10 @@ import java.util.List;
public class ExtractorHelper { public class ExtractorHelper {
private ExtractorHelper() {} private ExtractorHelper() {}
public static List<InfoItem> getStreamsOrLogError(Info info, ListExtractor extractor) { public static List<InfoItem> getInfoItemsOrLogError(Info info, ListExtractor extractor) {
StreamInfoItemCollector collector; InfoItemsCollector collector;
try { try {
collector = extractor.getStreams(); collector = extractor.getInfoItems();
} catch (Exception e) { } catch (Exception e) {
info.addError(e); info.addError(e);
return Collections.emptyList(); return Collections.emptyList();
@ -28,7 +28,7 @@ public class ExtractorHelper {
public static List<InfoItem> getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { public static List<InfoItem> getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) {
StreamInfoItemCollector collector; StreamInfoItemsCollector collector;
try { try {
collector = extractor.getRelatedVideos(); collector = extractor.getRelatedVideos();
} catch (Exception e) { } catch (Exception e) {
@ -39,7 +39,7 @@ public class ExtractorHelper {
return getInfoItems(info, collector); return getInfoItems(info, collector);
} }
private static List<InfoItem> getInfoItems(Info info, InfoItemCollector collector) { private static List<InfoItem> getInfoItems(Info info, InfoItemsCollector collector) {
List<InfoItem> result; List<InfoItem> result;
try { try {
result = collector.getItemList(); result = collector.getItemList();

8
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View File

@ -61,7 +61,7 @@ public class SoundcloudChannelExtractorTest {
public void testHasMoreStreams() throws Exception { public void testHasMoreStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
assertTrue("don't have more streams", extractor.hasMoreStreams()); assertTrue("don't have more streams", extractor.hasNextPage());
} }
@Test @Test
@ -73,10 +73,10 @@ public class SoundcloudChannelExtractorTest {
public void testGetNextStreams() throws Exception { public void testGetNextStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams(); ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty()); assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty()); assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());
} }
} }

23
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java
View File

@ -4,9 +4,11 @@ import org.junit.BeforeClass;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.util.List;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -39,16 +41,17 @@ public class SoundcloudChartsExtractorTest {
} }
@Test @Test
public void testId() throws Exception { public void testId() {
assertEquals(extractor.getId(), "Top 50"); assertEquals(extractor.getId(), "Top 50");
} }
@Test @Test
public void testGetStreams() throws Exception { public void testGetStreams() throws Exception {
StreamInfoItemCollector collector = extractor.getStreams(); InfoItemsCollector collector = extractor.getInfoItems();
if(!collector.getErrors().isEmpty()) { if(!collector.getErrors().isEmpty()) {
System.err.println("----------"); System.err.println("----------");
for(Throwable e: collector.getErrors()) { List<Throwable> errors = collector.getErrors();
for(Throwable e: errors) {
e.printStackTrace(); e.printStackTrace();
System.err.println("----------"); System.err.println("----------");
} }
@ -60,21 +63,21 @@ public class SoundcloudChartsExtractorTest {
@Test @Test
public void testGetStreamsErrors() throws Exception { public void testGetStreamsErrors() throws Exception {
assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); assertTrue("errors during stream list extraction", extractor.getInfoItems().getErrors().isEmpty());
} }
@Test @Test
public void testHasMoreStreams() throws Exception { public void testHasMoreStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getInfoItems();
assertTrue("has more streams", extractor.hasMoreStreams()); assertTrue("has more streams", extractor.hasNextPage());
} }
@Test @Test
public void testGetNextStreams() throws Exception { public void testGetNextStreams() throws Exception {
extractor.getStreams(); extractor.getInfoItems();
assertFalse("extractor has next streams", extractor.getNextStreams() == null assertFalse("extractor has next streams", extractor.getInfoItemPage() == null
|| extractor.getNextStreams().nextItemsList.isEmpty()); || extractor.getInfoItemPage().infoItemList.isEmpty());
} }
@Test @Test

4
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -81,7 +81,7 @@ public class SoundcloudPlaylistExtractorTest {
public void testHasMoreStreams() throws Exception { public void testHasMoreStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
assertTrue("extractor didn't have more streams", !extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams", !extractor.hasNextPage());
} }
@Test(expected = ExtractionException.class) @Test(expected = ExtractionException.class)
@ -90,7 +90,7 @@ public class SoundcloudPlaylistExtractorTest {
extractor.getStreams(); extractor.getStreams();
// This playlist don't have more streams, it should throw an error // This playlist don't have more streams, it should throw an error
extractor.getNextStreams(); extractor.getInfoItemPage();
fail("Expected exception wasn't thrown"); fail("Expected exception wasn't thrown");
} }

4
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import java.io.IOException; import java.io.IOException;
@ -100,7 +100,7 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test @Test
public void testGetRelatedVideos() throws ExtractionException, IOException { public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemCollector relatedVideos = extractor.getRelatedVideos(); StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos();
assertFalse(relatedVideos.getItemList().isEmpty()); assertFalse(relatedVideos.getItemList().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty());
} }

8
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -100,7 +100,7 @@ public class YoutubeChannelExtractorTest {
public void testHasMoreStreams() throws Exception { public void testHasMoreStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
assertTrue("don't have more streams", extractor.hasMoreStreams()); assertTrue("don't have more streams", extractor.hasNextPage());
} }
@Test @Test
@ -112,9 +112,9 @@ public class YoutubeChannelExtractorTest {
public void testGetNextStreams() throws Exception { public void testGetNextStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams(); ListExtractor.InfoItemPage nextItemsResult = extractor.getInfoItemPage();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty()); assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors); assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors);
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());
} }
} }

12
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -102,18 +102,18 @@ public class YoutubePlaylistExtractorTest {
public void testHasMoreStreams() throws Exception { public void testHasMoreStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
assertTrue("extractor didn't have more streams", extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams", extractor.hasNextPage());
} }
@Test @Ignore @Test @Ignore
public void testGetNextStreams() throws Exception { public void testGetNextPage() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams(); ListExtractor.InfoItemPage infoItemPage = extractor.getInfoItemPage();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty()); assertTrue("extractor didn't have next streams", !infoItemPage.infoItemList.isEmpty());
assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors); assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.errors);
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());
} }
} }

2
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View File

@ -138,7 +138,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetRelatedVideos() throws ExtractionException, IOException { public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemCollector relatedVideos = extractor.getRelatedVideos(); StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos();
Utils.printErrors(relatedVideos); Utils.printErrors(relatedVideos);
assertFalse(relatedVideos.getItemList().isEmpty()); assertFalse(relatedVideos.getItemList().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty());

14
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View File

@ -23,8 +23,8 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import static junit.framework.TestCase.assertFalse; import static junit.framework.TestCase.assertFalse;
@ -66,27 +66,27 @@ public class YoutubeTrendingExtractorTest {
@Test @Test
public void testGetStreamsQuantity() throws Exception { public void testGetStreamsQuantity() throws Exception {
StreamInfoItemCollector collector = extractor.getStreams(); InfoItemsCollector collector = extractor.getInfoItems();
Utils.printErrors(collector); Utils.printErrors(collector);
assertTrue("no streams are received", collector.getItemList().size() >= 20); assertTrue("no streams are received", collector.getItemList().size() >= 20);
} }
@Test @Test
public void testGetStreamsErrors() throws Exception { public void testGetStreamsErrors() throws Exception {
assertEmptyErrors("errors during stream list extraction", extractor.getStreams().getErrors()); assertEmptyErrors("errors during stream list extraction", extractor.getInfoItems().getErrors());
} }
@Test @Test
public void testHasMoreStreams() throws Exception { public void testHasMoreStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getInfoItems();
assertFalse("has more streams", extractor.hasMoreStreams()); assertFalse("has more streams", extractor.hasNextPage());
} }
@Test @Test
public void testGetNextStreams() throws Exception { public void testGetNextStreams() throws Exception {
assertTrue("extractor has next streams", extractor.getNextStreams() == null assertTrue("extractor has next streams", extractor.getInfoItemPage() == null
|| extractor.getNextStreams().getNextItemsList().isEmpty()); || extractor.getInfoItemPage().getNextItemsList().isEmpty());
} }
@Test @Test