mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2025-01-06 01:20:32 +05:30
Store errors that can happen during extraction of the next page
- Closes #24
This commit is contained in:
parent
82824cdd72
commit
9184fc509c
@ -69,9 +69,19 @@ public abstract class ListExtractor extends Extractor {
|
|||||||
*/
|
*/
|
||||||
public final String nextItemsUrl;
|
public final String nextItemsUrl;
|
||||||
|
|
||||||
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl) {
|
/**
|
||||||
|
* Errors that happened during the extraction
|
||||||
|
*/
|
||||||
|
public final List<Throwable> errors;
|
||||||
|
|
||||||
|
public NextItemsResult(InfoItemCollector collector, String nextItemsUrl) {
|
||||||
|
this(collector.getItemList(), nextItemsUrl, collector.getErrors());
|
||||||
|
}
|
||||||
|
|
||||||
|
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl, List<Throwable> errors) {
|
||||||
this.nextItemsList = nextItemsList;
|
this.nextItemsList = nextItemsList;
|
||||||
this.nextItemsUrl = nextItemsUrl;
|
this.nextItemsUrl = nextItemsUrl;
|
||||||
|
this.errors = errors;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean hasMoreStreams() {
|
public boolean hasMoreStreams() {
|
||||||
|
@ -102,6 +102,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
|
|||||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
|
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
|
||||||
|
|
||||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
return new NextItemsResult(collector, nextStreamsUrl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -104,6 +104,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
|||||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||||
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
|
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
|
||||||
|
|
||||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
return new NextItemsResult(collector, nextStreamsUrl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -153,7 +153,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||||||
setupNextStreamsAjax(NewPipe.getDownloader());
|
setupNextStreamsAjax(NewPipe.getDownloader());
|
||||||
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
|
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
|
||||||
|
|
||||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
return new NextItemsResult(collector, nextStreamsUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
|
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
|
||||||
|
@ -157,7 +157,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||||||
setupNextStreamsAjax(NewPipe.getDownloader());
|
setupNextStreamsAjax(NewPipe.getDownloader());
|
||||||
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
|
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
|
||||||
|
|
||||||
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
|
return new NextItemsResult(collector, nextStreamsUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
|
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
|
||||||
|
@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
|
|||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.schabi.newpipe.Downloader;
|
import org.schabi.newpipe.Downloader;
|
||||||
|
import org.schabi.newpipe.extractor.ListExtractor;
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||||
|
|
||||||
@ -70,7 +71,9 @@ public class SoundcloudChannelExtractorTest {
|
|||||||
public void testGetNextStreams() throws Exception {
|
public void testGetNextStreams() throws Exception {
|
||||||
// Setup the streams
|
// Setup the streams
|
||||||
extractor.getStreams();
|
extractor.getStreams();
|
||||||
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
|
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
|
||||||
|
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
|
||||||
|
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
|
||||||
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
|
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -4,6 +4,7 @@ import org.junit.Before;
|
|||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.schabi.newpipe.Downloader;
|
import org.schabi.newpipe.Downloader;
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||||
|
|
||||||
import static org.junit.Assert.*;
|
import static org.junit.Assert.*;
|
||||||
@ -79,4 +80,15 @@ public class SoundcloudPlaylistExtractorTest {
|
|||||||
extractor.getStreams();
|
extractor.getStreams();
|
||||||
assertTrue("extractor didn't have more streams", !extractor.hasMoreStreams());
|
assertTrue("extractor didn't have more streams", !extractor.hasMoreStreams());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test(expected = ExtractionException.class)
|
||||||
|
public void testGetNextStreamsNonExistent() throws Exception {
|
||||||
|
// Setup the streams
|
||||||
|
extractor.getStreams();
|
||||||
|
|
||||||
|
// This playlist don't have more streams, it should throw an error
|
||||||
|
extractor.getNextStreams();
|
||||||
|
|
||||||
|
fail("Expected exception wasn't thrown");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
|
|||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.schabi.newpipe.Downloader;
|
import org.schabi.newpipe.Downloader;
|
||||||
|
import org.schabi.newpipe.extractor.ListExtractor;
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||||
|
|
||||||
@ -99,7 +100,9 @@ public class YoutubeChannelExtractorTest {
|
|||||||
public void testGetNextStreams() throws Exception {
|
public void testGetNextStreams() throws Exception {
|
||||||
// Setup the streams
|
// Setup the streams
|
||||||
extractor.getStreams();
|
extractor.getStreams();
|
||||||
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
|
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
|
||||||
|
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
|
||||||
|
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
|
||||||
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
|
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
|
|||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.schabi.newpipe.Downloader;
|
import org.schabi.newpipe.Downloader;
|
||||||
|
import org.schabi.newpipe.extractor.ListExtractor;
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||||
|
|
||||||
@ -89,7 +90,9 @@ public class YoutubePlaylistExtractorTest {
|
|||||||
public void testGetNextStreams() throws Exception {
|
public void testGetNextStreams() throws Exception {
|
||||||
// Setup the streams
|
// Setup the streams
|
||||||
extractor.getStreams();
|
extractor.getStreams();
|
||||||
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
|
ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
|
||||||
|
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
|
||||||
|
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
|
||||||
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
|
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user