diff --git a/build.gradle b/build.gradle index a9197df67..e54215a4b 100644 --- a/build.gradle +++ b/build.gradle @@ -24,4 +24,12 @@ task sourcesJar(type: Jar, dependsOn: classes) { artifacts { archives sourcesJar +} + +tasks.withType(Test) { + testLogging { + events "skipped", "failed" + showStandardStreams = true + exceptionFormat = 'full' + } } \ No newline at end of file diff --git a/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/src/main/java/org/schabi/newpipe/extractor/Extractor.java index 3e0456a94..e71321836 100644 --- a/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -33,12 +33,16 @@ public abstract class Extractor { */ @Nullable private String cleanUrl; + private boolean pageFetched = false; + private final Downloader downloader; public Extractor(StreamingService service, String url) throws ExtractionException { if(service == null) throw new NullPointerException("service is null"); if(url == null) throw new NullPointerException("url is null"); this.service = service; this.originalUrl = url; + this.downloader = NewPipe.getDownloader(); + if(downloader == null) throw new NullPointerException("downloader is null"); } /** @@ -49,8 +53,26 @@ public abstract class Extractor { /** * Fetch the current page. + * @throws IOException if the page can not be loaded + * @throws ExtractionException if the pages content is not understood */ - public abstract void fetchPage() throws IOException, ExtractionException; + public void fetchPage() throws IOException, ExtractionException { + if(pageFetched) return; + onFetchPage(downloader); + pageFetched = true; + } + + protected void assertPageFetched() { + if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()"); + } + + /** + * Fetch the current page. + * @param downloader the download to use + * @throws IOException if the page can not be loaded + * @throws ExtractionException if the pages content is not understood + */ + public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException; @Nonnull public abstract String getId() throws ParsingException; diff --git a/src/main/java/org/schabi/newpipe/extractor/Info.java b/src/main/java/org/schabi/newpipe/extractor/Info.java index c2445bd61..8e9fc79d0 100644 --- a/src/main/java/org/schabi/newpipe/extractor/Info.java +++ b/src/main/java/org/schabi/newpipe/extractor/Info.java @@ -16,7 +16,7 @@ public abstract class Info implements Serializable { public final String url; public final String name; - public List errors = new ArrayList<>(); + public final List errors = new ArrayList<>(); public void addError(Throwable throwable) { this.errors.add(throwable); diff --git a/src/main/java/org/schabi/newpipe/extractor/Subtitles.java b/src/main/java/org/schabi/newpipe/extractor/Subtitles.java index 0542b385e..9427bf528 100644 --- a/src/main/java/org/schabi/newpipe/extractor/Subtitles.java +++ b/src/main/java/org/schabi/newpipe/extractor/Subtitles.java @@ -3,9 +3,10 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.stream.SubtitlesFormat; public class Subtitles { - private SubtitlesFormat format; - private String languageCode, URL; - private boolean autoGenerated; + private final SubtitlesFormat format; + private final String languageCode; + private final String URL; + private final boolean autoGenerated; public Subtitles(SubtitlesFormat format, String languageCode, String URL, boolean autoGenerated) { this.format = format; diff --git a/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java b/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java index 983adc07e..d7c9cdca7 100644 --- a/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java @@ -27,7 +27,7 @@ import java.util.List; public abstract class SuggestionExtractor { - private int serviceId; + private final int serviceId; public SuggestionExtractor(int serviceId) { this.serviceId = serviceId; diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 5df579e7a..923f5d033 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -30,7 +30,7 @@ import java.io.IOException; public abstract class KioskExtractor extends ListExtractor { private String contentCountry = null; - private String id = null; + private final String id; public KioskExtractor(StreamingService streamingService, String url, diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index bbaab0092..5256af8e0 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -19,8 +19,8 @@ public class KioskList { throws ExtractionException, IOException; } - private int service_id; - private HashMap kioskList = new HashMap<>(); + private final int service_id; + private final HashMap kioskList = new HashMap<>(); private String defaultKiosk = null; private class KioskEntry { @@ -28,8 +28,8 @@ public class KioskList { extractorFactory = ef; handler = h; } - KioskExtractorFactory extractorFactory; - UrlIdHandler handler; + final KioskExtractorFactory extractorFactory; + final UrlIdHandler handler; } public KioskList(int service_id) { diff --git a/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java b/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java index b9f9fa393..878888324 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java @@ -35,7 +35,7 @@ public abstract class SearchEngine { } } - private InfoItemSearchCollector collector; + private final InfoItemSearchCollector collector; public SearchEngine(int serviceId) { collector = new InfoItemSearchCollector(serviceId); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 845d77217..e81fe2cb1 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -24,14 +24,13 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader dl = NewPipe.getDownloader(); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { userId = getUrlIdHandler().getId(getOriginalUrl()); String apiUrl = "https://api.soundcloud.com/users/" + userId + "?client_id=" + SoundcloudParsingHelper.clientId(); - String response = dl.download(apiUrl); + String response = downloader.download(apiUrl); try { user = JsonParser.object().from(response); } catch (JsonParserException e) { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java index 773e8e405..532c059b8 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -4,7 +4,7 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { - private JsonObject searchResult; + private final JsonObject searchResult; public SoundcloudChannelInfoItemExtractor(JsonObject searchResult) { this.searchResult = searchResult; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index 24e51b1ac..ac5f933bc 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -23,7 +24,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } @Override - public void fetchPage() { + public void onFetchPage(@Nonnull Downloader downloader) { } @Nonnull diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index d2381bc5f..f68fff07a 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -67,13 +67,13 @@ public class SoundcloudParsingHelper { * * See https://developers.soundcloud.com/docs/api/reference#resolve */ - public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException { + public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException { String apiUrl = "https://api.soundcloud.com/resolve" + "?url=" + URLEncoder.encode(url, "UTF-8") + "&client_id=" + clientId(); try { - return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl)); + return JsonParser.object().from(downloader.download(apiUrl)); } catch (JsonParserException e) { throw new ParsingException("Could not parse json response", e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 3f51be7c3..35da76425 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -24,15 +24,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader dl = NewPipe.getDownloader(); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { playlistId = getUrlIdHandler().getId(getOriginalUrl()); String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId + "?client_id=" + SoundcloudParsingHelper.clientId() + "&representation=compact"; - String response = dl.download(apiUrl); + String response = downloader.download(apiUrl); try { playlist = JsonParser.object().from(response); } catch (JsonParserException e) { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java index 6002f054e..276f2c6a5 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java @@ -9,7 +9,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr private static final String AVATAR_URL_KEY = "avatar_url"; private static final String ARTWORK_URL_KEY = "artwork_url"; - private JsonObject searchResult; + private final JsonObject searchResult; public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) { this.searchResult = searchResult; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index d323a262c..ca02b7a0c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -21,12 +21,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { super(service, url); - fetchPage(); } @Override - public void fetchPage() throws IOException, ExtractionException { - track = SoundcloudParsingHelper.resolveFor(getOriginalUrl()); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl()); String policy = track.getString("policy", ""); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java index 9d9663de3..1950c0b5e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java @@ -146,8 +146,8 @@ public class ItagItem { return mediaFormat; } - public int id; - public ItagType itagType; + public final int id; + public final ItagType itagType; // Audio fields public int avgBitrate = -1; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index 054e6d798..d1b9ae165 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -59,9 +59,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader downloader = NewPipe.getDownloader(); - + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS; String pageContent = downloader.download(channelUrl); doc = Jsoup.parse(pageContent, channelUrl); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java index d6d5d4620..a04d85622 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java @@ -26,7 +26,7 @@ import org.schabi.newpipe.extractor.utils.Utils; */ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { - private Element el; + private final Element el; public YoutubeChannelInfoItemExtractor(Element el) { this.el = el; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index 02619fc6e..c6e5c064d 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -36,9 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader downloader = NewPipe.getDownloader(); - + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { String pageContent = downloader.download(getCleanUrl()); doc = Jsoup.parse(pageContent, getCleanUrl()); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java index 552f58f41..fe6e7768f 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.utils.Utils; public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { - private Element el; + private final Element el; public YoutubePlaylistInfoItemExtractor(Element el) { this.el = el; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 515af7483..66e72ef29 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -86,7 +86,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { super(service, url); - fetchPage(); } /*////////////////////////////////////////////////////////////////////////// @@ -106,6 +105,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getName() throws ParsingException { + assertPageFetched(); String name = getStringFromMetaData("title"); if(name == null) { // Fallback to HTML method @@ -124,6 +124,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploadDate() throws ParsingException { + assertPageFetched(); try { return doc.select("meta[itemprop=datePublished]").attr(CONTENT); } catch (Exception e) {//todo: add fallback method @@ -134,6 +135,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getThumbnailUrl() throws ParsingException { + assertPageFetched(); // Try to get high resolution thumbnail first, if it fails, use low res from the player instead try { return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); @@ -157,6 +159,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getDescription() throws ParsingException { + assertPageFetched(); try { return doc.select("p[id=\"eow-description\"]").first().html(); } catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know @@ -166,6 +169,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() throws ParsingException { + assertPageFetched(); if (!isAgeRestricted) { return NO_AGE_LIMIT; } @@ -179,6 +183,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getLength() throws ParsingException { + assertPageFetched(); if(playerArgs != null) { try { long returnValue = Long.parseLong(playerArgs.get("length_seconds") + ""); @@ -217,6 +222,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getViewCount() throws ParsingException { + assertPageFetched(); try { return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT)); } catch (Exception e) {//todo: find fallback method @@ -226,6 +232,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getLikeCount() throws ParsingException { + assertPageFetched(); String likesString = ""; try { Element button = doc.select("button.like-button-renderer-like-button").first(); @@ -245,6 +252,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getDislikeCount() throws ParsingException { + assertPageFetched(); String dislikesString = ""; try { Element button = doc.select("button.like-button-renderer-dislike-button").first(); @@ -265,6 +273,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploaderUrl() throws ParsingException { + assertPageFetched(); try { return doc.select("div[class=\"yt-user-info\"]").first().children() .select("a").first().attr("abs:href"); @@ -276,6 +285,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable private String getStringFromMetaData(String field) { + assertPageFetched(); String value = null; if(playerArgs != null) { // This can not fail @@ -291,6 +301,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploaderName() throws ParsingException { + assertPageFetched(); String name = getStringFromMetaData("author"); if(name == null) { @@ -310,6 +321,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploaderAvatarUrl() throws ParsingException { + assertPageFetched(); try { return doc.select("a[class*=\"yt-user-photo\"]").first() .select("img").first() @@ -321,6 +333,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getDashMpdUrl() throws ParsingException { + assertPageFetched(); try { String dashManifestUrl; if (videoInfoPage.containsKey("dashmpd")) { @@ -347,6 +360,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public List getAudioStreams() throws IOException, ExtractionException { + assertPageFetched(); List audioStreams = new ArrayList<>(); try { for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) { @@ -366,6 +380,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public List getVideoStreams() throws IOException, ExtractionException { + assertPageFetched(); List videoStreams = new ArrayList<>(); try { for (Map.Entry entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) { @@ -385,6 +400,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public List getVideoOnlyStreams() throws IOException, ExtractionException { + assertPageFetched(); List videoOnlyStreams = new ArrayList<>(); try { for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { @@ -411,11 +427,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override @Nullable public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { + assertPageFetched(); if(isAgeRestricted) { // If the video is age restricted getPlayerConfig will fail return null; } - JsonObject playerConfig = getPlayerConfig(getPageHtml()); + // TODO: This should be done in onFetchPage() + JsonObject playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader())); String playerResponse = playerConfig.getObject("args").getString("player_response"); JsonObject captions; @@ -459,6 +477,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItem getNextVideo() throws IOException, ExtractionException { + assertPageFetched(); try { StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); collector.commit(extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]") @@ -472,6 +491,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException { + assertPageFetched(); try { StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); Element ul = doc.select("ul[id=\"watch-related\"]").first(); @@ -526,30 +546,27 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String GET_VIDEO_INFO_URL = "https://www.youtube.com/get_video_info?video_id=" + "%s" + "&el=info&ps=default&eurl=&gl=US&hl=en"; - private static volatile String decryptionCode = ""; + private volatile String decryptionCode = ""; - private static String pageHtml = null; + private String pageHtml = null; - private String getPageHtml() throws IOException, ExtractionException{ + private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{ if (pageHtml == null) { - Downloader dl = NewPipe.getDownloader(); - pageHtml = dl.download(getCleanUrl()); + pageHtml = downloader.download(getCleanUrl()); } return pageHtml; } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader dl = NewPipe.getDownloader(); - - String pageContent = getPageHtml(); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + String pageContent = getPageHtml(downloader); doc = Jsoup.parse(pageContent, getCleanUrl()); String playerUrl; // Check if the video is age restricted if (pageContent.contains("