diff --git a/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/src/main/java/org/schabi/newpipe/extractor/Extractor.java
index 3e0456a94..e71321836 100644
--- a/src/main/java/org/schabi/newpipe/extractor/Extractor.java
+++ b/src/main/java/org/schabi/newpipe/extractor/Extractor.java
@@ -33,12 +33,16 @@ public abstract class Extractor {
*/
@Nullable
private String cleanUrl;
+ private boolean pageFetched = false;
+ private final Downloader downloader;
public Extractor(StreamingService service, String url) throws ExtractionException {
if(service == null) throw new NullPointerException("service is null");
if(url == null) throw new NullPointerException("url is null");
this.service = service;
this.originalUrl = url;
+ this.downloader = NewPipe.getDownloader();
+ if(downloader == null) throw new NullPointerException("downloader is null");
}
/**
@@ -49,8 +53,26 @@ public abstract class Extractor {
/**
* Fetch the current page.
+ * @throws IOException if the page can not be loaded
+ * @throws ExtractionException if the pages content is not understood
*/
- public abstract void fetchPage() throws IOException, ExtractionException;
+ public void fetchPage() throws IOException, ExtractionException {
+ if(pageFetched) return;
+ onFetchPage(downloader);
+ pageFetched = true;
+ }
+
+ protected void assertPageFetched() {
+ if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()");
+ }
+
+ /**
+ * Fetch the current page.
+ * @param downloader the download to use
+ * @throws IOException if the page can not be loaded
+ * @throws ExtractionException if the pages content is not understood
+ */
+ public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
@Nonnull
public abstract String getId() throws ParsingException;
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
index 845d77217..e81fe2cb1 100644
--- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
+++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
@@ -24,14 +24,13 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
}
@Override
- public void fetchPage() throws IOException, ExtractionException {
- Downloader dl = NewPipe.getDownloader();
+ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
userId = getUrlIdHandler().getId(getOriginalUrl());
String apiUrl = "https://api.soundcloud.com/users/" + userId +
"?client_id=" + SoundcloudParsingHelper.clientId();
- String response = dl.download(apiUrl);
+ String response = downloader.download(apiUrl);
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
index 24e51b1ac..ac5f933bc 100644
--- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
+++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -23,7 +24,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
}
@Override
- public void fetchPage() {
+ public void onFetchPage(@Nonnull Downloader downloader) {
}
@Nonnull
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
index 770305376..2c766d015 100644
--- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
+++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
@@ -66,13 +66,13 @@ public class SoundcloudParsingHelper {
* Call the endpoint "/resolve" of the api.
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
- public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
+ public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
String apiUrl = "https://api.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId();
try {
- return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl));
+ return JsonParser.object().from(downloader.download(apiUrl));
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
index 3f51be7c3..35da76425 100644
--- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
+++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
@@ -24,15 +24,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
}
@Override
- public void fetchPage() throws IOException, ExtractionException {
- Downloader dl = NewPipe.getDownloader();
+ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
playlistId = getUrlIdHandler().getId(getOriginalUrl());
String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
- String response = dl.download(apiUrl);
+ String response = downloader.download(apiUrl);
try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
index d323a262c..ca02b7a0c 100644
--- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
+++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
@@ -21,12 +21,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url);
- fetchPage();
}
@Override
- public void fetchPage() throws IOException, ExtractionException {
- track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
+ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+ track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
String policy = track.getString("policy", "");
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
index 054e6d798..d1b9ae165 100644
--- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
+++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
@@ -59,9 +59,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
- public void fetchPage() throws IOException, ExtractionException {
- Downloader downloader = NewPipe.getDownloader();
-
+ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS;
String pageContent = downloader.download(channelUrl);
doc = Jsoup.parse(pageContent, channelUrl);
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
index 02619fc6e..c6e5c064d 100644
--- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
+++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
@@ -36,9 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
- public void fetchPage() throws IOException, ExtractionException {
- Downloader downloader = NewPipe.getDownloader();
-
+ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getCleanUrl());
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
index 515af7483..33c94972a 100644
--- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
+++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
@@ -86,7 +86,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url);
- fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
@@ -415,7 +414,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// If the video is age restricted getPlayerConfig will fail
return null;
}
- JsonObject playerConfig = getPlayerConfig(getPageHtml());
+ // TODO: This should be done in onFetchPage()
+ JsonObject playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
String playerResponse = playerConfig.getObject("args").getString("player_response");
JsonObject captions;
@@ -530,26 +530,23 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static String pageHtml = null;
- private String getPageHtml() throws IOException, ExtractionException{
+ private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{
if (pageHtml == null) {
- Downloader dl = NewPipe.getDownloader();
- pageHtml = dl.download(getCleanUrl());
+ pageHtml = downloader.download(getCleanUrl());
}
return pageHtml;
}
@Override
- public void fetchPage() throws IOException, ExtractionException {
- Downloader dl = NewPipe.getDownloader();
-
- String pageContent = getPageHtml();
+ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+ String pageContent = getPageHtml(downloader);
doc = Jsoup.parse(pageContent, getCleanUrl());
String playerUrl;
// Check if the video is age restricted
if (pageContent.contains("