From ef2ce6854cef6824d2b780b7817212f1d77d2a7e Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 1 Jun 2018 19:08:49 +0200 Subject: [PATCH] add soundcloud SearchExtractor --- .../soundcloud/SoundcloudSearchExtractor.java | 113 ++++++++++++++++++ ...a => SoundcloudSearchQueryUrlHandler.java} | 7 +- .../soundcloud/SoundcloudService.java | 10 +- .../extractors/YoutubeSearchExtractor.java | 8 +- 4 files changed, 127 insertions(+), 11 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java rename extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/{SoundcloudQueryUrlHandler.java => SoundcloudSearchQueryUrlHandler.java} (90%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java new file mode 100644 index 000000000..50452bc22 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -0,0 +1,113 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.search.SearchQueryUrlHandler; +import org.schabi.newpipe.extractor.utils.Parser; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; + +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryUrlHandler.ITEMS_PER_PAGE; + +public class SoundcloudSearchExtractor extends SearchExtractor { + + private JsonArray searchCollection; + + public SoundcloudSearchExtractor(StreamingService service, + SearchQueryUrlHandler urlIdHandler, + String contentCountry) { + super(service, urlIdHandler, contentCountry); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + return null; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return new InfoItemsPage<>(collectItems(searchCollection), getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return getNextPageUrlFromCurrentUrl(getUrl()); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + final Downloader dl = getDownloader(); + try { + searchCollection = JsonParser.object().from(dl.download(pageUrl)).getArray("collection"); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } + + return new InfoItemsPage<>(collectItems(searchCollection), getNextPageUrlFromCurrentUrl(pageUrl)); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final Downloader dl = getDownloader(); + final String url = getUrl(); + try { + searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection"); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } + + if (searchCollection.size() == 0) { + throw new SearchEngine.NothingFoundException("Nothing found"); + } + } + + private InfoItemsCollector collectItems(JsonArray searchCollection) { + final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + + for (Object result : searchCollection) { + if (!(result instanceof JsonObject)) continue; + //noinspection ConstantConditions + JsonObject searchResult = (JsonObject) result; + String kind = searchResult.getString("kind", ""); + switch (kind) { + case "user": + collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); + break; + case "track": + collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult)); + break; + case "playlist": + collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult)); + break; + } + } + + return collector; + } + + private String getNextPageUrlFromCurrentUrl(String currentUrl) + throws MalformedURLException, UnsupportedEncodingException { + final int pageOffset = Integer.parseInt( + Parser.compatParseMap( + new URL(currentUrl) + .getQuery()) + .get("offset")); + + return currentUrl.replace("&offset=" + + Integer.toString(pageOffset), + "&offset=" + Integer.toString(pageOffset + ITEMS_PER_PAGE)); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudQueryUrlHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryUrlHandler.java similarity index 90% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudQueryUrlHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryUrlHandler.java index 6a99e8655..92d4c6bc8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudQueryUrlHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryUrlHandler.java @@ -8,7 +8,7 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -public class SoundcloudQueryUrlHandler extends SearchQueryUrlHandler { +public class SoundcloudSearchQueryUrlHandler extends SearchQueryUrlHandler { public static final String CHARSET_UTF_8 = "UTF-8"; public static final String TRACKS = "tracks"; @@ -16,6 +16,8 @@ public class SoundcloudQueryUrlHandler extends SearchQueryUrlHandler { public static final String PLAYLIST = "playlist"; public static final String ANY = "any"; + public static final int ITEMS_PER_PAGE = 10; + @Override public String getUrl() throws ParsingException { try { @@ -40,7 +42,8 @@ public class SoundcloudQueryUrlHandler extends SearchQueryUrlHandler { return url + "?q=" + URLEncoder.encode(id, CHARSET_UTF_8) + "&client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=10"; + + "&limit=" + ITEMS_PER_PAGE + + "&offset=0"; } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 036acd1b0..73334a343 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -31,12 +31,12 @@ public class SoundcloudService extends StreamingService { @Override public SearchExtractor getSearchExtractor(SearchQueryUrlHandler queryHandler, String contentCountry) { - return null; + return new SoundcloudSearchExtractor(this, queryHandler, contentCountry); } @Override public SearchQueryUrlHandler getSearchQueryHandler() { - return null; + return new SoundcloudSearchQueryUrlHandler(); } @Override @@ -56,17 +56,17 @@ public class SoundcloudService extends StreamingService { @Override - public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException { + public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) { return new SoundcloudStreamExtractor(this, urlIdHandler); } @Override - public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) { return new SoundcloudChannelExtractor(this, urlIdHandler); } @Override - public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) { return new SoundcloudPlaylistExtractor(this, urlIdHandler); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index fa7cdf950..5d51ed1ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -80,14 +80,14 @@ public class YoutubeSearchExtractor extends SearchExtractor { private String getNextPageUrlFromCurrentUrl(String currentUrl) throws MalformedURLException, UnsupportedEncodingException { - int nextPageNr = Integer.parseInt( + final int pageNr = Integer.parseInt( Parser.compatParseMap( new URL(currentUrl) .getQuery()) - .get("page")) + 1; + .get("page")); - return currentUrl.replace("&page=" + Integer.toString( nextPageNr-1), - "&page=" + Integer.toString(nextPageNr)); + return currentUrl.replace("&page=" + Integer.toString( pageNr), + "&page=" + Integer.toString(pageNr + 1)); } private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException {