diff --git a/NewPipeExtractor.iml b/NewPipeExtractor.iml new file mode 100644 index 000000000..fa027b002 --- /dev/null +++ b/NewPipeExtractor.iml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/build.gradle b/build.gradle index 1138e8e57..8c9923265 100644 --- a/build.gradle +++ b/build.gradle @@ -1,18 +1,20 @@ apply plugin: 'java-library' +allprojects { + sourceCompatibility = 1.7 + targetCompatibility = 1.7 +} + repositories { jcenter() } dependencies { - implementation 'com.github.openjson:openjson:1.0.8' + implementation 'com.grack:nanojson:1.1' implementation 'org.jsoup:jsoup:1.9.2' implementation 'org.mozilla:rhino:1.7.7.1' testImplementation 'junit:junit:4.12' - - sourceCompatibility = 1.7 - targetCompatibility = 1.7 } task sourcesJar(type: Jar, dependsOn: classes) { diff --git a/settings.gradle b/settings.gradle index 9a1b299d4..f897ec98f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1,2 @@ +include ':' rootProject.name = 'NewPipeExtractor' diff --git a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index d4746270b..584bb4283 100644 --- a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -69,9 +69,19 @@ public abstract class ListExtractor extends Extractor { */ public final String nextItemsUrl; - public NextItemsResult(List nextItemsList, String nextItemsUrl) { + /** + * Errors that happened during the extraction + */ + public final List errors; + + public NextItemsResult(InfoItemCollector collector, String nextItemsUrl) { + this(collector.getItemList(), nextItemsUrl, collector.getErrors()); + } + + public NextItemsResult(List nextItemsList, String nextItemsUrl, List errors) { this.nextItemsList = nextItemsList; this.nextItemsUrl = nextItemsUrl; + this.errors = errors; } public boolean hasMoreStreams() { diff --git a/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java b/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java index 5c5eff884..8507f0ac8 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java @@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; @@ -32,6 +34,7 @@ public class InfoItemSearchCollector extends InfoItemCollector { private String suggestion = ""; private StreamInfoItemCollector streamCollector; private ChannelInfoItemCollector userCollector; + private PlaylistInfoItemCollector playlistCollector; private SearchResult result = new SearchResult(); @@ -39,6 +42,7 @@ public class InfoItemSearchCollector extends InfoItemCollector { super(serviceId); streamCollector = new StreamInfoItemCollector(serviceId); userCollector = new ChannelInfoItemCollector(serviceId); + playlistCollector = new PlaylistInfoItemCollector(serviceId); } public void setSuggestion(String suggestion) { @@ -49,6 +53,7 @@ public class InfoItemSearchCollector extends InfoItemCollector { addFromCollector(userCollector); addFromCollector(streamCollector); + addFromCollector(playlistCollector); result.suggestion = suggestion; result.errors = getErrors(); @@ -74,4 +79,14 @@ public class InfoItemSearchCollector extends InfoItemCollector { addError(e); } } + + public void commit(PlaylistInfoItemExtractor extractor) { + try { + result.resultList.add(playlistCollector.extract(extractor)); + } catch (FoundAdException ae) { + System.err.println("Found ad"); + } catch (Exception e) { + addError(e); + } + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java b/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java index 83308136b..b9f9fa393 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.search; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import java.io.IOException; -import java.util.EnumSet; /* * Created by Christian Schabesberger on 10.08.15. @@ -27,7 +26,7 @@ import java.util.EnumSet; public abstract class SearchEngine { public enum Filter { - STREAM, CHANNEL, PLAYLIST + ANY, STREAM, CHANNEL, PLAYLIST } public static class NothingFoundException extends ExtractionException { @@ -46,8 +45,6 @@ public abstract class SearchEngine { return collector; } - //Result search(String query, int page); - public abstract InfoItemSearchCollector search( - String query, int page, String contentCountry, EnumSet filter) + public abstract InfoItemSearchCollector search(String query, int page, String contentCountry, Filter filter) throws IOException, ExtractionException; } diff --git a/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java b/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java index 11c9850ba..4a6a50fdf 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import java.io.IOException; import java.util.ArrayList; -import java.util.EnumSet; import java.util.List; /* @@ -29,8 +28,7 @@ import java.util.List; */ public class SearchResult { - public static SearchResult getSearchResult(SearchEngine engine, String query, - int page, String languageCode, EnumSet filter) + public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter) throws IOException, ExtractionException { SearchResult result = engine diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 42c1bd141..aa4fe48f6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -14,7 +16,7 @@ import java.io.IOException; @SuppressWarnings("WeakerAccess") public class SoundcloudChannelExtractor extends ChannelExtractor { private String userId; - private JSONObject user; + private JsonObject user; public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { super(service, url, nextStreamsUrl); @@ -29,16 +31,16 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { "?client_id=" + SoundcloudParsingHelper.clientId(); String response = dl.download(apiUrl); - user = new JSONObject(response); + try { + user = JsonParser.object().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } } @Override public String getCleanUrl() { - try { - return user.getString("permalink_url"); - } catch (Exception e) { - return getOriginalUrl(); - } + return user.isString("permalink_url") ? user.getString("permalink_url") : getOriginalUrl(); } @Override @@ -53,16 +55,12 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() { - return user.optString("avatar_url"); + return user.getString("avatar_url"); } @Override public String getBannerUrl() throws ParsingException { - try { - return user.getJSONObject("visuals").getJSONArray("visuals").getJSONObject(0).getString("visual_url"); - } catch (Exception e) { - throw new ParsingException("Could not get Banner", e); - } + return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url", ""); } @Override @@ -72,12 +70,12 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() { - return user.optLong("followers_count"); + return user.getNumber("followers_count", 0).longValue(); } @Override public String getDescription() throws ParsingException { - return user.optString("description"); + return user.getString("description", ""); } @Override @@ -102,6 +100,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl); - return new NextItemsResult(collector.getItemList(), nextStreamsUrl); + return new NextItemsResult(collector, nextStreamsUrl); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java index ea7df69cc..773e8e405 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { - private JSONObject searchResult; + private JsonObject searchResult; - public SoundcloudChannelInfoItemExtractor(JSONObject searchResult) { + public SoundcloudChannelInfoItemExtractor(JsonObject searchResult) { this.searchResult = searchResult; } @@ -22,21 +22,21 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac @Override public String getThumbnailUrl() { - return searchResult.optString("avatar_url"); + return searchResult.getString("avatar_url", ""); } @Override public long getSubscriberCount() { - return searchResult.optLong("followers_count"); + return searchResult.getNumber("followers_count", 0).longValue(); } @Override public long getStreamCount() { - return searchResult.optLong("track_count"); + return searchResult.getNumber("track_count", 0).longValue(); } @Override public String getDescription() { - return searchResult.optString("description"); + return searchResult.getString("description", ""); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index f4aca4e2d..e7b65acaf 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -40,7 +40,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true); - return new NextItemsResult(collector.getItemList(), nextStreamsUrl); + return new NextItemsResult(collector, nextStreamsUrl); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index fd20ada45..29ffb7ec6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -1,7 +1,9 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import com.github.openjson.JSONArray; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -65,12 +67,16 @@ public class SoundcloudParsingHelper { * Call the endpoint "/resolve" of the api.
* See https://developers.soundcloud.com/docs/api/reference#resolve */ - public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException { + public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException { String apiUrl = "https://api.soundcloud.com/resolve" + "?url=" + URLEncoder.encode(url, "UTF-8") + "&client_id=" + clientId(); - return new JSONObject(NewPipe.getDownloader().download(apiUrl)); + try { + return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl)); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } } /** @@ -123,15 +129,16 @@ public class SoundcloudParsingHelper { */ public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException { String response = NewPipe.getDownloader().download(apiUrl); - JSONObject responseObject = new JSONObject(response); + JsonObject responseObject; + try { + responseObject = JsonParser.object().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } - JSONArray responseCollection = responseObject.getJSONArray("collection"); - for (int i = 0; i < responseCollection.length(); i++) { - if (charts) { - collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i).getJSONObject("track"))); - } else { - collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i))); - } + JsonArray responseCollection = responseObject.getArray("collection"); + for (Object o : responseCollection) { + if (o instanceof JsonObject) collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) o)); } String nextStreamsUrl; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 7be086fa1..b21e585be 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -1,10 +1,13 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; @@ -13,7 +16,7 @@ import java.io.IOException; @SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private String playlistId; - private JSONObject playlist; + private JsonObject playlist; public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { super(service, url, nextStreamsUrl); @@ -29,16 +32,16 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { "&representation=compact"; String response = dl.download(apiUrl); - playlist = new JSONObject(response); + try { + playlist = JsonParser.object().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } } @Override public String getCleanUrl() { - try { - return playlist.getString("permalink_url"); - } catch (Exception e) { - return getOriginalUrl(); - } + return playlist.isString("permalink_url") ? playlist.getString("permalink_url") : getOriginalUrl(); } @Override @@ -48,12 +51,12 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public String getName() { - return playlist.optString("title"); + return playlist.getString("title"); } @Override public String getThumbnailUrl() { - return playlist.optString("artwork_url"); + return playlist.getString("artwork_url"); } @Override @@ -63,22 +66,22 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() { - return playlist.getJSONObject("user").getString("permalink_url"); + return playlist.getObject("user").getString("permalink_url", ""); } @Override public String getUploaderName() { - return playlist.getJSONObject("user").getString("username"); + return playlist.getObject("user").getString("username", ""); } @Override public String getUploaderAvatarUrl() { - return playlist.getJSONObject("user").getString("avatar_url"); + return playlist.getObject("user", new JsonObject()).getString("avatar_url", ""); } @Override public long getStreamCount() { - return playlist.getLong("track_count"); + return playlist.getNumber("track_count", 0).longValue(); } @Override @@ -104,6 +107,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl); - return new NextItemsResult(collector.getItemList(), nextStreamsUrl); + return new NextItemsResult(collector, nextStreamsUrl); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java new file mode 100644 index 000000000..6002f054e --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java @@ -0,0 +1,77 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; + +public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + private static final String USER_KEY = "user"; + private static final String AVATAR_URL_KEY = "avatar_url"; + private static final String ARTWORK_URL_KEY = "artwork_url"; + + private JsonObject searchResult; + + public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) { + this.searchResult = searchResult; + } + + @Override + public String getName() throws ParsingException { + return searchResult.getString("title"); + } + + @Override + public String getUrl() throws ParsingException { + return searchResult.getString("permalink_url"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + // Over-engineering at its finest + if (searchResult.isString(ARTWORK_URL_KEY)) { + final String artworkUrl = searchResult.getString(ARTWORK_URL_KEY, ""); + if (!artworkUrl.isEmpty()) return artworkUrl; + } + + try { + // Look for artwork url inside the track list + for (Object track : searchResult.getArray("tracks")) { + final JsonObject trackObject = (JsonObject) track; + + // First look for track artwork url + if (trackObject.isString(ARTWORK_URL_KEY)) { + final String url = trackObject.getString(ARTWORK_URL_KEY, ""); + if (!url.isEmpty()) return url; + } + + // Then look for track creator avatar url + final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject()); + final String creatorAvatar = creator.getString(AVATAR_URL_KEY, ""); + if (!creatorAvatar.isEmpty()) return creatorAvatar; + } + } catch (Exception ignored) { + // Try other method + } + + try { + // Last resort, use user avatar url. If still not found, then throw exception. + return searchResult.getObject(USER_KEY).getString(AVATAR_URL_KEY, ""); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist thumbnail url", e); + } + } + + @Override + public String getUploaderName() throws ParsingException { + try { + return searchResult.getObject(USER_KEY).getString("username"); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist uploader", e); + } + } + + @Override + public long getStreamCount() throws ParsingException { + return searchResult.getNumber("track_count", 0).longValue(); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java index 2197b2551..b271d3885 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java @@ -1,16 +1,18 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import com.github.openjson.JSONArray; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; import org.schabi.newpipe.extractor.search.SearchEngine; import java.io.IOException; import java.net.URLEncoder; -import java.util.EnumSet; public class SoundcloudSearchEngine extends SearchEngine { public static final String CHARSET_UTF_8 = "UTF-8"; @@ -20,17 +22,27 @@ public class SoundcloudSearchEngine extends SearchEngine { } @Override - public InfoItemSearchCollector search(String query, int page, String languageCode, EnumSet filter) throws IOException, ExtractionException { + public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException { InfoItemSearchCollector collector = getInfoItemSearchCollector(); - Downloader downloader = NewPipe.getDownloader(); + Downloader dl = NewPipe.getDownloader(); String url = "https://api-v2.soundcloud.com/search"; - if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) { - url += "/tracks"; - } else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) { - url += "/users"; + switch (filter) { + case STREAM: + url += "/tracks"; + break; + case CHANNEL: + url += "/users"; + break; + case PLAYLIST: + url += "/playlists"; + break; + case ANY: + // Don't append any parameter to search for everything + default: + break; } url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) @@ -38,21 +50,32 @@ public class SoundcloudSearchEngine extends SearchEngine { + "&limit=10" + "&offset=" + Integer.toString(page * 10); - String searchJson = downloader.download(url); - JSONObject search = new JSONObject(searchJson); - JSONArray searchCollection = search.getJSONArray("collection"); + JsonArray searchCollection; + try { + searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection"); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } - if (searchCollection.length() == 0) { + if (searchCollection.size() == 0) { throw new NothingFoundException("Nothing found"); } - for (int i = 0; i < searchCollection.length(); i++) { - JSONObject searchResult = searchCollection.getJSONObject(i); - String kind = searchResult.getString("kind"); - if (kind.equals("user")) { - collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); - } else if (kind.equals("track")) { - collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult)); + for (Object result : searchCollection) { + if (!(result instanceof JsonObject)) continue; + //noinspection ConstantConditions + JsonObject searchResult = (JsonObject) result; + String kind = searchResult.getString("kind", ""); + switch (kind) { + case "user": + collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); + break; + case "track": + collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult)); + break; + case "playlist": + collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult)); + break; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index ab7fe3247..8a8933786 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; @@ -16,7 +18,7 @@ import java.util.ArrayList; import java.util.List; public class SoundcloudStreamExtractor extends StreamExtractor { - private JSONObject track; + private JsonObject track; public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { super(service, url); @@ -26,7 +28,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public void fetchPage() throws IOException, ExtractionException { track = SoundcloudParsingHelper.resolveFor(getOriginalUrl()); - String policy = track.getString("policy"); + String policy = track.getString("policy", ""); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { throw new ContentNotAvailableException("Content not available: policy " + policy); } @@ -34,11 +36,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public String getCleanUrl() { - try { - return track.getString("permalink_url"); - } catch (Exception e) { - return getOriginalUrl(); - } + return track.isString("permalink_url") ? track.getString("permalink_url") : getOriginalUrl(); } @Override @@ -48,7 +46,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public String getName() { - return track.optString("title"); + return track.getString("title"); } @Override @@ -58,12 +56,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public String getThumbnailUrl() { - return track.optString("artwork_url"); + return track.getString("artwork_url", ""); } @Override public String getDescription() { - return track.optString("description"); + return track.getString("description"); } @Override @@ -73,7 +71,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public long getLength() { - return track.getLong("duration") / 1000L; + return track.getNumber("duration", 0).longValue() / 1000L; } @Override @@ -125,12 +123,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public long getViewCount() { - return track.getLong("playback_count"); + return track.getNumber("playback_count", 0).longValue(); } @Override public long getLikeCount() { - return track.getLong("likes_count"); + return track.getNumber("likes_count", 0).longValue(); } @Override @@ -140,17 +138,17 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public String getUploaderUrl() { - return track.getJSONObject("user").getString("permalink_url"); + return track.getObject("user").getString("permalink_url", ""); } @Override public String getUploaderName() { - return track.getJSONObject("user").getString("username"); + return track.getObject("user").getString("username", ""); } @Override public String getUploaderAvatarUrl() { - return track.getJSONObject("user").optString("avatar_url"); + return track.getObject("user", new JsonObject()).getString("avatar_url", ""); } @Override @@ -167,10 +165,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor { + "?client_id=" + SoundcloudParsingHelper.clientId(); String response = dl.download(apiUrl); - JSONObject responseObject = new JSONObject(response); + JsonObject responseObject; + try { + responseObject = JsonParser.object().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } - AudioStream audioStream = new AudioStream(responseObject.getString("http_mp3_128_url"), MediaFormat.MP3.id, 128); - audioStreams.add(audioStream); + String mp3Url = responseObject.getString("http_mp3_128_url"); + if (mp3Url != null && !mp3Url.isEmpty()) { + audioStreams.add(new AudioStream(mp3Url, MediaFormat.MP3.id, 128)); + } else { + throw new ExtractionException("Could not get SoundCloud's track audio url"); + } return audioStreams; } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java index 0b831f68b..ca405db6e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java @@ -1,15 +1,15 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { - private final JSONObject searchResult; + private final JsonObject searchResult; - public SoundcloudStreamInfoItemExtractor(JSONObject searchResult) { + public SoundcloudStreamInfoItemExtractor(JsonObject searchResult) { this.searchResult = searchResult; } @@ -25,27 +25,28 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto @Override public long getDuration() { - return searchResult.getLong("duration") / 1000L; + return searchResult.getNumber("duration", 0).longValue() / 1000L; } @Override public String getUploaderName() { - return searchResult.getJSONObject("user").getString("username"); + //return searchResult.getObject("user").getString("username"); + return searchResult.getObject("track").getObject("user").getString("username"); } @Override public String getUploadDate() throws ParsingException { - return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at")); + return SoundcloudParsingHelper.toDateString(searchResult.getObject("track").getString("created_at")); } @Override public long getViewCount() { - return searchResult.getLong("playback_count"); + return searchResult.getNumber("playback_count", 0).longValue(); } @Override public String getThumbnailUrl() { - return searchResult.optString("artwork_url"); + return searchResult.getString("artwork_url"); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java index 45354a4a9..2a35b54d0 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java @@ -1,12 +1,14 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import com.github.openjson.JSONArray; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.SuggestionExtractor; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.IOException; import java.net.URLEncoder; @@ -22,7 +24,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor { } @Override - public List suggestionList(String query, String contentCountry) throws RegexException, ReCaptchaException, IOException { + public List suggestionList(String query, String contentCountry) throws IOException, ExtractionException { List suggestions = new ArrayList<>(); Downloader dl = NewPipe.getDownloader(); @@ -33,14 +35,15 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor { + "&limit=10"; String response = dl.download(url); - JSONObject responseObject = new JSONObject(response); - JSONArray responseCollection = responseObject.getJSONArray("collection"); + try { + JsonArray collection = JsonParser.object().from(response).getArray("collection"); + for (Object suggestion : collection) { + if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion).getString("query")); + } - for (int i = 0; i < responseCollection.length(); i++) { - JSONObject suggestion = responseCollection.getJSONObject(i); - suggestions.add(suggestion.getString("query")); + return suggestions; + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); } - - return suggestions; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index 35397a4e2..8d2914f56 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -1,8 +1,9 @@ package org.schabi.newpipe.extractor.services.youtube; -import com.github.openjson.JSONException; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -44,6 +45,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; + private String channelName = ""; //Small hack used to make the channelName available to NextStreams + private Document doc; /** * It's lazily initialized (when getNextStreams is called) @@ -58,7 +61,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public void fetchPage() throws IOException, ExtractionException { Downloader downloader = NewPipe.getDownloader(); - String channelUrl = getCleanUrl() + CHANNEL_URL_PARAMETERS; + String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS; String pageContent = downloader.download(channelUrl); doc = Jsoup.parse(pageContent, channelUrl); @@ -66,21 +69,34 @@ public class YoutubeChannelExtractor extends ChannelExtractor { nextStreamsAjax = null; } + @Override + public String getCleanUrl() { + try { + return "https://www.youtube.com/channel/" + getId(); + } catch (ParsingException e) { + return super.getCleanUrl(); + } + } + @Override public String getId() throws ParsingException { try { - return getUrlIdHandler().getId(getCleanUrl()); + Element element = doc.getElementsByClass("yt-uix-subscription-button").first(); + if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first(); + + return element.attr("data-channel-external-id"); } catch (Exception e) { - throw new ParsingException("Could not get channel id"); + throw new ParsingException("Could not get channel id", e); } } @Override public String getName() throws ParsingException { try { - return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); + channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); + return channelName; } catch (Exception e) { - throw new ParsingException("Could not get channel name"); + throw new ParsingException("Could not get channel name", e); } } @@ -109,8 +125,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getFeedUrl() throws ParsingException { try { - String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id"); - return channelId == null ? "" : CHANNEL_FEED_BASE + channelId; + return CHANNEL_FEED_BASE + getId(); } catch (Exception e) { throw new ParsingException("Could not get feed url", e); } @@ -153,13 +168,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor { setupNextStreamsAjax(NewPipe.getDownloader()); collectStreamsFrom(collector, nextStreamsAjax.select("body").first()); - return new NextItemsResult(collector.getItemList(), nextStreamsUrl); + return new NextItemsResult(collector, nextStreamsUrl); } private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { String ajaxDataRaw = downloader.download(nextStreamsUrl); try { - JSONObject ajaxData = new JSONObject(ajaxDataRaw); + JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw); String htmlDataRaw = ajaxData.getString("content_html"); nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl); @@ -170,7 +185,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } else { nextStreamsUrl = ""; } - } catch (JSONException e) { + } catch (JsonParserException e) { throw new ParsingException("Could not parse json data for next streams", e); } } @@ -189,7 +204,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } } - private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException { + private void collectStreamsFrom(StreamInfoItemCollector collector, + Element element) throws ParsingException { collector.getItemList().clear(); for (final Element li : element.children()) { @@ -219,7 +235,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getUploaderName() throws ParsingException { - return YoutubeChannelExtractor.this.getName(); + if(channelName.isEmpty()) { + return ""; + } else { + return channelName; + } } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index f03b6c0f2..db3d2331b 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube; -import com.github.openjson.JSONException; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -157,13 +158,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { setupNextStreamsAjax(NewPipe.getDownloader()); collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first()); - return new NextItemsResult(collector.getItemList(), nextStreamsUrl); + return new NextItemsResult(collector, nextStreamsUrl); } private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { String ajaxDataRaw = downloader.download(nextStreamsUrl); try { - JSONObject ajaxData = new JSONObject(ajaxDataRaw); + JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw); String htmlDataRaw = "" + ajaxData.getString("content_html") + "
"; nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl); @@ -174,7 +175,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } else { nextStreamsUrl = ""; } - } catch (JSONException e) { + } catch (JsonParserException e) { throw new ParsingException("Could not parse json data for next streams", e); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java new file mode 100644 index 000000000..552f58f41 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java @@ -0,0 +1,92 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Utils; + +public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + private Element el; + + public YoutubePlaylistInfoItemExtractor(Element el) { + this.el = el; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + String url; + + try { + Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first() + .select("img").first(); + url = te.attr("abs:src"); + + if (url.contains(".gif")) { + url = te.attr("abs:data-thumb"); + } + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist thumbnail url", e); + } + + return url; + } + + @Override + public String getName() throws ParsingException { + String name; + try { + final Element title = el.select("[class=\"yt-lockup-title\"]").first() + .select("a").first(); + + name = title == null ? "" : title.text(); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist name", e); + } + + return name; + } + + @Override + public String getUrl() throws ParsingException { + String url; + + try { + final Element href = el.select("div[class=\"yt-lockup-meta\"]").first() + .select("a").first(); + + url = href.attr("abs:href"); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist url", e); + } + + return url; + } + + @Override + public String getUploaderName() throws ParsingException { + String name; + + try { + final Element div = el.select("div[class=\"yt-lockup-byline\"]").first() + .select("a").first(); + + name = div.text(); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist uploader", e); + } + + return name; + } + + @Override + public long getStreamCount() throws ParsingException { + try { + final Element count = el.select("span[class=\"formatted-video-count-label\"]").first() + .select("b").first(); + + return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text())); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist stream count", e); + } + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java index 74bcc0863..0c24f993c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.utils.Parser; public class YoutubePlaylistUrlIdHandler implements UrlIdHandler { private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler(); - private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})"; + private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{10,})"; public static YoutubePlaylistUrlIdHandler getInstance() { return instance; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java index 62beae1dd..af0813507 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java @@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.search.SearchEngine; import java.io.IOException; import java.net.URLEncoder; -import java.util.EnumSet; /* @@ -44,23 +43,29 @@ public class YoutubeSearchEngine extends SearchEngine { } @Override - public InfoItemSearchCollector search(String query, - int page, - String languageCode, - EnumSet filter) + public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException { InfoItemSearchCollector collector = getInfoItemSearchCollector(); - - Downloader downloader = NewPipe.getDownloader(); String url = "https://www.youtube.com/results" + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) + "&page=" + Integer.toString(page + 1); - if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) { - url += "&sp=EgIQAQ%253D%253D"; - } else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) { - url += "&sp=EgIQAg%253D%253D"; + + switch (filter) { + case STREAM: + url += "&sp=EgIQAVAU"; + break; + case CHANNEL: + url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU + break; + case PLAYLIST: + url += "&sp=EgIQA1AU"; //EgIQA( one )AU + break; + case ANY: + // Don't append any parameter to search for everything + default: + break; } String site; @@ -105,6 +110,8 @@ public class YoutubeSearchEngine extends SearchEngine { collector.commit(new YoutubeStreamInfoItemExtractor(el)); } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) { collector.commit(new YoutubeChannelInfoItemExtractor(el)); + } else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null) { + collector.commit(new YoutubePlaylistInfoItemExtractor(el)); } else { // noinspection ConstantConditions // simply ignore not known items diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index f99211983..189597e3a 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube; -import com.github.openjson.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -74,7 +75,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { /*//////////////////////////////////////////////////////////////////////////*/ private Document doc; - private JSONObject playerArgs; + private JsonObject playerArgs; private Map videoInfoPage; private boolean isAgeRestricted; @@ -137,7 +138,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { - return playerArgs.getString("thumbnail_url"); + if (playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url"); } catch (Exception ignored) { // Try other method... } @@ -174,7 +175,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getLength() throws ParsingException { try { - return playerArgs.getLong("length_seconds"); + long returnValue = playerArgs.getNumber("length_seconds", -1).longValue(); + if (returnValue >= 0) return returnValue; } catch (Exception ignored) { // Try other method... } @@ -342,8 +344,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { String dashManifestUrl; if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) { dashManifestUrl = videoInfoPage.get("dashmpd"); - } else if (playerArgs.get("dashmpd") != null) { - dashManifestUrl = playerArgs.optString("dashmpd"); + } else if (playerArgs.isString("dashmpd")) { + dashManifestUrl = playerArgs.getString("dashmpd", ""); } else { return ""; } @@ -513,7 +515,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerUrl = getPlayerUrlFromRestrictedVideo(); isAgeRestricted = true; } else { - JSONObject ytPlayerConfig = getPlayerConfig(pageContent); + JsonObject ytPlayerConfig = getPlayerConfig(pageContent); playerArgs = getPlayerArgs(ytPlayerConfig); playerUrl = getPlayerUrl(ytPlayerConfig); isAgeRestricted = false; @@ -524,11 +526,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private JSONObject getPlayerConfig(String pageContent) throws ParsingException { + private JsonObject getPlayerConfig(String pageContent) throws ParsingException { try { - String ytPlayerConfigRaw = - Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); - return new JSONObject(ytPlayerConfigRaw); + String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); + return JsonParser.object().from(ytPlayerConfigRaw); } catch (Parser.RegexException e) { String errorReason = getErrorMessage(); switch (errorReason) { @@ -544,13 +545,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException { - JSONObject playerArgs; + private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException { + JsonObject playerArgs; //attempt to load the youtube js player JSON arguments boolean isLiveStream = false; //used to determine if this is a livestream or not try { - playerArgs = playerConfig.getJSONObject("args"); + playerArgs = playerConfig.getObject("args"); // check if we have a live stream. We need to filter it, since its not yet supported. if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live")) @@ -567,14 +568,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { return playerArgs; } - private String getPlayerUrl(JSONObject playerConfig) throws ParsingException { + private String getPlayerUrl(JsonObject playerConfig) throws ParsingException { try { // The Youtube service needs to be initialized by downloading the // js-Youtube-player. This is done in order to get the algorithm // for decrypting cryptic signatures inside certain stream urls. String playerUrl; - JSONObject ytAssets = playerConfig.getJSONObject("assets"); + JsonObject ytAssets = playerConfig.getObject("assets"); playerUrl = ytAssets.getString("js"); if (playerUrl.startsWith("//")) { @@ -582,8 +583,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } return playerUrl; } catch (Exception e) { - throw new ParsingException( - "Could not load decryption code for the Youtube service.", e); + throw new ParsingException("Could not load decryption code for the Youtube service.", e); } } @@ -682,10 +682,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { Map urlAndItags = new LinkedHashMap<>(); String encodedUrlMap = ""; - if (videoInfoPage != null && videoInfoPage.containsKey(encodedUrlMapKey)) { + if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) { + encodedUrlMap = playerArgs.getString(encodedUrlMapKey, ""); + } else if (videoInfoPage != null && videoInfoPage.containsKey(encodedUrlMapKey)) { encodedUrlMap = videoInfoPage.get(encodedUrlMapKey); - } else if (playerArgs != null && playerArgs.get(encodedUrlMapKey) != null) { - encodedUrlMap = playerArgs.optString(encodedUrlMapKey); } for (String url_data_str : encodedUrlMap.split(",")) { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java index 2e4e7cff1..8b48c1f28 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java @@ -74,7 +74,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { try { if (getStreamType() == StreamType.LIVE_STREAM) return -1; - return YoutubeParsingHelper.parseDurationString(item.select("span[class*=\"video-time\"]").first().text()); + final Element duration = item.select("span[class*=\"video-time\"]").first(); + // apparently on youtube, video-time element will not show up if the video has a duration of 00:00 + // see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf + return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text()); } catch (Exception e) { throw new ParsingException("Could not get Duration: " + getUrl(), e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java index 6419bdd95..79b61dc6b 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube; -import com.github.openjson.JSONArray; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.SuggestionExtractor; @@ -53,14 +55,12 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { String response = dl.download(url); try { - JSONArray suggestionsArray = new JSONArray(response).getJSONArray(1); - for (int i = 0; i < suggestionsArray.length(); i++) { - suggestions.add(suggestionsArray.get(i).toString()); - } - } catch (Exception e) { - throw new ParsingException("Could not parse suggestions response.", e); - } + JsonArray collection = JsonParser.array().from(response).getArray(1); + for (Object suggestion : collection) suggestions.add(suggestion.toString()); - return suggestions; + return suggestions; + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } } } diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index e8cd8f9b6..8d23c7089 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -4,9 +4,7 @@ import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.utils.DashMpdParser; -import org.schabi.newpipe.extractor.utils.Utils; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -144,26 +142,28 @@ public class StreamInfo extends Info { if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new ArrayList<>(); if (streamInfo.audio_streams == null) streamInfo.audio_streams = new ArrayList<>(); + Exception dashMpdError = null; if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) { try { DashMpdParser.getStreams(streamInfo); } catch (Exception e) { - // Sometimes we receive 403 (forbidden) error when trying to download the manifest, - // (similar to https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L1888) - // just skip the exception, as we later check if we have any streams - if (!Utils.hasCauseThrowable(e, FileNotFoundException.class)) { - streamInfo.addException(new ExtractionException("Couldn't get streams from dash mpd", e)); - } + // Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl), + // just skip the exception (but store it somewhere), as we later check if we have streams anyway. + dashMpdError = e; } } - // either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream, - // and therefore failed. (Since video_only_streams are just optional they don't caunt). + // Either audio or video has to be available, otherwise we didn't get a stream (since videoOnly are optional, they don't count). if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty()) - && (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty()) - && (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) { - throw new StreamExtractException( - "Could not get any stream. See error variable to get further details."); + && (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())) { + + if (dashMpdError != null) { + // If we don't have any video or audio and the dashMpd 'errored', add it to the error list + // (it's optional and it don't get added automatically, but it's good to have some additional error context) + streamInfo.addException(dashMpdError); + } + + throw new StreamExtractException("Could not get any stream. See error variable to get further details."); } return streamInfo; diff --git a/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 126f36e3f..364e78834 100644 --- a/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -20,31 +20,6 @@ public class Utils { return toRemove.replaceAll("\\D+", ""); } - /** - * Check if throwable have the cause - */ - public static boolean hasCauseThrowable(Throwable throwable, Class... causesToCheck) { - // Check if getCause is not the same as cause (the getCause is already the root), - // as it will cause a infinite loop if it is - Throwable cause, getCause = throwable; - - for (Class causesEl : causesToCheck) { - if (throwable.getClass().isAssignableFrom(causesEl)) { - return true; - } - } - - while ((cause = throwable.getCause()) != null && getCause != cause) { - getCause = cause; - for (Class causesEl : causesToCheck) { - if (cause.getClass().isAssignableFrom(causesEl)) { - return true; - } - } - } - return false; - } - /** * Check if the url matches the pattern. * diff --git a/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java b/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java index 294badec2..eb367b295 100644 --- a/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java @@ -4,12 +4,10 @@ import org.junit.Test; import java.util.HashSet; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl; public class NewPipeTest { @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index ef1e723e1..f7eb2cc76 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -31,17 +32,17 @@ public class SoundcloudChannelExtractorTest { @Test public void testGetName() throws Exception { - assertEquals(extractor.getName(), "LIL UZI VERT"); + assertEquals("LIL UZI VERT", extractor.getName()); } @Test public void testGetDescription() throws Exception { - assertEquals(extractor.getDescription(), ""); + assertTrue(extractor.getDescription() != null); } @Test public void testGetAvatarUrl() throws Exception { - assertEquals(extractor.getAvatarUrl(), "https://a1.sndcdn.com/images/default_avatar_large.png"); + assertTrue(extractor.getAvatarUrl().contains("https://")); } @Test @@ -70,7 +71,9 @@ public class SoundcloudChannelExtractorTest { public void testGetNextStreams() throws Exception { // Setup the streams extractor.getStreams(); - assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); + ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams(); + assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty()); + assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty()); assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java new file mode 100644 index 000000000..1e38d2acf --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -0,0 +1,28 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; + +public class SoundcloudParsingHelperTest { + @BeforeClass + public static void setUp() { + NewPipe.init(Downloader.getInstance()); + } + + @Test + public void resolveUrlWithEmbedPlayerTest() throws Exception { + Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743")); + Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159")); + } + + @Test + public void resolveIdWithEmbedPlayerTest() throws Exception { + Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity")); + Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds")); + + } + +} \ No newline at end of file diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 480101055..18e0ea851 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -4,6 +4,7 @@ import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import static org.junit.Assert.*; @@ -40,7 +41,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetThumbnailUrl() throws Exception { - assertEquals(extractor.getThumbnailUrl(), "https://i1.sndcdn.com/artworks-000174203688-bweu12-large.jpg"); + assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://")); } @Test @@ -55,7 +56,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetUploaderAvatarUrl() throws Exception { - assertEquals(extractor.getUploaderAvatarUrl(), "https://a1.sndcdn.com/images/default_avatar_large.png"); + assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("https://")); } @Test @@ -79,4 +80,15 @@ public class SoundcloudPlaylistExtractorTest { extractor.getStreams(); assertTrue("extractor didn't have more streams", !extractor.hasMoreStreams()); } + + @Test(expected = ExtractionException.class) + public void testGetNextStreamsNonExistent() throws Exception { + // Setup the streams + extractor.getStreams(); + + // This playlist don't have more streams, it should throw an error + extractor.getNextStreams(); + + fail("Expected exception wasn't thrown"); + } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java index ea82ffb69..890c2fd38 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java @@ -8,8 +8,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchResult; -import java.util.EnumSet; - import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -27,9 +25,8 @@ public class SoundcloudSearchEngineAllTest { // SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert" // keep in mind that the suggestions can NOT change by country (the parameter "de") - result = engine.search("lill uzi vert", 0, "de", - EnumSet.of(SearchEngine.Filter.CHANNEL, - SearchEngine.Filter.STREAM)).getSearchResult(); + result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.ANY) + .getSearchResult(); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java index c2474df08..e4ccee6e2 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java @@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchResult; -import java.util.EnumSet; - import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -27,8 +25,8 @@ public class SoundcloudSearchEngineChannelTest { // SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert" // keep in mind that the suggestions can NOT change by country (the parameter "de") - result = engine.search("lill uzi vert", 0, "de", - EnumSet.of(SearchEngine.Filter.CHANNEL)).getSearchResult(); + result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.CHANNEL) + .getSearchResult(); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java new file mode 100644 index 000000000..89c5dac35 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java @@ -0,0 +1,76 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.search.SearchResult; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + + +/* + * Created by Christian Schabesberger on 29.12.15. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeSearchEngineStreamTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/** + * Test for {@link SearchEngine} + */ +public class SoundcloudSearchEnginePlaylistTest { + private SearchResult result; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + SearchEngine engine = SoundCloud.getService().getSearchEngine(); + + // Search by country not yet implemented + result = engine.search("parkmemme", 0, "", SearchEngine.Filter.PLAYLIST) + .getSearchResult(); + } + + @Test + public void testResultList() { + assertFalse(result.resultList.isEmpty()); + } + + @Test + public void testUserItemType() { + for (InfoItem infoItem : result.resultList) { + assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type); + } + } + + @Test + public void testResultErrors() { + if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); + assertTrue(result.errors == null || result.errors.isEmpty()); + } + + @Ignore + @Test + public void testSuggestion() { + //todo write a real test + assertTrue(result.suggestion != null); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java index 062726494..b0b554ebc 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java @@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchResult; -import java.util.EnumSet; - import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -27,8 +25,8 @@ public class SoundcloudSearchEngineStreamTest { // SoundCloud will suggest "lil uzi vert" instead of "lil uzi vert", // keep in mind that the suggestions can NOT change by country (the parameter "de") - result = engine.search("lill uzi vert", 0, "de", - EnumSet.of(SearchEngine.Filter.STREAM)).getSearchResult(); + result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.STREAM) + .getSearchResult(); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 8347a72f3..2d76bdd30 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -77,12 +77,12 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetThumbnailUrl() throws ParsingException { - assertEquals(extractor.getThumbnailUrl(), "https://i1.sndcdn.com/artworks-000174195399-iw6seg-large.jpg"); + assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://")); } @Test public void testGetUploaderAvatarUrl() throws ParsingException { - assertEquals(extractor.getUploaderAvatarUrl(), "https://a1.sndcdn.com/images/default_avatar_large.png"); + assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("https://")); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 0ce256910..b57f61bad 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -40,7 +41,7 @@ public class YoutubeChannelExtractorTest { public void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); extractor = YouTube.getService() - .getChannelExtractor("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); + .getChannelExtractor("https://www.youtube.com/user/Gronkh"); } @Test @@ -53,6 +54,16 @@ public class YoutubeChannelExtractorTest { assertEquals(extractor.getName(), "Gronkh"); } + @Test + public void testGetId() throws Exception { + assertEquals(extractor.getId(), "UCYJ61XIK64sp6ZFFS8sctxw"); + } + + @Test + public void testGetUrl() throws Exception { + assertEquals(extractor.getCleanUrl(), "https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); + } + @Test public void testGetDescription() throws Exception { assertEquals(extractor.getDescription(), "★ ★ ★ KLICK MICH HART, DU SAU! :D ★ ★ ★ Zart im Schmelz und süffig im Abgang. Ungebremster Spieltrieb seit 1896. Tägliche Folgen nonstop seit dem 01.04.2010!..."); @@ -70,7 +81,7 @@ public class YoutubeChannelExtractorTest { @Test public void testGetFeedUrl() throws Exception { - assertTrue(extractor.getFeedUrl(), extractor.getFeedUrl().contains("feed")); + assertEquals(extractor.getFeedUrl(), "https://www.youtube.com/feeds/videos.xml?channel_id=UCYJ61XIK64sp6ZFFS8sctxw"); } @Test @@ -99,7 +110,9 @@ public class YoutubeChannelExtractorTest { public void testGetNextStreams() throws Exception { // Setup the streams extractor.getStreams(); - assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); + ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams(); + assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty()); + assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty()); assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 5c1fbbd79..384f67f4c 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; @@ -89,7 +90,9 @@ public class YoutubePlaylistExtractorTest { public void testGetNextStreams() throws Exception { // Setup the streams extractor.getStreams(); - assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); + ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams(); + assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty()); + assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty()); assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java index 0e2c61ec8..640c97707 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java @@ -8,8 +8,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchResult; -import java.util.EnumSet; - import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -48,9 +46,8 @@ public class YoutubeSearchEngineAllTest { // Youtube will suggest "asdf" instead of "asdgff" // keep in mind that the suggestions can change by country (the parameter "de") - result = engine.search("asdgff", 0, "de", - EnumSet.of(SearchEngine.Filter.CHANNEL, - SearchEngine.Filter.STREAM)).getSearchResult(); + result = engine.search("asdgff", 0, "de", SearchEngine.Filter.ANY) + .getSearchResult(); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java index 64a12efbd..9f10c3bbd 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java @@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchResult; -import java.util.EnumSet; - import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -48,8 +46,8 @@ public class YoutubeSearchEngineChannelTest { // Youtube will suggest "gronkh" instead of "grrunkh" // keep in mind that the suggestions can change by country (the parameter "de") - result = engine.search("grrunkh", 0, "de", - EnumSet.of(SearchEngine.Filter.CHANNEL)).getSearchResult(); + result = engine.search("grrunkh", 0, "de", SearchEngine.Filter.CHANNEL) + .getSearchResult(); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java new file mode 100644 index 000000000..29c6ed441 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java @@ -0,0 +1,77 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.search.SearchResult; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + + +/* + * Created by Christian Schabesberger on 29.12.15. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeSearchEngineStreamTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/** + * Test for {@link SearchEngine} + */ +public class YoutubeSearchEnginePlaylistTest { + private SearchResult result; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + SearchEngine engine = YouTube.getService().getSearchEngine(); + + // Youtube will suggest "gronkh" instead of "grrunkh" + // keep in mind that the suggestions can change by country (the parameter "de") + result = engine.search("grrunkh", 0, "de", SearchEngine.Filter.PLAYLIST) + .getSearchResult(); + } + + @Test + public void testResultList() { + assertFalse(result.resultList.isEmpty()); + } + + @Test + public void testUserItemType() { + for (InfoItem infoItem : result.resultList) { + assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type); + } + } + + @Test + public void testResultErrors() { + if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); + assertTrue(result.errors == null || result.errors.isEmpty()); + } + + @Ignore + @Test + public void testSuggestion() { + //todo write a real test + assertTrue(result.suggestion != null); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java index b9feef529..f2b06a0be 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java @@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchResult; -import java.util.EnumSet; - import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -48,8 +46,8 @@ public class YoutubeSearchEngineStreamTest { // Youtube will suggest "results" instead of "rsults", // keep in mind that the suggestions can change by country (the parameter "de") - result = engine.search("rsults", 0, "de", - EnumSet.of(SearchEngine.Filter.STREAM)).getSearchResult(); + result = engine.search("rsults", 0, "de", SearchEngine.Filter.STREAM) + .getSearchResult(); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index b9cbfddae..44663611a 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -13,9 +13,7 @@ import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; /* diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index bd8e2c148..fadd13df4 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -67,7 +67,7 @@ public class YoutubeStreamExtractorRestrictedTest { @Test public void testGetViews() throws ParsingException { - assertTrue(extractor.getLength() > 0); + assertTrue(extractor.getViewCount() > 0); } @Test