kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2024-12-15 14:50:33 +05:30
Code Issues Projects Releases Wiki Activity

Merge pull request #532 from B0pol/utf8

Browse Source 
Code improvement
This commit is contained in:
bopol 2021-02-15 18:56:28 +01:00 committed by GitHub
commit 416cf17b7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 195 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -14,6 +14,8 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class KioskList { public class KioskList {
public interface KioskExtractorFactory { public interface KioskExtractorFactory {
@ -70,7 +72,7 @@ public class KioskList {
public KioskExtractor getDefaultKioskExtractor(Page nextPage, Localization localization) public KioskExtractor getDefaultKioskExtractor(Page nextPage, Localization localization)
throws ExtractionException, IOException { throws ExtractionException, IOException {
if (defaultKiosk != null && !defaultKiosk.equals("")) { if (!isNullOrEmpty(defaultKiosk)) {
return getExtractorById(defaultKiosk, nextPage, localization); return getExtractorById(defaultKiosk, nextPage, localization);
} else { } else {
if (!kioskList.isEmpty()) { if (!kioskList.isEmpty()) {

12
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java
View File

@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
/////////////////////////////////// ///////////////////////////////////
@ -12,7 +14,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
/////////////////////////////////// ///////////////////////////////////
@Override @Override
public abstract String getUrl(String querry, List<String> contentFilter, String sortFilter) throws ParsingException; public abstract String getUrl(String query, List<String> contentFilter, String sortFilter) throws ParsingException;
public String getSearchString(String url) { public String getSearchString(String url) {
return ""; return "";
@ -28,14 +30,14 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
} }
@Override @Override
public SearchQueryHandler fromQuery(String querry, public SearchQueryHandler fromQuery(String query,
List<String> contentFilter, List<String> contentFilter,
String sortFilter) throws ParsingException { String sortFilter) throws ParsingException {
return new SearchQueryHandler(super.fromQuery(querry, contentFilter, sortFilter)); return new SearchQueryHandler(super.fromQuery(query, contentFilter, sortFilter));
} }
public SearchQueryHandler fromQuery(String querry) throws ParsingException { public SearchQueryHandler fromQuery(String query) throws ParsingException {
return fromQuery(querry, new ArrayList<String>(0), ""); return fromQuery(query, new ArrayList<String>(0), EMPTY_STRING);
} }
/** /**

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
View File

@ -7,6 +7,8 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String ALL = "all"; public static final String ALL = "all";
public static final String CONFERENCES = "conferences"; public static final String CONFERENCES = "conferences";
@ -31,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
final String sortFilter) throws ParsingException { final String sortFilter) throws ParsingException {
try { try {
return "https://media.ccc.de/public/events/search?q=" return "https://media.ccc.de/public/events/search?q="
+ URLEncoder.encode(query, "UTF-8"); + URLEncoder.encode(query, UTF_8);
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not create search string with query: " + query, e); throw new ParsingException("Could not create search string with query: " + query, e);
} }

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java
View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube.extractors; package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.ServiceList;
@ -13,6 +12,8 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.util.Objects; import java.util.Objects;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject item; private final JsonObject item;
private final String url; private final String url;
@ -68,7 +69,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
final Document doc = Jsoup.parse(htmlText); final Document doc = Jsoup.parse(htmlText);
return doc.body().text(); return doc.body().text();
} catch (Exception e) { } catch (Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", EMPTY_STRING);
} }
} }

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -18,15 +18,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
@ -40,6 +32,8 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class PeertubeStreamExtractor extends StreamExtractor { public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl; private final String baseUrl;
private JsonObject json; private JsonObject json;
@ -322,7 +316,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
params.append("start=0&count=8&sort=-createdAt"); params.append("start=0&count=8&sort=-createdAt");
for (final String tag : tags) { for (final String tag : tags) {
params.append("&tagsOneOf="); params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8")); params.append(URLEncoder.encode(tag, UTF_8));
} }
return url + "?" + params.toString(); return url + "?" + params.toString();
} }

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
View File

@ -8,9 +8,10 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String VIDEOS = "videos"; public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String SEPIA_BASE_URL = "https://sepiasearch.org"; public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
@ -35,7 +36,7 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException { public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException {
try { try {
final String url = baseUrl + SEARCH_ENDPOINT final String url = baseUrl + SEARCH_ENDPOINT
+ "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8); + "?search=" + URLEncoder.encode(searchString, UTF_8);
return url; return url;
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -37,9 +37,7 @@ import java.util.List;
import static java.util.Collections.singletonList; import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper { public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20 private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20
@ -117,7 +115,7 @@ public class SoundcloudParsingHelper {
*/ */
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
String apiUrl = "https://api-v2.soundcloud.com/resolve" String apiUrl = "https://api-v2.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8") + "?url=" + URLEncoder.encode(url, UTF_8)
+ "&client_id=" + clientId(); + "&client_id=" + clientId();
try { try {
@ -136,7 +134,7 @@ public class SoundcloudParsingHelper {
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException { public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody(); + URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href"); return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
} }
@ -158,7 +156,7 @@ public class SoundcloudParsingHelper {
} }
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody(); + URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
// handle playlists / sets different and get playlist id via uir field in JSON // handle playlists / sets different and get playlist id via uir field in JSON
if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets")) if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response); return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -15,11 +14,10 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java
View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -4,7 +4,7 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -4,13 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -19,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
@ -26,10 +21,8 @@ import java.net.URL;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudSearchExtractor extends SearchExtractor { public class SoundcloudSearchExtractor extends SearchExtractor {

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
@ -17,15 +16,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
@ -34,11 +28,7 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.utils.Utils.*;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudStreamExtractor extends StreamExtractor { public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track; private JsonObject track;
@ -60,7 +50,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public String getId() { public String getId() {
return track.getInt("id") + ""; return track.getInt("id") + EMPTY_STRING;
} }
@Nonnull @Nonnull
@ -73,8 +63,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override @Override
public String getTextualUploadDate() { public String getTextualUploadDate() {
return track.getString("created_at") return track.getString("created_at")
.replace("T"," ") .replace("T", " ")
.replace("Z", ""); .replace("Z", EMPTY_STRING);
} }
@Nonnull @Nonnull
@ -232,7 +222,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
private static String urlEncode(String value) { private static String urlEncode(String value) {
try { try {
return URLEncoder.encode(value, "UTF-8"); return URLEncoder.encode(value, UTF_8);
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
View File

@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java
View File

@ -17,9 +17,9 @@ import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor { import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public static final String CHARSET_UTF_8 = "UTF-8"; public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public SoundcloudSuggestionExtractor(StreamingService service) { public SoundcloudSuggestionExtractor(StreamingService service) {
super(service); super(service);
@ -32,7 +32,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
Downloader dl = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search/queries" String url = "https://api-v2.soundcloud.com/search/queries"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) + "?q=" + URLEncoder.encode(query, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId() + "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10"; + "&limit=10";

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java
View File

@ -11,8 +11,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String TRACKS = "tracks"; public static final String TRACKS = "tracks";
public static final String USERS = "users"; public static final String USERS = "users";
@ -43,7 +44,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
} }
} }
return url + "?q=" + URLEncoder.encode(id, CHARSET_UTF_8) return url + "?q=" + URLEncoder.encode(id, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId() + "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=" + ITEMS_PER_PAGE + "&limit=" + ITEMS_PER_PAGE
+ "&offset=0"; + "&offset=0";

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -33,7 +33,6 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*; import static org.schabi.newpipe.extractor.utils.Utils.*;
/* /*
@ -124,6 +123,7 @@ public class YoutubeParsingHelper {
/** /**
* Parses the duration string of the video expecting ":" or "." as separators * Parses the duration string of the video expecting ":" or "." as separators
*
* @return the duration in seconds * @return the duration in seconds
* @throws ParsingException when more than 3 separators are found * @throws ParsingException when more than 3 separators are found
*/ */
@ -193,6 +193,7 @@ public class YoutubeParsingHelper {
/** /**
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist) * Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD" * Ids from a YouTube Mix start with "RD"
*
* @param playlistId * @param playlistId
* @return Whether given id belongs to a YouTube Mix * @return Whether given id belongs to a YouTube Mix
*/ */
@ -203,15 +204,18 @@ public class YoutubeParsingHelper {
/** /**
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK" * Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
*
* @param playlistId * @param playlistId
* @return Whether given id belongs to a YouTube Music Mix * @return Whether given id belongs to a YouTube Music Mix
*/ */
public static boolean isYoutubeMusicMixId(final String playlistId) { public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK"); return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
} }
/** /**
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
* Ids from a YouTube channel Mix start with "RDCM" * Ids from a YouTube channel Mix start with "RDCM"
*
* @return Whether given id belongs to a YouTube Channel Mix * @return Whether given id belongs to a YouTube Channel Mix
*/ */
public static boolean isYoutubeChannelMixId(final String playlistId) { public static boolean isYoutubeChannelMixId(final String playlistId) {
@ -220,6 +224,7 @@ public class YoutubeParsingHelper {
/** /**
* Extracts the video id from the playlist id for Mixes. * Extracts the video id from the playlist id for Mixes.
*
* @throws ParsingException If the playlistId is a Channel Mix or not a mix. * @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/ */
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException { public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
@ -311,7 +316,8 @@ public class YoutubeParsingHelper {
clientVersion = contextClientVersion; clientVersion = contextClientVersion;
break; break;
} }
} catch (Parser.RegexException ignored) { } } catch (Parser.RegexException ignored) {
}
} }
if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) { if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
@ -323,7 +329,8 @@ public class YoutubeParsingHelper {
} catch (Parser.RegexException e) { } catch (Parser.RegexException e) {
try { try {
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html); key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException ignored) { } } catch (Parser.RegexException ignored) {
}
} }
} }
@ -390,7 +397,7 @@ public class YoutubeParsingHelper {
.end() .end()
.value("query", "test") .value("query", "test")
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
.end().done().getBytes("UTF-8"); .end().done().getBytes(UTF_8);
// @formatter:on // @formatter:on
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
@ -454,7 +461,7 @@ public class YoutubeParsingHelper {
if (param.split("=")[0].equals("q")) { if (param.split("=")[0].equals("q")) {
String url; String url;
try { try {
url = URLDecoder.decode(param.split("=")[1], "UTF-8"); url = URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
return null; return null;
} }
@ -502,6 +509,7 @@ public class YoutubeParsingHelper {
/** /**
* Get the text from a JSON object that has either a simpleText or a runs array. * Get the text from a JSON object that has either a simpleText or a runs array.
*
* @param textObject JSON object to get the text from * @param textObject JSON object to get the text from
* @param html whether to return HTML, by parsing the navigationEndpoint * @param html whether to return HTML, by parsing the navigationEndpoint
* @return text in the JSON object or {@code null} * @return text in the JSON object or {@code null}
@ -729,7 +737,7 @@ public class YoutubeParsingHelper {
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle")); final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle"));
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text")); final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text"));
if (title == null || text == null) { if (title == null || text == null) {
throw new ParsingException("Could not extract clarification renderer content"); throw new ParsingException("Could not extract clarification renderer content");
} }
metaInfo.setTitle(title); metaInfo.setTitle(title);
@ -773,6 +781,7 @@ public class YoutubeParsingHelper {
/** /**
* Sometimes, YouTube provides URLs which use Google's cache. They look like * Sometimes, YouTube provides URLs which use Google's cache. They look like
* {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL} * {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL}
*
* @param url the URL which might refer to the Google's webcache * @param url the URL which might refer to the Google's webcache
* @return the URL which is referring to the original site * @return the URL which is referring to the original site
*/ */

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -21,7 +21,7 @@ import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@ -19,6 +18,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
@ -27,9 +27,8 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import static java.util.Collections.singletonList; import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeCommentsExtractor extends CommentsExtractor { public class YoutubeCommentsExtractor extends CommentsExtractor {
@ -152,9 +151,9 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
first = false; first = false;
else else
result.append("&"); result.append("&");
result.append(URLEncoder.encode(entry.getKey(), "UTF-8")); result.append(URLEncoder.encode(entry.getKey(), UTF_8));
result.append("="); result.append("=");
result.append(URLEncoder.encode(entry.getValue(), "UTF-8")); result.append(URLEncoder.encode(entry.getValue(), UTF_8));
} }
return result.toString(); return result.toString();
} }

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -13,6 +12,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
@ -46,7 +46,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try { try {
return getTextFromObject(JsonUtils.getObject(json, "authorText")); return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) { } catch (Exception e) {
return ""; return EMPTY_STRING;
} }
} }
@ -86,7 +86,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
if (contentText.isEmpty()) { if (contentText.isEmpty()) {
// completely empty comments as described in // completely empty comments as described in
// https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584 // https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
return ""; return EMPTY_STRING;
} }
final String commentText = getTextFromObject(contentText); final String commentText = getTextFromObject(contentText);
// youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff> // youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
@ -130,7 +130,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try { try {
return getTextFromObject(JsonUtils.getObject(json, "authorText")); return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) { } catch (Exception e) {
return ""; return EMPTY_STRING;
} }
} }
@ -139,7 +139,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try { try {
return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId"); return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
} catch (Exception e) { } catch (Exception e) {
return ""; return EMPTY_STRING;
} }
} }

28
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -1,11 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.*;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
@ -24,25 +19,16 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeMusicSearchExtractor extends SearchExtractor { public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData; private JsonObject initialData;
@ -107,7 +93,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.end() .end()
.value("query", getSearchString()) .value("query", getSearchString())
.value("params", params) .value("params", params)
.end().done().getBytes("UTF-8"); .end().done().getBytes(UTF_8);
// @formatter:on // @formatter:on
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
@ -231,7 +217,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.value("enableSafetyMode", false) .value("enableSafetyMode", false)
.end() .end()
.end() .end()
.end().done().getBytes("UTF-8"); .end().done().getBytes(UTF_8);
// @formatter:on // @formatter:on
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -82,7 +82,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
final String name = getTextFromObject(playlistInfo.getObject("title")); final String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null && !name.isEmpty()) return name; if (!isNullOrEmpty(name)) return name;
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
} }

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -1,11 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.*;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
@ -20,16 +15,15 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.util.*;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
@ -171,7 +165,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.object("user").end() .object("user").end()
.end() .end()
.value("continuation", page.getId()) .value("continuation", page.getId())
.end().done().getBytes("UTF-8"); .end().done().getBytes(UTF_8);
// @formatter:on // @formatter:on
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -29,17 +28,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
@ -47,26 +36,13 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.io.UnsupportedEncodingException; import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
@ -102,17 +78,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/ /*//////////////////////////////////////////////////////////////////////////*/
@Nullable private static String cachedDeobfuscationCode = null; @Nullable
@Nullable private String playerJsUrl = null; private static String cachedDeobfuscationCode = null;
@Nullable
private String playerJsUrl = null;
private JsonArray initialAjaxJson; private JsonArray initialAjaxJson;
private JsonObject initialData; private JsonObject initialData;
@Nonnull private final Map<String, String> videoInfoPage = new HashMap<>(); @Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse; private JsonObject playerResponse;
private JsonObject videoPrimaryInfoRenderer; private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer;
private int ageLimit = -1; private int ageLimit = -1;
@Nullable private List<SubtitlesStream> subtitles = null; @Nullable
private List<SubtitlesStream> subtitles = null;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
@ -787,8 +767,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null; Parser.RegexException exception = null;
for (final String regex : REGEXES) { for (final String regex : REGEXES) {
@ -1138,6 +1116,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public List<MetaInfo> getMetaInfo() throws ParsingException { public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo( return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnWatchNextResults") initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents")); .getObject("results").getObject("results").getArray("contents"));
} }
} }

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -17,10 +17,8 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -15,6 +15,8 @@ import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/* /*
* Created by Christian Schabesberger on 28.09.16. * Created by Christian Schabesberger on 28.09.16.
* *
@ -37,8 +39,6 @@ import java.util.List;
public class YoutubeSuggestionExtractor extends SuggestionExtractor { public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSuggestionExtractor(StreamingService service) { public YoutubeSuggestionExtractor(StreamingService service) {
super(service); super(service);
} }
@ -52,8 +52,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
+ "&jsonp=" + "JP" + "&jsonp=" + "JP"
+ "&ds=" + "yt" + "&ds=" + "yt"
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8) + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8); + "&q=" + URLEncoder.encode(query, UTF_8);
String response = dl.get(url, getExtractorLocalization()).responseBody(); String response = dl.get(url, getExtractorLocalization()).responseBody();
// trim JSONP part "JP(...)" // trim JSONP part "JP(...)"

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
View File

@ -7,8 +7,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String ALL = "all"; public static final String ALL = "all";
public static final String VIDEOS = "videos"; public static final String VIDEOS = "videos";
@ -37,21 +38,21 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
default: default:
break; break;
case VIDEOS: case VIDEOS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D"; return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D";
case CHANNELS: case CHANNELS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D"; return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAg%253D%253D";
case PLAYLISTS: case PLAYLISTS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D"; return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAw%253D%253D";
case MUSIC_SONGS: case MUSIC_SONGS:
case MUSIC_VIDEOS: case MUSIC_VIDEOS:
case MUSIC_ALBUMS: case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS: case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS: case MUSIC_ARTISTS:
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
} }
} }
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); return SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e); throw new ParsingException("Could not encode query", e);
} }

18
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java
View File

@ -1,13 +1,16 @@
package org.schabi.newpipe.extractor.stream; package org.schabi.newpipe.extractor.stream;
import java.io.Serializable; import java.io.Serializable;
import java.util.Objects;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class Description implements Serializable { public class Description implements Serializable {
public static final int HTML = 1; public static final int HTML = 1;
public static final int MARKDOWN = 2; public static final int MARKDOWN = 2;
public static final int PLAIN_TEXT = 3; public static final int PLAIN_TEXT = 3;
public static final Description emptyDescription = new Description("", PLAIN_TEXT); public static final Description emptyDescription = new Description(EMPTY_STRING, PLAIN_TEXT);
private String content; private String content;
private int type; private int type;
@ -28,4 +31,17 @@ public class Description implements Serializable {
public int getType() { public int getType() {
return type; return type;
} }
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Description that = (Description) o;
return type == that.type && Objects.equals(content, that.content);
}
@Override
public int hashCode() {
return Objects.hash(content, type);
}
} }

1
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View File

@ -13,7 +13,6 @@ import java.util.List;
public class JsonUtils { public class JsonUtils {
public static final JsonObject EMPTY_OBJECT = new JsonObject(); public static final JsonObject EMPTY_OBJECT = new JsonObject();
public static final JsonArray EMPTY_ARRAY = new JsonArray(); public static final JsonArray EMPTY_ARRAY = new JsonArray();
public static final String EMPTY_STRING = "";
private JsonUtils() { private JsonUtils() {
} }

4
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
View File

@ -14,6 +14,8 @@ import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/* /*
* Created by Christian Schabesberger on 02.02.16. * Created by Christian Schabesberger on 02.02.16.
* *
@ -87,7 +89,7 @@ public class Parser {
for (String arg : input.split("&")) { for (String arg : input.split("&")) {
String[] splitArg = arg.split("="); String[] splitArg = arg.split("=");
if (splitArg.length > 1) { if (splitArg.length > 1) {
map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8")); map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8));
} else { } else {
map.put(splitArg[0], ""); map.put(splitArg[0], "");
} }

23
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -6,16 +6,14 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.util.Collection; import java.util.*;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
public class Utils { public class Utils {
public static final String HTTP = "http://"; public static final String HTTP = "http://";
public static final String HTTPS = "https://"; public static final String HTTPS = "https://";
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private Utils() { private Utils() {
//no instance //no instance
@ -117,7 +115,7 @@ public class Utils {
String query; String query;
try { try {
query = URLDecoder.decode(params[0], "UTF-8"); query = URLDecoder.decode(params[0], UTF_8);
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding"); System.err.println("Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace(); e.printStackTrace();
@ -126,7 +124,7 @@ public class Utils {
if (query.equals(parameterName)) { if (query.equals(parameterName)) {
try { try {
return URLDecoder.decode(params[1], "UTF-8"); return URLDecoder.decode(params[1], UTF_8);
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding"); System.err.println("Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace(); e.printStackTrace();
@ -200,6 +198,7 @@ public class Utils {
/** /**
* If the provided url is a Google search redirect, then the actual url is extracted from the * If the provided url is a Google search redirect, then the actual url is extracted from the
* {@code url=} query value and returned, otherwise the original url is returned. * {@code url=} query value and returned, otherwise the original url is returned.
*
* @param url the url which can possibly be a Google search redirect * @param url the url which can possibly be a Google search redirect
* @return an url with no Google search redirects * @return an url with no Google search redirects
*/ */
@ -208,7 +207,7 @@ public class Utils {
try { try {
final URL decoded = Utils.stringToURL(url); final URL decoded = Utils.stringToURL(url);
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) { if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8"); return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
} }
} catch (final Exception ignored) { } catch (final Exception ignored) {
} }
@ -231,12 +230,12 @@ public class Utils {
return map == null || map.isEmpty(); return map == null || map.isEmpty();
} }
public static boolean isWhitespace(final int c){ public static boolean isWhitespace(final int c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r'; return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r';
} }
public static boolean isBlank(final String string) { public static boolean isBlank(final String string) {
if (string == null || string.isEmpty()) { if (isNullOrEmpty(string)) {
return true; return true;
} }
@ -263,10 +262,10 @@ public class Utils {
} }
public static String join(final String delimiter, final String mapJoin, public static String join(final String delimiter, final String mapJoin,
final Map<? extends CharSequence, ? extends CharSequence> elements) { final Map<? extends CharSequence, ? extends CharSequence> elements) {
final List<String> list = new LinkedList<>(); final List<String> list = new LinkedList<>();
for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
.entrySet()) { .entrySet()) {
list.add(entry.getKey() + mapJoin + entry.getValue()); list.add(entry.getKey() + mapJoin + entry.getValue());
} }
return join(delimiter, list); return join(delimiter, list);

14
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View File

@ -19,6 +19,7 @@ import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*; import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchExtractorTest { public class SoundcloudSearchExtractorTest {
@ -33,6 +34,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; } @Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; } @Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; } @Override public String expectedName() { return QUERY; }
@ -41,6 +43,7 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); } @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; } @Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; } @Nullable @Override public String expectedSearchSuggestion() { return null; }
// @formatter:on
} }
public static class Tracks extends DefaultSearchExtractorTest { public static class Tracks extends DefaultSearchExtractorTest {
@ -54,6 +57,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; } @Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; } @Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; } @Override public String expectedName() { return QUERY; }
@ -62,8 +66,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); } @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; } @Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; } @Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
// @formatter:on
} }
public static class Users extends DefaultSearchExtractorTest { public static class Users extends DefaultSearchExtractorTest {
@ -77,6 +81,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; } @Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; } @Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; } @Override public String expectedName() { return QUERY; }
@ -85,8 +90,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); } @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; } @Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; } @Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
// @formatter:on
} }
public static class Playlists extends DefaultSearchExtractorTest { public static class Playlists extends DefaultSearchExtractorTest {
@ -100,6 +105,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; } @Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; } @Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; } @Override public String expectedName() { return QUERY; }
@ -108,8 +114,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); } @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; } @Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; } @Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
// @formatter:on
} }
public static class PagingTest { public static class PagingTest {
@ -128,7 +134,7 @@ public class SoundcloudSearchExtractorTest {
private static String urlEncode(String value) { private static String urlEncode(String value) {
try { try {
return URLEncoder.encode(value, CHARSET_UTF_8); return URLEncoder.encode(value, UTF_8);
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View File

@ -12,15 +12,12 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.*;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.FileUtils.resolveTestResource; import static org.schabi.newpipe.FileUtils.resolveTestResource;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/** /**
* Test for {@link YoutubeSubscriptionExtractor} * Test for {@link YoutubeSubscriptionExtractor}
@ -56,7 +53,7 @@ public class YoutubeSubscriptionExtractorTest {
@Test @Test
public void testEmptySourceException() throws Exception { public void testEmptySourceException() throws Exception {
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream( final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8))); new ByteArrayInputStream("[]".getBytes(UTF_8)));
assertTrue(items.isEmpty()); assertTrue(items.isEmpty());
} }
@ -64,7 +61,7 @@ public class YoutubeSubscriptionExtractorTest {
public void testSubscriptionWithEmptyTitleInSource() throws Exception { public void testSubscriptionWithEmptyTitleInSource() throws Exception {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]"; final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream( final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); new ByteArrayInputStream(source.getBytes(UTF_8)));
assertEquals(1, items.size()); assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId()); assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
@ -77,7 +74,7 @@ public class YoutubeSubscriptionExtractorTest {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," + final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]"; "{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream( final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); new ByteArrayInputStream(source.getBytes(UTF_8)));
assertEquals(1, items.size()); assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId()); assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
@ -101,7 +98,7 @@ public class YoutubeSubscriptionExtractorTest {
for (String invalidContent : invalidList) { for (String invalidContent : invalidList) {
try { try {
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8); byte[] bytes = invalidContent.getBytes(UTF_8);
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception"); fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} catch (final Exception e) { } catch (final Exception e) {

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
View File

@ -31,6 +31,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplica
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeSearchExtractorTest { public class YoutubeSearchExtractorTest {

27
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java
View File

@ -7,6 +7,7 @@ import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeSearchQHTest { public class YoutubeSearchQHTest {
@ -19,37 +20,37 @@ public class YoutubeSearchQHTest {
assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl()); assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl()); assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl()); assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
} }
@Test @Test
public void testGetContentFilter() throws Exception { public void testGetContentFilter() throws Exception {
assertEquals(VIDEOS, YouTube.getSearchQHFactory() assertEquals(VIDEOS, YouTube.getSearchQHFactory()
.fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0)); .fromQuery(EMPTY_STRING, asList(new String[]{VIDEOS}), EMPTY_STRING).getContentFilters().get(0));
assertEquals(CHANNELS, YouTube.getSearchQHFactory() assertEquals(CHANNELS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0)); .fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getContentFilters().get(0));
assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory() assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0)); .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getContentFilters().get(0));
} }
@Test @Test
public void testWithContentfilter() throws Exception { public void testWithContentfilter() throws Exception {
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAQ%253D%253D", YouTube.getSearchQHFactory() assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAQ%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl()); .fromQuery("asdf", asList(new String[]{VIDEOS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAg%253D%253D", YouTube.getSearchQHFactory() assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAg%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl()); .fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAw%253D%253D", YouTube.getSearchQHFactory() assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAw%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl()); .fromQuery("asdf", asList(new String[]{PLAYLISTS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory() assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); .fromQuery("asdf", asList(new String[]{"fjiijie"}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory() assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
} }
@Test @Test

20
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamSegment; import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
@ -23,11 +24,10 @@ import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import javax.annotation.Nullable;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
/* /*
* Created by Christian Schabesberger on 30.12.15. * Created by Christian Schabesberger on 30.12.15.
@ -88,6 +88,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; } @Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; } @Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; } @Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; }
@ -110,6 +111,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 5212900; } @Override public long expectedLikeCountAtLeast() { return 5212900; }
@Override public long expectedDislikeCountAtLeast() { return 30600; } @Override public long expectedDislikeCountAtLeast() { return 30600; }
@Override public int expectedStreamSegmentsCount() { return 0; } @Override public int expectedStreamSegmentsCount() { return 0; }
// @formatter:on
} }
public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest { public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
@ -125,6 +127,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; } @Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; } @Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "This Smartphone Changes Everything..."; } @Override public String expectedName() { return "This Smartphone Changes Everything..."; }
@ -147,13 +150,14 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; } @Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; }
@Override public long expectedLikeCountAtLeast() { return 340100; } @Override public long expectedLikeCountAtLeast() { return 340100; }
@Override public long expectedDislikeCountAtLeast() { return 18700; } @Override public long expectedDislikeCountAtLeast() { return 18700; }
// @formatter:on
@Override @Override
@Test @Test
@Ignore("TODO fix") @Ignore("TODO fix")
public void testDescription() throws Exception { public void testDescription() throws Exception {
super.testDescription(); super.testDescription();
} }
} }
@Ignore("TODO fix") @Ignore("TODO fix")
@ -171,6 +175,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; } @Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; } @Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; } @Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; }
@ -189,6 +194,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; } @Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; }
@Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; }
// @formatter:on
} }
public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest { public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest {
@ -205,6 +211,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; } @Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; } @Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; } @Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; }
@ -235,6 +242,7 @@ public class YoutubeStreamExtractorDefaultTest {
assertEquals(BASE_URL + ID + "?t=589", segment.getUrl()); assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
assertNotNull(segment.getPreviewUrl()); assertNotNull(segment.getPreviewUrl());
} }
// @formatter:on
} }
public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest { public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
@ -304,6 +312,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage(); extractor.fetchPage();
} }
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; } @Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; } @Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; } @Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; }
@ -324,13 +333,13 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedDislikeCountAtLeast() { return 500; } @Override public long expectedDislikeCountAtLeast() { return 500; }
@Override public List<MetaInfo> expectedMetaInfo() throws MalformedURLException { @Override public List<MetaInfo> expectedMetaInfo() throws MalformedURLException {
return Collections.singletonList(new MetaInfo( return Collections.singletonList(new MetaInfo(
"", EMPTY_STRING,
new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT), new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT),
Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")), Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")),
Collections.singletonList("Wikipedia (German)") Collections.singletonList("Wikipedia (German)")
)); ));
} }
// @formatter:on
@Override @Override
@Ignore("TODO fix") @Ignore("TODO fix")
@Test @Test
@ -344,6 +353,7 @@ public class YoutubeStreamExtractorDefaultTest {
public void testMetaInfo() throws Exception { public void testMetaInfo() throws Exception {
super.testMetaInfo(); super.testMetaInfo();
} }
} }
} }