From 9ca52ca68ae4de3786052895c3081d938227daad Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 7 Feb 2021 22:12:22 +0100 Subject: [PATCH] use only one constant UTF-8 --- .../MediaCCCSearchQueryHandlerFactory.java | 4 ++- .../extractors/PeertubeStreamExtractor.java | 14 +++------- .../PeertubeSearchQueryHandlerFactory.java | 5 ++-- .../soundcloud/SoundcloudParsingHelper.java | 9 +++---- .../extractors/SoundcloudStreamExtractor.java | 20 +++++--------- .../SoundcloudSuggestionExtractor.java | 6 ++--- .../SoundcloudSearchQueryHandlerFactory.java | 5 ++-- .../youtube/YoutubeParsingHelper.java | 20 ++++++++++---- .../extractors/YoutubeCommentsExtractor.java | 9 +++---- .../YoutubeMusicSearchExtractor.java | 26 +++++-------------- .../extractors/YoutubeSearchExtractor.java | 24 +++++++---------- .../YoutubeSuggestionExtractor.java | 8 +++--- .../YoutubeSearchQueryHandlerFactory.java | 13 +++++----- .../newpipe/extractor/utils/Parser.java | 4 ++- .../schabi/newpipe/extractor/utils/Utils.java | 20 +++++++------- .../search/SoundcloudSearchExtractorTest.java | 14 +++++++--- .../YoutubeSubscriptionExtractorTest.java | 15 +++++------ 17 files changed, 100 insertions(+), 116 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index 6d3486b54..264dcccf1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -7,6 +7,8 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.List; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; + public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public static final String ALL = "all"; public static final String CONFERENCES = "conferences"; @@ -31,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory final String sortFilter) throws ParsingException { try { return "https://media.ccc.de/public/events/search?q=" - + URLEncoder.encode(query, "UTF-8"); + + URLEncoder.encode(query, UTF_8); } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not create search string with query: " + query, e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 61ca7fb79..cfcd1e7b5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -18,15 +18,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Description; -import org.schabi.newpipe.extractor.stream.Stream; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamSegment; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; @@ -40,6 +32,8 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; + public class PeertubeStreamExtractor extends StreamExtractor { private final String baseUrl; private JsonObject json; @@ -322,7 +316,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { params.append("start=0&count=8&sort=-createdAt"); for (final String tag : tags) { params.append("&tagsOneOf="); - params.append(URLEncoder.encode(tag, "UTF-8")); + params.append(URLEncoder.encode(tag, UTF_8)); } return url + "?" + params.toString(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 98c25b0a3..87cf6ed8c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -8,9 +8,10 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.List; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; + public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - public static final String CHARSET_UTF_8 = "UTF-8"; public static final String VIDEOS = "videos"; public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index public static final String SEPIA_BASE_URL = "https://sepiasearch.org"; @@ -35,7 +36,7 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory public String getUrl(String searchString, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { try { final String url = baseUrl + SEARCH_ENDPOINT - + "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8); + + "?search=" + URLEncoder.encode(searchString, UTF_8); return url; } catch (UnsupportedEncodingException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 2b39f4492..23d9ff535 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -38,8 +38,7 @@ import java.util.List; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; +import static org.schabi.newpipe.extractor.utils.Utils.*; public class SoundcloudParsingHelper { private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20 @@ -117,7 +116,7 @@ public class SoundcloudParsingHelper { */ public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { String apiUrl = "https://api-v2.soundcloud.com/resolve" - + "?url=" + URLEncoder.encode(url, "UTF-8") + + "?url=" + URLEncoder.encode(url, UTF_8) + "&client_id=" + clientId(); try { @@ -136,7 +135,7 @@ public class SoundcloudParsingHelper { public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException { String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody(); + + URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody(); return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href"); } @@ -158,7 +157,7 @@ public class SoundcloudParsingHelper { } String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody(); + + URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody(); // handle playlists / sets different and get playlist id via uir field in JSON if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets")) return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 62e79cb2d..94df8599e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.NewPipe; @@ -17,15 +16,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Description; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamSegment; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.*; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -34,10 +28,8 @@ import java.util.Collections; import java.util.List; import java.util.Locale; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudStreamExtractor extends StreamExtractor { @@ -73,7 +65,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public String getTextualUploadDate() { return track.getString("created_at") - .replace("T"," ") + .replace("T", " ") .replace("Z", ""); } @@ -232,7 +224,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { private static String urlEncode(String value) { try { - return URLEncoder.encode(value, "UTF-8"); + return URLEncoder.encode(value, UTF_8); } catch (UnsupportedEncodingException e) { throw new IllegalStateException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java index a204c9951..d8b6a71eb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java @@ -17,9 +17,9 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; -public class SoundcloudSuggestionExtractor extends SuggestionExtractor { +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; - public static final String CHARSET_UTF_8 = "UTF-8"; +public class SoundcloudSuggestionExtractor extends SuggestionExtractor { public SoundcloudSuggestionExtractor(StreamingService service) { super(service); @@ -32,7 +32,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor { Downloader dl = NewPipe.getDownloader(); String url = "https://api-v2.soundcloud.com/search/queries" - + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) + + "?q=" + URLEncoder.encode(query, UTF_8) + "&client_id=" + SoundcloudParsingHelper.clientId() + "&limit=10"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java index ad19acda6..212f906d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java @@ -11,8 +11,9 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.List; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; + public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - public static final String CHARSET_UTF_8 = "UTF-8"; public static final String TRACKS = "tracks"; public static final String USERS = "users"; @@ -43,7 +44,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto } } - return url + "?q=" + URLEncoder.encode(id, CHARSET_UTF_8) + return url + "?q=" + URLEncoder.encode(id, UTF_8) + "&client_id=" + SoundcloudParsingHelper.clientId() + "&limit=" + ITEMS_PER_PAGE + "&offset=0"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 11fc424b2..e81141324 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -124,6 +124,7 @@ public class YoutubeParsingHelper { /** * Parses the duration string of the video expecting ":" or "." as separators + * * @return the duration in seconds * @throws ParsingException when more than 3 separators are found */ @@ -193,6 +194,7 @@ public class YoutubeParsingHelper { /** * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) * Ids from a YouTube Mix start with "RD" + * * @param playlistId * @return Whether given id belongs to a YouTube Mix */ @@ -203,15 +205,18 @@ public class YoutubeParsingHelper { /** * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) * Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK" + * * @param playlistId * @return Whether given id belongs to a YouTube Music Mix */ public static boolean isYoutubeMusicMixId(final String playlistId) { return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK"); } + /** * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) * Ids from a YouTube channel Mix start with "RDCM" + * * @return Whether given id belongs to a YouTube Channel Mix */ public static boolean isYoutubeChannelMixId(final String playlistId) { @@ -220,6 +225,7 @@ public class YoutubeParsingHelper { /** * Extracts the video id from the playlist id for Mixes. + * * @throws ParsingException If the playlistId is a Channel Mix or not a mix. */ public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException { @@ -311,7 +317,8 @@ public class YoutubeParsingHelper { clientVersion = contextClientVersion; break; } - } catch (Parser.RegexException ignored) { } + } catch (Parser.RegexException ignored) { + } } if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) { @@ -323,7 +330,8 @@ public class YoutubeParsingHelper { } catch (Parser.RegexException e) { try { key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html); - } catch (Parser.RegexException ignored) { } + } catch (Parser.RegexException ignored) { + } } } @@ -390,7 +398,7 @@ public class YoutubeParsingHelper { .end() .value("query", "test") .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") - .end().done().getBytes("UTF-8"); + .end().done().getBytes(UTF_8); // @formatter:on final Map> headers = new HashMap<>(); @@ -454,7 +462,7 @@ public class YoutubeParsingHelper { if (param.split("=")[0].equals("q")) { String url; try { - url = URLDecoder.decode(param.split("=")[1], "UTF-8"); + url = URLDecoder.decode(param.split("=")[1], UTF_8); } catch (UnsupportedEncodingException e) { return null; } @@ -502,6 +510,7 @@ public class YoutubeParsingHelper { /** * Get the text from a JSON object that has either a simpleText or a runs array. + * * @param textObject JSON object to get the text from * @param html whether to return HTML, by parsing the navigationEndpoint * @return text in the JSON object or {@code null} @@ -729,7 +738,7 @@ public class YoutubeParsingHelper { final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle")); final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text")); - if (title == null || text == null) { + if (title == null || text == null) { throw new ParsingException("Could not extract clarification renderer content"); } metaInfo.setTitle(title); @@ -773,6 +782,7 @@ public class YoutubeParsingHelper { /** * Sometimes, YouTube provides URLs which use Google's cache. They look like * {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL} + * * @param url the URL which might refer to the Google's webcache * @return the URL which is referring to the original site */ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 6b5a992c8..fa03f7e13 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; - import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -19,6 +18,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; +import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -27,9 +27,8 @@ import java.util.List; import java.util.Map; import java.util.regex.Pattern; -import javax.annotation.Nonnull; - import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeCommentsExtractor extends CommentsExtractor { @@ -152,9 +151,9 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { first = false; else result.append("&"); - result.append(URLEncoder.encode(entry.getKey(), "UTF-8")); + result.append(URLEncoder.encode(entry.getKey(), UTF_8)); result.append("="); - result.append(URLEncoder.encode(entry.getValue(), "UTF-8")); + result.append(URLEncoder.encode(entry.getValue(), UTF_8)); } return result.toString(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index b31944677..7871f9307 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -1,11 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import com.grack.nanojson.JsonWriter; - +import com.grack.nanojson.*; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.Page; @@ -24,24 +19,17 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.annotation.Nonnull; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeMusicSearchExtractor extends SearchExtractor { @@ -107,7 +95,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .end() .value("query", getSearchString()) .value("params", params) - .end().done().getBytes("UTF-8"); + .end().done().getBytes(UTF_8); // @formatter:on final Map> headers = new HashMap<>(); @@ -231,7 +219,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .value("enableSafetyMode", false) .end() .end() - .end().done().getBytes("UTF-8"); + .end().done().getBytes(UTF_8); // @formatter:on final Map> headers = new HashMap<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 783dbf1e7..c444a6ae5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -1,11 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import com.grack.nanojson.JsonWriter; - +import com.grack.nanojson.*; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.Page; @@ -20,16 +15,15 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; -import java.io.IOException; -import java.util.*; - import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /* @@ -171,7 +165,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .object("user").end() .end() .value("continuation", page.getId()) - .end().done().getBytes("UTF-8"); + .end().done().getBytes(UTF_8); // @formatter:on final Map> headers = new HashMap<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index 2c4c74bee..a98d9c926 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -15,6 +15,8 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; + /* * Created by Christian Schabesberger on 28.09.16. * @@ -37,8 +39,6 @@ import java.util.List; public class YoutubeSuggestionExtractor extends SuggestionExtractor { - public static final String CHARSET_UTF_8 = "UTF-8"; - public YoutubeSuggestionExtractor(StreamingService service) { super(service); } @@ -52,8 +52,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml + "&jsonp=" + "JP" + "&ds=" + "yt" - + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8) - + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8); + + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8) + + "&q=" + URLEncoder.encode(query, UTF_8); String response = dl.get(url, getExtractorLocalization()).responseBody(); // trim JSONP part "JP(...)" diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 866c18569..cb3933a03 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -7,8 +7,9 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.List; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; + public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - public static final String CHARSET_UTF_8 = "UTF-8"; public static final String ALL = "all"; public static final String VIDEOS = "videos"; @@ -37,21 +38,21 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory default: break; case VIDEOS: - return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D"; + return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D"; case CHANNELS: - return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D"; + return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAg%253D%253D"; case PLAYLISTS: - return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D"; + return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAw%253D%253D"; case MUSIC_SONGS: case MUSIC_VIDEOS: case MUSIC_ALBUMS: case MUSIC_PLAYLISTS: case MUSIC_ARTISTS: - return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); + return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8); } } - return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); + return SEARCH_URL + URLEncoder.encode(searchString, UTF_8); } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 7b386f005..44368de2e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -14,6 +14,8 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; + /* * Created by Christian Schabesberger on 02.02.16. * @@ -87,7 +89,7 @@ public class Parser { for (String arg : input.split("&")) { String[] splitArg = arg.split("="); if (splitArg.length > 1) { - map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8")); + map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8)); } else { map.put(splitArg[0], ""); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 3c2bc7d9f..ed2295774 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -6,16 +6,13 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; public class Utils { public static final String HTTP = "http://"; public static final String HTTPS = "https://"; + public static final String UTF_8 = "UTF-8"; private Utils() { //no instance @@ -117,7 +114,7 @@ public class Utils { String query; try { - query = URLDecoder.decode(params[0], "UTF-8"); + query = URLDecoder.decode(params[0], UTF_8); } catch (UnsupportedEncodingException e) { System.err.println("Cannot decode string with UTF-8. using the string without decoding"); e.printStackTrace(); @@ -126,7 +123,7 @@ public class Utils { if (query.equals(parameterName)) { try { - return URLDecoder.decode(params[1], "UTF-8"); + return URLDecoder.decode(params[1], UTF_8); } catch (UnsupportedEncodingException e) { System.err.println("Cannot decode string with UTF-8. using the string without decoding"); e.printStackTrace(); @@ -200,6 +197,7 @@ public class Utils { /** * If the provided url is a Google search redirect, then the actual url is extracted from the * {@code url=} query value and returned, otherwise the original url is returned. + * * @param url the url which can possibly be a Google search redirect * @return an url with no Google search redirects */ @@ -208,7 +206,7 @@ public class Utils { try { final URL decoded = Utils.stringToURL(url); if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) { - return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8"); + return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8); } } catch (final Exception ignored) { } @@ -231,7 +229,7 @@ public class Utils { return map == null || map.isEmpty(); } - public static boolean isWhitespace(final int c){ + public static boolean isWhitespace(final int c) { return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r'; } @@ -263,10 +261,10 @@ public class Utils { } public static String join(final String delimiter, final String mapJoin, - final Map elements) { + final Map elements) { final List list = new LinkedList<>(); for (final Map.Entry entry : elements - .entrySet()) { + .entrySet()) { list.add(entry.getKey() + mapJoin + entry.getValue()); } return join(delimiter, list); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java index b9f5b8cb5..94779f4d3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java @@ -19,6 +19,7 @@ import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; public class SoundcloudSearchExtractorTest { @@ -33,6 +34,7 @@ public class SoundcloudSearchExtractorTest { extractor.fetchPage(); } + // @formatter:off @Override public SearchExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return SoundCloud; } @Override public String expectedName() { return QUERY; } @@ -41,6 +43,7 @@ public class SoundcloudSearchExtractorTest { @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return null; } + // @formatter:on } public static class Tracks extends DefaultSearchExtractorTest { @@ -54,6 +57,7 @@ public class SoundcloudSearchExtractorTest { extractor.fetchPage(); } + // @formatter:off @Override public SearchExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return SoundCloud; } @Override public String expectedName() { return QUERY; } @@ -62,8 +66,8 @@ public class SoundcloudSearchExtractorTest { @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return null; } - @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + // @formatter:on } public static class Users extends DefaultSearchExtractorTest { @@ -77,6 +81,7 @@ public class SoundcloudSearchExtractorTest { extractor.fetchPage(); } + // @formatter:off @Override public SearchExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return SoundCloud; } @Override public String expectedName() { return QUERY; } @@ -85,8 +90,8 @@ public class SoundcloudSearchExtractorTest { @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return null; } - @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + // @formatter:on } public static class Playlists extends DefaultSearchExtractorTest { @@ -100,6 +105,7 @@ public class SoundcloudSearchExtractorTest { extractor.fetchPage(); } + // @formatter:off @Override public SearchExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return SoundCloud; } @Override public String expectedName() { return QUERY; } @@ -108,8 +114,8 @@ public class SoundcloudSearchExtractorTest { @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return null; } - @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + // @formatter:on } public static class PagingTest { @@ -128,7 +134,7 @@ public class SoundcloudSearchExtractorTest { private static String urlEncode(String value) { try { - return URLEncoder.encode(value, CHARSET_UTF_8); + return URLEncoder.encode(value, UTF_8); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index 70689e23e..c8f87a901 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -12,15 +12,12 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionItem; import java.io.ByteArrayInputStream; import java.io.FileInputStream; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import static org.schabi.newpipe.FileUtils.resolveTestResource; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; /** * Test for {@link YoutubeSubscriptionExtractor} @@ -56,7 +53,7 @@ public class YoutubeSubscriptionExtractorTest { @Test public void testEmptySourceException() throws Exception { final List items = subscriptionExtractor.fromInputStream( - new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8))); + new ByteArrayInputStream("[]".getBytes(UTF_8))); assertTrue(items.isEmpty()); } @@ -64,7 +61,7 @@ public class YoutubeSubscriptionExtractorTest { public void testSubscriptionWithEmptyTitleInSource() throws Exception { final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]"; final List items = subscriptionExtractor.fromInputStream( - new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); + new ByteArrayInputStream(source.getBytes(UTF_8))); assertEquals(1, items.size()); assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId()); @@ -77,7 +74,7 @@ public class YoutubeSubscriptionExtractorTest { final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," + "{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]"; final List items = subscriptionExtractor.fromInputStream( - new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); + new ByteArrayInputStream(source.getBytes(UTF_8))); assertEquals(1, items.size()); assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId()); @@ -101,7 +98,7 @@ public class YoutubeSubscriptionExtractorTest { for (String invalidContent : invalidList) { try { - byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8); + byte[] bytes = invalidContent.getBytes(UTF_8); subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); fail("Extracting from \"" + invalidContent + "\" didn't throw an exception"); } catch (final Exception e) {