mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2024-12-13 22:00:32 +05:30
use only one constant UTF-8
This commit is contained in:
bopol
parent
50e5718bd1
commit
9ca52ca68a
@ -7,6 +7,8 @@ import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
|
||||
public static final String ALL = "all";
|
||||
public static final String CONFERENCES = "conferences";
|
||||
@ -31,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
|
||||
final String sortFilter) throws ParsingException {
|
||||
try {
|
||||
return "https://media.ccc.de/public/events/search?q="
|
||||
+ URLEncoder.encode(query, "UTF-8");
|
||||
+ URLEncoder.encode(query, UTF_8);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new ParsingException("Could not create search string with query: " + query, e);
|
||||
}
|
||||
|
||||
@ -18,15 +18,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamSegment;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.schabi.newpipe.extractor.stream.*;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
@ -40,6 +32,8 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
private final String baseUrl;
|
||||
private JsonObject json;
|
||||
@ -322,7 +316,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
||||
params.append("start=0&count=8&sort=-createdAt");
|
||||
for (final String tag : tags) {
|
||||
params.append("&tagsOneOf=");
|
||||
params.append(URLEncoder.encode(tag, "UTF-8"));
|
||||
params.append(URLEncoder.encode(tag, UTF_8));
|
||||
}
|
||||
return url + "?" + params.toString();
|
||||
}
|
||||
|
||||
@ -8,9 +8,10 @@ import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
|
||||
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
public static final String VIDEOS = "videos";
|
||||
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
|
||||
public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
|
||||
@ -35,7 +36,7 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
|
||||
public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException {
|
||||
try {
|
||||
final String url = baseUrl + SEARCH_ENDPOINT
|
||||
+ "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
|
||||
+ "?search=" + URLEncoder.encode(searchString, UTF_8);
|
||||
|
||||
return url;
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
|
||||
@ -38,8 +38,7 @@ import java.util.List;
|
||||
import static java.util.Collections.singletonList;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
|
||||
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.*;
|
||||
|
||||
public class SoundcloudParsingHelper {
|
||||
private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20
|
||||
@ -117,7 +116,7 @@ public class SoundcloudParsingHelper {
|
||||
*/
|
||||
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
|
||||
String apiUrl = "https://api-v2.soundcloud.com/resolve"
|
||||
+ "?url=" + URLEncoder.encode(url, "UTF-8")
|
||||
+ "?url=" + URLEncoder.encode(url, UTF_8)
|
||||
+ "&client_id=" + clientId();
|
||||
|
||||
try {
|
||||
@ -136,7 +135,7 @@ public class SoundcloudParsingHelper {
|
||||
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
||||
|
||||
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
|
||||
+ URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody();
|
||||
+ URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
|
||||
|
||||
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
|
||||
}
|
||||
@ -158,7 +157,7 @@ public class SoundcloudParsingHelper {
|
||||
}
|
||||
|
||||
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
|
||||
+ URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody();
|
||||
+ URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
|
||||
// handle playlists / sets different and get playlist id via uir field in JSON
|
||||
if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
|
||||
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
|
||||
|
||||
@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.MetaInfo;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
@ -17,15 +16,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamSegment;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.schabi.newpipe.extractor.stream.*;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
@ -34,10 +28,8 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
@ -73,7 +65,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
@Override
|
||||
public String getTextualUploadDate() {
|
||||
return track.getString("created_at")
|
||||
.replace("T"," ")
|
||||
.replace("T", " ")
|
||||
.replace("Z", "");
|
||||
}
|
||||
|
||||
@ -232,7 +224,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
|
||||
private static String urlEncode(String value) {
|
||||
try {
|
||||
return URLEncoder.encode(value, "UTF-8");
|
||||
return URLEncoder.encode(value, UTF_8);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
|
||||
@ -17,9 +17,9 @@ import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
|
||||
|
||||
public SoundcloudSuggestionExtractor(StreamingService service) {
|
||||
super(service);
|
||||
@ -32,7 +32,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
|
||||
String url = "https://api-v2.soundcloud.com/search/queries"
|
||||
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
|
||||
+ "?q=" + URLEncoder.encode(query, UTF_8)
|
||||
+ "&client_id=" + SoundcloudParsingHelper.clientId()
|
||||
+ "&limit=10";
|
||||
|
||||
|
||||
@ -11,8 +11,9 @@ import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
|
||||
public static final String TRACKS = "tracks";
|
||||
public static final String USERS = "users";
|
||||
@ -43,7 +44,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
|
||||
}
|
||||
}
|
||||
|
||||
return url + "?q=" + URLEncoder.encode(id, CHARSET_UTF_8)
|
||||
return url + "?q=" + URLEncoder.encode(id, UTF_8)
|
||||
+ "&client_id=" + SoundcloudParsingHelper.clientId()
|
||||
+ "&limit=" + ITEMS_PER_PAGE
|
||||
+ "&offset=0";
|
||||
|
||||
@ -124,6 +124,7 @@ public class YoutubeParsingHelper {
|
||||
|
||||
/**
|
||||
* Parses the duration string of the video expecting ":" or "." as separators
|
||||
*
|
||||
* @return the duration in seconds
|
||||
* @throws ParsingException when more than 3 separators are found
|
||||
*/
|
||||
@ -193,6 +194,7 @@ public class YoutubeParsingHelper {
|
||||
/**
|
||||
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
|
||||
* Ids from a YouTube Mix start with "RD"
|
||||
*
|
||||
* @param playlistId
|
||||
* @return Whether given id belongs to a YouTube Mix
|
||||
*/
|
||||
@ -203,15 +205,18 @@ public class YoutubeParsingHelper {
|
||||
/**
|
||||
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
|
||||
* Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
|
||||
*
|
||||
* @param playlistId
|
||||
* @return Whether given id belongs to a YouTube Music Mix
|
||||
*/
|
||||
public static boolean isYoutubeMusicMixId(final String playlistId) {
|
||||
return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
|
||||
* Ids from a YouTube channel Mix start with "RDCM"
|
||||
*
|
||||
* @return Whether given id belongs to a YouTube Channel Mix
|
||||
*/
|
||||
public static boolean isYoutubeChannelMixId(final String playlistId) {
|
||||
@ -220,6 +225,7 @@ public class YoutubeParsingHelper {
|
||||
|
||||
/**
|
||||
* Extracts the video id from the playlist id for Mixes.
|
||||
*
|
||||
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
|
||||
*/
|
||||
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
|
||||
@ -311,7 +317,8 @@ public class YoutubeParsingHelper {
|
||||
clientVersion = contextClientVersion;
|
||||
break;
|
||||
}
|
||||
} catch (Parser.RegexException ignored) { }
|
||||
} catch (Parser.RegexException ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
|
||||
@ -323,7 +330,8 @@ public class YoutubeParsingHelper {
|
||||
} catch (Parser.RegexException e) {
|
||||
try {
|
||||
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
|
||||
} catch (Parser.RegexException ignored) { }
|
||||
} catch (Parser.RegexException ignored) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -390,7 +398,7 @@ public class YoutubeParsingHelper {
|
||||
.end()
|
||||
.value("query", "test")
|
||||
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
|
||||
.end().done().getBytes("UTF-8");
|
||||
.end().done().getBytes(UTF_8);
|
||||
// @formatter:on
|
||||
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
@ -454,7 +462,7 @@ public class YoutubeParsingHelper {
|
||||
if (param.split("=")[0].equals("q")) {
|
||||
String url;
|
||||
try {
|
||||
url = URLDecoder.decode(param.split("=")[1], "UTF-8");
|
||||
url = URLDecoder.decode(param.split("=")[1], UTF_8);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return null;
|
||||
}
|
||||
@ -502,6 +510,7 @@ public class YoutubeParsingHelper {
|
||||
|
||||
/**
|
||||
* Get the text from a JSON object that has either a simpleText or a runs array.
|
||||
*
|
||||
* @param textObject JSON object to get the text from
|
||||
* @param html whether to return HTML, by parsing the navigationEndpoint
|
||||
* @return text in the JSON object or {@code null}
|
||||
@ -729,7 +738,7 @@ public class YoutubeParsingHelper {
|
||||
|
||||
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle"));
|
||||
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text"));
|
||||
if (title == null || text == null) {
|
||||
if (title == null || text == null) {
|
||||
throw new ParsingException("Could not extract clarification renderer content");
|
||||
}
|
||||
metaInfo.setTitle(title);
|
||||
@ -773,6 +782,7 @@ public class YoutubeParsingHelper {
|
||||
/**
|
||||
* Sometimes, YouTube provides URLs which use Google's cache. They look like
|
||||
* {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL}
|
||||
*
|
||||
* @param url the URL which might refer to the Google's webcache
|
||||
* @return the URL which is referring to the original site
|
||||
*/
|
||||
|
||||
@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
|
||||
@ -19,6 +18,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
@ -27,9 +27,8 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
public class YoutubeCommentsExtractor extends CommentsExtractor {
|
||||
@ -152,9 +151,9 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
|
||||
first = false;
|
||||
else
|
||||
result.append("&");
|
||||
result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
|
||||
result.append(URLEncoder.encode(entry.getKey(), UTF_8));
|
||||
result.append("=");
|
||||
result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
|
||||
result.append(URLEncoder.encode(entry.getValue(), UTF_8));
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
@ -1,11 +1,6 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import com.grack.nanojson.*;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.MetaInfo;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
@ -24,24 +19,17 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
|
||||
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
public class YoutubeMusicSearchExtractor extends SearchExtractor {
|
||||
@ -107,7 +95,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
|
||||
.end()
|
||||
.value("query", getSearchString())
|
||||
.value("params", params)
|
||||
.end().done().getBytes("UTF-8");
|
||||
.end().done().getBytes(UTF_8);
|
||||
// @formatter:on
|
||||
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
@ -231,7 +219,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
|
||||
.value("enableSafetyMode", false)
|
||||
.end()
|
||||
.end()
|
||||
.end().done().getBytes("UTF-8");
|
||||
.end().done().getBytes(UTF_8);
|
||||
// @formatter:on
|
||||
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
|
||||
@ -1,11 +1,6 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import com.grack.nanojson.*;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.MetaInfo;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
@ -20,16 +15,15 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/*
|
||||
@ -171,7 +165,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
||||
.object("user").end()
|
||||
.end()
|
||||
.value("continuation", page.getId())
|
||||
.end().done().getBytes("UTF-8");
|
||||
.end().done().getBytes(UTF_8);
|
||||
// @formatter:on
|
||||
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
|
||||
@ -15,6 +15,8 @@ import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 28.09.16.
|
||||
*
|
||||
@ -37,8 +39,6 @@ import java.util.List;
|
||||
|
||||
public class YoutubeSuggestionExtractor extends SuggestionExtractor {
|
||||
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
|
||||
public YoutubeSuggestionExtractor(StreamingService service) {
|
||||
super(service);
|
||||
}
|
||||
@ -52,8 +52,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
|
||||
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
|
||||
+ "&jsonp=" + "JP"
|
||||
+ "&ds=" + "yt"
|
||||
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8)
|
||||
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
|
||||
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
|
||||
+ "&q=" + URLEncoder.encode(query, UTF_8);
|
||||
|
||||
String response = dl.get(url, getExtractorLocalization()).responseBody();
|
||||
// trim JSONP part "JP(...)"
|
||||
|
||||
@ -7,8 +7,9 @@ import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
|
||||
public static final String ALL = "all";
|
||||
public static final String VIDEOS = "videos";
|
||||
@ -37,21 +38,21 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
|
||||
default:
|
||||
break;
|
||||
case VIDEOS:
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D";
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D";
|
||||
case CHANNELS:
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D";
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAg%253D%253D";
|
||||
case PLAYLISTS:
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D";
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAw%253D%253D";
|
||||
case MUSIC_SONGS:
|
||||
case MUSIC_VIDEOS:
|
||||
case MUSIC_ALBUMS:
|
||||
case MUSIC_PLAYLISTS:
|
||||
case MUSIC_ARTISTS:
|
||||
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
|
||||
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new ParsingException("Could not encode query", e);
|
||||
}
|
||||
|
||||
@ -14,6 +14,8 @@ import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 02.02.16.
|
||||
*
|
||||
@ -87,7 +89,7 @@ public class Parser {
|
||||
for (String arg : input.split("&")) {
|
||||
String[] splitArg = arg.split("=");
|
||||
if (splitArg.length > 1) {
|
||||
map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
|
||||
map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8));
|
||||
} else {
|
||||
map.put(splitArg[0], "");
|
||||
}
|
||||
|
||||
@ -6,16 +6,13 @@ import java.io.UnsupportedEncodingException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.net.URLDecoder;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
|
||||
public class Utils {
|
||||
|
||||
public static final String HTTP = "http://";
|
||||
public static final String HTTPS = "https://";
|
||||
public static final String UTF_8 = "UTF-8";
|
||||
|
||||
private Utils() {
|
||||
//no instance
|
||||
@ -117,7 +114,7 @@ public class Utils {
|
||||
|
||||
String query;
|
||||
try {
|
||||
query = URLDecoder.decode(params[0], "UTF-8");
|
||||
query = URLDecoder.decode(params[0], UTF_8);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
|
||||
e.printStackTrace();
|
||||
@ -126,7 +123,7 @@ public class Utils {
|
||||
|
||||
if (query.equals(parameterName)) {
|
||||
try {
|
||||
return URLDecoder.decode(params[1], "UTF-8");
|
||||
return URLDecoder.decode(params[1], UTF_8);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
|
||||
e.printStackTrace();
|
||||
@ -200,6 +197,7 @@ public class Utils {
|
||||
/**
|
||||
* If the provided url is a Google search redirect, then the actual url is extracted from the
|
||||
* {@code url=} query value and returned, otherwise the original url is returned.
|
||||
*
|
||||
* @param url the url which can possibly be a Google search redirect
|
||||
* @return an url with no Google search redirects
|
||||
*/
|
||||
@ -208,7 +206,7 @@ public class Utils {
|
||||
try {
|
||||
final URL decoded = Utils.stringToURL(url);
|
||||
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
|
||||
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8");
|
||||
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
|
||||
}
|
||||
} catch (final Exception ignored) {
|
||||
}
|
||||
@ -231,7 +229,7 @@ public class Utils {
|
||||
return map == null || map.isEmpty();
|
||||
}
|
||||
|
||||
public static boolean isWhitespace(final int c){
|
||||
public static boolean isWhitespace(final int c) {
|
||||
return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r';
|
||||
}
|
||||
|
||||
@ -263,10 +261,10 @@ public class Utils {
|
||||
}
|
||||
|
||||
public static String join(final String delimiter, final String mapJoin,
|
||||
final Map<? extends CharSequence, ? extends CharSequence> elements) {
|
||||
final Map<? extends CharSequence, ? extends CharSequence> elements) {
|
||||
final List<String> list = new LinkedList<>();
|
||||
for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
|
||||
.entrySet()) {
|
||||
.entrySet()) {
|
||||
list.add(entry.getKey() + mapJoin + entry.getValue());
|
||||
}
|
||||
return join(delimiter, list);
|
||||
|
||||
@ -19,6 +19,7 @@ import static java.util.Collections.singletonList;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
|
||||
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
|
||||
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
public class SoundcloudSearchExtractorTest {
|
||||
|
||||
@ -33,6 +34,7 @@ public class SoundcloudSearchExtractorTest {
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
// @formatter:off
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return SoundCloud; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@ -41,6 +43,7 @@ public class SoundcloudSearchExtractorTest {
|
||||
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
public static class Tracks extends DefaultSearchExtractorTest {
|
||||
@ -54,6 +57,7 @@ public class SoundcloudSearchExtractorTest {
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
// @formatter:off
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return SoundCloud; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@ -62,8 +66,8 @@ public class SoundcloudSearchExtractorTest {
|
||||
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
public static class Users extends DefaultSearchExtractorTest {
|
||||
@ -77,6 +81,7 @@ public class SoundcloudSearchExtractorTest {
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
// @formatter:off
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return SoundCloud; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@ -85,8 +90,8 @@ public class SoundcloudSearchExtractorTest {
|
||||
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
public static class Playlists extends DefaultSearchExtractorTest {
|
||||
@ -100,6 +105,7 @@ public class SoundcloudSearchExtractorTest {
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
// @formatter:off
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return SoundCloud; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@ -108,8 +114,8 @@ public class SoundcloudSearchExtractorTest {
|
||||
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
public static class PagingTest {
|
||||
@ -128,7 +134,7 @@ public class SoundcloudSearchExtractorTest {
|
||||
|
||||
private static String urlEncode(String value) {
|
||||
try {
|
||||
return URLEncoder.encode(value, CHARSET_UTF_8);
|
||||
return URLEncoder.encode(value, UTF_8);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
@ -12,15 +12,12 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.junit.Assert.*;
|
||||
import static org.schabi.newpipe.FileUtils.resolveTestResource;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
|
||||
/**
|
||||
* Test for {@link YoutubeSubscriptionExtractor}
|
||||
@ -56,7 +53,7 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
@Test
|
||||
public void testEmptySourceException() throws Exception {
|
||||
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
|
||||
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
|
||||
new ByteArrayInputStream("[]".getBytes(UTF_8)));
|
||||
assertTrue(items.isEmpty());
|
||||
}
|
||||
|
||||
@ -64,7 +61,7 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
|
||||
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
|
||||
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
|
||||
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
|
||||
new ByteArrayInputStream(source.getBytes(UTF_8)));
|
||||
|
||||
assertEquals(1, items.size());
|
||||
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
|
||||
@ -77,7 +74,7 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
|
||||
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
|
||||
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
|
||||
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
|
||||
new ByteArrayInputStream(source.getBytes(UTF_8)));
|
||||
|
||||
assertEquals(1, items.size());
|
||||
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
|
||||
@ -101,7 +98,7 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
|
||||
for (String invalidContent : invalidList) {
|
||||
try {
|
||||
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
|
||||
byte[] bytes = invalidContent.getBytes(UTF_8);
|
||||
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
|
||||
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
|
||||
} catch (final Exception e) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user