kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2024-12-14 22:30:33 +05:30
Code Issues Projects Releases Wiki Activity

Merge pull request #532 from B0pol/utf8

Browse Source 
Code improvement
This commit is contained in:
bopol 2021-02-15 18:56:28 +01:00 committed by GitHub
commit 416cf17b7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 195 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -14,6 +14,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class KioskList {
public interface KioskExtractorFactory {
@ -70,7 +72,7 @@ public class KioskList {
public KioskExtractor getDefaultKioskExtractor(Page nextPage, Localization localization)
throws ExtractionException, IOException {
if (defaultKiosk != null && !defaultKiosk.equals("")) {
if (!isNullOrEmpty(defaultKiosk)) {
return getExtractorById(defaultKiosk, nextPage, localization);
} else {
if (!kioskList.isEmpty()) {

12
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java
View File

@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
///////////////////////////////////
@ -12,7 +14,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
///////////////////////////////////
@Override
public abstract String getUrl(String querry, List<String> contentFilter, String sortFilter) throws ParsingException;
public abstract String getUrl(String query, List<String> contentFilter, String sortFilter) throws ParsingException;
public String getSearchString(String url) {
return "";
@ -28,14 +30,14 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
}
@Override
public SearchQueryHandler fromQuery(String querry,
public SearchQueryHandler fromQuery(String query,
List<String> contentFilter,
String sortFilter) throws ParsingException {
return new SearchQueryHandler(super.fromQuery(querry, contentFilter, sortFilter));
return new SearchQueryHandler(super.fromQuery(query, contentFilter, sortFilter));
}
public SearchQueryHandler fromQuery(String querry) throws ParsingException {
return fromQuery(querry, new ArrayList<String>(0), "");
public SearchQueryHandler fromQuery(String query) throws ParsingException {
return fromQuery(query, new ArrayList<String>(0), EMPTY_STRING);
}
/**

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
View File

@ -7,6 +7,8 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String ALL = "all";
public static final String CONFERENCES = "conferences";
@ -31,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
final String sortFilter) throws ParsingException {
try {
return "https://media.ccc.de/public/events/search?q="
+ URLEncoder.encode(query, "UTF-8");
+ URLEncoder.encode(query, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not create search string with query: " + query, e);
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java
View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.ServiceList;
@ -13,6 +12,8 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.util.Objects;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject item;
private final String url;
@ -68,7 +69,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
final Document doc = Jsoup.parse(htmlText);
return doc.body().text();
} catch (Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", EMPTY_STRING);
}
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -18,15 +18,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
@ -40,6 +32,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json;
@ -322,7 +316,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
params.append("start=0&count=8&sort=-createdAt");
for (final String tag : tags) {
params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8"));
params.append(URLEncoder.encode(tag, UTF_8));
}
return url + "?" + params.toString();
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
View File

@ -8,9 +8,10 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
@ -35,7 +36,7 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException {
try {
final String url = baseUrl + SEARCH_ENDPOINT
+ "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
+ "?search=" + URLEncoder.encode(searchString, UTF_8);
return url;
} catch (UnsupportedEncodingException e) {

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -37,9 +37,7 @@ import java.util.List;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20
@ -117,7 +115,7 @@ public class SoundcloudParsingHelper {
*/
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
String apiUrl = "https://api-v2.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "?url=" + URLEncoder.encode(url, UTF_8)
+ "&client_id=" + clientId();
try {
@ -136,7 +134,7 @@ public class SoundcloudParsingHelper {
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody();
+ URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
}
@ -158,7 +156,7 @@ public class SoundcloudParsingHelper {
}
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody();
+ URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
// handle playlists / sets different and get playlist id via uir field in JSON
if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -15,11 +14,10 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java
View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -4,7 +4,7 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -4,13 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -19,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
@ -26,10 +21,8 @@ import java.net.URL;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudSearchExtractor extends SearchExtractor {

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
@ -17,15 +16,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -34,11 +28,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
@ -60,7 +50,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getId() {
return track.getInt("id") + "";
return track.getInt("id") + EMPTY_STRING;
}
@Nonnull
@ -73,8 +63,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public String getTextualUploadDate() {
return track.getString("created_at")
.replace("T"," ")
.replace("Z", "");
.replace("T", " ")
.replace("Z", EMPTY_STRING);
}
@Nonnull
@ -232,7 +222,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
private static String urlEncode(String value) {
try {
return URLEncoder.encode(value, "UTF-8");
return URLEncoder.encode(value, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new IllegalStateException(e);
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
View File

@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java
View File

@ -17,9 +17,9 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public static final String CHARSET_UTF_8 = "UTF-8";
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public SoundcloudSuggestionExtractor(StreamingService service) {
super(service);
@ -32,7 +32,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search/queries"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "?q=" + URLEncoder.encode(query, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java
View File

@ -11,8 +11,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String TRACKS = "tracks";
public static final String USERS = "users";
@ -43,7 +44,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
}
}
return url + "?q=" + URLEncoder.encode(id, CHARSET_UTF_8)
return url + "?q=" + URLEncoder.encode(id, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=" + ITEMS_PER_PAGE
+ "&offset=0";

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -33,7 +33,6 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*;
/*
@ -124,6 +123,7 @@ public class YoutubeParsingHelper {
/**
* Parses the duration string of the video expecting ":" or "." as separators
*
* @return the duration in seconds
* @throws ParsingException when more than 3 separators are found
*/
@ -193,6 +193,7 @@ public class YoutubeParsingHelper {
/**
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD"
*
* @param playlistId
* @return Whether given id belongs to a YouTube Mix
*/
@ -203,15 +204,18 @@ public class YoutubeParsingHelper {
/**
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
*
* @param playlistId
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
}
/**
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
* Ids from a YouTube channel Mix start with "RDCM"
*
* @return Whether given id belongs to a YouTube Channel Mix
*/
public static boolean isYoutubeChannelMixId(final String playlistId) {
@ -220,6 +224,7 @@ public class YoutubeParsingHelper {
/**
* Extracts the video id from the playlist id for Mixes.
*
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
@ -311,7 +316,8 @@ public class YoutubeParsingHelper {
clientVersion = contextClientVersion;
break;
}
} catch (Parser.RegexException ignored) { }
} catch (Parser.RegexException ignored) {
}
}
if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
@ -323,7 +329,8 @@ public class YoutubeParsingHelper {
} catch (Parser.RegexException e) {
try {
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException ignored) { }
} catch (Parser.RegexException ignored) {
}
}
}
@ -390,7 +397,7 @@ public class YoutubeParsingHelper {
.end()
.value("query", "test")
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
@ -454,7 +461,7 @@ public class YoutubeParsingHelper {
if (param.split("=")[0].equals("q")) {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], "UTF-8");
url = URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (UnsupportedEncodingException e) {
return null;
}
@ -502,6 +509,7 @@ public class YoutubeParsingHelper {
/**
* Get the text from a JSON object that has either a simpleText or a runs array.
*
* @param textObject JSON object to get the text from
* @param html whether to return HTML, by parsing the navigationEndpoint
* @return text in the JSON object or {@code null}
@ -729,7 +737,7 @@ public class YoutubeParsingHelper {
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle"));
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text"));
if (title == null || text == null) {
if (title == null || text == null) {
throw new ParsingException("Could not extract clarification renderer content");
}
metaInfo.setTitle(title);
@ -773,6 +781,7 @@ public class YoutubeParsingHelper {
/**
* Sometimes, YouTube provides URLs which use Google's cache. They look like
* {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL}
*
* @param url the URL which might refer to the Google's webcache
* @return the URL which is referring to the original site
*/

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -21,7 +21,7 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@ -19,6 +18,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -27,9 +27,8 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeCommentsExtractor extends CommentsExtractor {
@ -152,9 +151,9 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
first = false;
else
result.append("&");
result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
result.append(URLEncoder.encode(entry.getKey(), UTF_8));
result.append("=");
result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
result.append(URLEncoder.encode(entry.getValue(), UTF_8));
}
return result.toString();
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -13,6 +12,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
@ -46,7 +46,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}
@ -86,7 +86,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
if (contentText.isEmpty()) {
// completely empty comments as described in
// https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
return "";
return EMPTY_STRING;
}
final String commentText = getTextFromObject(contentText);
// youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
@ -130,7 +130,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}
@ -139,7 +139,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}

28
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -1,11 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import com.grack.nanojson.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
@ -24,25 +19,16 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData;
@ -107,7 +93,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.end()
.value("query", getSearchString())
.value("params", params)
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
@ -231,7 +217,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.value("enableSafetyMode", false)
.end()
.end()
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -82,7 +82,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null && !name.isEmpty()) return name;
if (!isNullOrEmpty(name)) return name;
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -1,11 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import com.grack.nanojson.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
@ -20,16 +15,15 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.util.*;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
@ -171,7 +165,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.object("user").end()
.end()
.value("continuation", page.getId())
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -29,17 +28,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -47,26 +36,13 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
@ -102,17 +78,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/
@Nullable private static String cachedDeobfuscationCode = null;
@Nullable private String playerJsUrl = null;
@Nullable
private static String cachedDeobfuscationCode = null;
@Nullable
private String playerJsUrl = null;
private JsonArray initialAjaxJson;
private JsonObject initialData;
@Nonnull private final Map<String, String> videoInfoPage = new HashMap<>();
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit = -1;
@Nullable private List<SubtitlesStream> subtitles = null;
@Nullable
private List<SubtitlesStream> subtitles = null;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
@ -787,8 +767,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
@ -1138,6 +1116,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents"));
.getObject("results").getObject("results").getArray("contents"));
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -17,10 +17,8 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -15,6 +15,8 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/*
* Created by Christian Schabesberger on 28.09.16.
*
@ -37,8 +39,6 @@ import java.util.List;
public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSuggestionExtractor(StreamingService service) {
super(service);
}
@ -52,8 +52,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
+ "&jsonp=" + "JP"
+ "&ds=" + "yt"
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
+ "&q=" + URLEncoder.encode(query, UTF_8);
String response = dl.get(url, getExtractorLocalization()).responseBody();
// trim JSONP part "JP(...)"

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
View File

@ -7,8 +7,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String ALL = "all";
public static final String VIDEOS = "videos";
@ -37,21 +38,21 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
default:
break;
case VIDEOS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D";
case CHANNELS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAg%253D%253D";
case PLAYLISTS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAw%253D%253D";
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS:
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
}
}
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java
View File

@ -1,13 +1,16 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
import java.util.Objects;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class Description implements Serializable {
public static final int HTML = 1;
public static final int MARKDOWN = 2;
public static final int PLAIN_TEXT = 3;
public static final Description emptyDescription = new Description("", PLAIN_TEXT);
public static final Description emptyDescription = new Description(EMPTY_STRING, PLAIN_TEXT);
private String content;
private int type;
@ -28,4 +31,17 @@ public class Description implements Serializable {
public int getType() {
return type;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Description that = (Description) o;
return type == that.type && Objects.equals(content, that.content);
}
@Override
public int hashCode() {
return Objects.hash(content, type);
}
}

1
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View File

@ -13,7 +13,6 @@ import java.util.List;
public class JsonUtils {
public static final JsonObject EMPTY_OBJECT = new JsonObject();
public static final JsonArray EMPTY_ARRAY = new JsonArray();
public static final String EMPTY_STRING = "";
private JsonUtils() {
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
View File

@ -14,6 +14,8 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/*
* Created by Christian Schabesberger on 02.02.16.
*
@ -87,7 +89,7 @@ public class Parser {
for (String arg : input.split("&")) {
String[] splitArg = arg.split("=");
if (splitArg.length > 1) {
map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8));
} else {
map.put(splitArg[0], "");
}

23
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -6,16 +6,14 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.*;
public class Utils {
public static final String HTTP = "http://";
public static final String HTTPS = "https://";
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private Utils() {
//no instance
@ -117,7 +115,7 @@ public class Utils {
String query;
try {
query = URLDecoder.decode(params[0], "UTF-8");
query = URLDecoder.decode(params[0], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
@ -126,7 +124,7 @@ public class Utils {
if (query.equals(parameterName)) {
try {
return URLDecoder.decode(params[1], "UTF-8");
return URLDecoder.decode(params[1], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
@ -200,6 +198,7 @@ public class Utils {
/**
* If the provided url is a Google search redirect, then the actual url is extracted from the
* {@code url=} query value and returned, otherwise the original url is returned.
*
* @param url the url which can possibly be a Google search redirect
* @return an url with no Google search redirects
*/
@ -208,7 +207,7 @@ public class Utils {
try {
final URL decoded = Utils.stringToURL(url);
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8");
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
}
} catch (final Exception ignored) {
}
@ -231,12 +230,12 @@ public class Utils {
return map == null || map.isEmpty();
}
public static boolean isWhitespace(final int c){
public static boolean isWhitespace(final int c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r';
}
public static boolean isBlank(final String string) {
if (string == null || string.isEmpty()) {
if (isNullOrEmpty(string)) {
return true;
}
@ -263,10 +262,10 @@ public class Utils {
}
public static String join(final String delimiter, final String mapJoin,
final Map<? extends CharSequence, ? extends CharSequence> elements) {
final Map<? extends CharSequence, ? extends CharSequence> elements) {
final List<String> list = new LinkedList<>();
for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
.entrySet()) {
.entrySet()) {
list.add(entry.getKey() + mapJoin + entry.getValue());
}
return join(delimiter, list);

14
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View File

@ -19,6 +19,7 @@ import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchExtractorTest {
@ -33,6 +34,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@ -41,6 +43,7 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
// @formatter:on
}
public static class Tracks extends DefaultSearchExtractorTest {
@ -54,6 +57,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@ -62,8 +66,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
// @formatter:on
}
public static class Users extends DefaultSearchExtractorTest {
@ -77,6 +81,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@ -85,8 +90,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
// @formatter:on
}
public static class Playlists extends DefaultSearchExtractorTest {
@ -100,6 +105,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@ -108,8 +114,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
// @formatter:on
}
public static class PagingTest {
@ -128,7 +134,7 @@ public class SoundcloudSearchExtractorTest {
private static String urlEncode(String value) {
try {
return URLEncoder.encode(value, CHARSET_UTF_8);
return URLEncoder.encode(value, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View File

@ -12,15 +12,12 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assert.*;
import static org.schabi.newpipe.FileUtils.resolveTestResource;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/**
* Test for {@link YoutubeSubscriptionExtractor}
@ -56,7 +53,7 @@ public class YoutubeSubscriptionExtractorTest {
@Test
public void testEmptySourceException() throws Exception {
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream("[]".getBytes(UTF_8)));
assertTrue(items.isEmpty());
}
@ -64,7 +61,7 @@ public class YoutubeSubscriptionExtractorTest {
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream(source.getBytes(UTF_8)));
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
@ -77,7 +74,7 @@ public class YoutubeSubscriptionExtractorTest {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream(source.getBytes(UTF_8)));
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
@ -101,7 +98,7 @@ public class YoutubeSubscriptionExtractorTest {
for (String invalidContent : invalidList) {
try {
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
byte[] bytes = invalidContent.getBytes(UTF_8);
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} catch (final Exception e) {

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
View File

@ -31,6 +31,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplica
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeSearchExtractorTest {

27
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java
View File

@ -7,6 +7,7 @@ import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeSearchQHTest {
@ -19,37 +20,37 @@ public class YoutubeSearchQHTest {
assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
}
@Test
public void testGetContentFilter() throws Exception {
assertEquals(VIDEOS, YouTube.getSearchQHFactory()
.fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0));
.fromQuery(EMPTY_STRING, asList(new String[]{VIDEOS}), EMPTY_STRING).getContentFilters().get(0));
assertEquals(CHANNELS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0));
.fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getContentFilters().get(0));
assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0));
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getContentFilters().get(0));
}
@Test
public void testWithContentfilter() throws Exception {
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAQ%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{VIDEOS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAg%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAw%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl());
.fromQuery("asdf", asList(new String[]{"fjiijie"}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
}
@Test

20
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
@ -23,11 +24,10 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
/*
* Created by Christian Schabesberger on 30.12.15.
@ -88,6 +88,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; }
@ -110,6 +111,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 5212900; }
@Override public long expectedDislikeCountAtLeast() { return 30600; }
@Override public int expectedStreamSegmentsCount() { return 0; }
// @formatter:on
}
public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
@ -125,6 +127,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "This Smartphone Changes Everything..."; }
@ -147,13 +150,14 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; }
@Override public long expectedLikeCountAtLeast() { return 340100; }
@Override public long expectedDislikeCountAtLeast() { return 18700; }
// @formatter:on
@Override
@Test
@Ignore("TODO fix")
public void testDescription() throws Exception {
super.testDescription();
}
}
@Ignore("TODO fix")
@ -171,6 +175,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; }
@ -189,6 +194,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
// @formatter:on
}
public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest {
@ -205,6 +211,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; }
@ -235,6 +242,7 @@ public class YoutubeStreamExtractorDefaultTest {
assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
// @formatter:on
}
public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
@ -304,6 +312,7 @@ public class YoutubeStreamExtractorDefaultTest {
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; }
@ -324,13 +333,13 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedDislikeCountAtLeast() { return 500; }
@Override public List<MetaInfo> expectedMetaInfo() throws MalformedURLException {
return Collections.singletonList(new MetaInfo(
"",
EMPTY_STRING,
new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT),
Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")),
Collections.singletonList("Wikipedia (German)")
));
}
// @formatter:on
@Override
@Ignore("TODO fix")
@Test
@ -344,6 +353,7 @@ public class YoutubeStreamExtractorDefaultTest {
public void testMetaInfo() throws Exception {
super.testMetaInfo();
}
}
}