mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2024-12-14 22:30:33 +05:30
merged upstream/dev
This commit is contained in:
yausername
commit
d7582734e7
@ -7,15 +7,21 @@ import java.util.Map;
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class DownloadResponse {
|
||||
private final int responseCode;
|
||||
private final String responseBody;
|
||||
private final Map<String, List<String>> responseHeaders;
|
||||
|
||||
public DownloadResponse(String responseBody, Map<String, List<String>> headers) {
|
||||
public DownloadResponse(int responseCode, String responseBody, Map<String, List<String>> headers) {
|
||||
super();
|
||||
this.responseCode = responseCode;
|
||||
this.responseBody = responseBody;
|
||||
this.responseHeaders = headers;
|
||||
}
|
||||
|
||||
public int getResponseCode() {
|
||||
return responseCode;
|
||||
}
|
||||
|
||||
public String getResponseBody() {
|
||||
return responseBody;
|
||||
}
|
||||
@ -33,5 +39,4 @@ public class DownloadResponse {
|
||||
else
|
||||
return cookies;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -60,6 +60,11 @@ public interface Downloader {
|
||||
*/
|
||||
String download(String siteUrl) throws IOException, ReCaptchaException;
|
||||
|
||||
DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException;
|
||||
|
||||
DownloadResponse get(String siteUrl, Localization localization)
|
||||
throws IOException, ReCaptchaException;
|
||||
|
||||
DownloadResponse get(String siteUrl, DownloadRequest request)
|
||||
throws IOException, ReCaptchaException;
|
||||
|
||||
|
||||
@ -21,7 +21,14 @@ package org.schabi.newpipe.extractor.exceptions;
|
||||
*/
|
||||
|
||||
public class ReCaptchaException extends ExtractionException {
|
||||
public ReCaptchaException(String message) {
|
||||
private String url;
|
||||
|
||||
public ReCaptchaException(String message, String url) {
|
||||
super(message);
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
@ -24,7 +24,9 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() {
|
||||
return itemObject.getString("avatar_url", "");
|
||||
String avatarUrl = itemObject.getString("avatar_url", "");
|
||||
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
|
||||
return avatarUrlBetterResolution;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -7,9 +7,12 @@ import com.grack.nanojson.JsonParserException;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.schabi.newpipe.extractor.DownloadResponse;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
@ -21,40 +24,59 @@ import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudParsingHelper {
|
||||
private static final String HARDCODED_CLIENT_ID = "LHzSAKe8eP9Yy3FgBugfBapRPLncO6Ng"; // Updated on 22/10/19
|
||||
private static String clientId;
|
||||
|
||||
private SoundcloudParsingHelper() {
|
||||
}
|
||||
|
||||
public static String clientId() throws ReCaptchaException, IOException, RegexException {
|
||||
public static String clientId() throws ExtractionException, IOException {
|
||||
if (clientId != null && !clientId.isEmpty()) return clientId;
|
||||
|
||||
Downloader dl = NewPipe.getDownloader();
|
||||
String response = dl.download("https://soundcloud.com");
|
||||
|
||||
Document doc = Jsoup.parse(response);
|
||||
Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
|
||||
|
||||
final String clientIdPattern = ",client_id:\"(.*?)\"";
|
||||
|
||||
try {
|
||||
final HashMap<String, String> headers = new HashMap<>();
|
||||
headers.put("Range", "bytes=0-16384");
|
||||
String js = dl.download(jsElement.attr("src"), headers);
|
||||
|
||||
return clientId = Parser.matchGroup1(clientIdPattern, js);
|
||||
} catch (IOException | RegexException ignored) {
|
||||
// Ignore it and proceed to download the whole js file
|
||||
clientId = HARDCODED_CLIENT_ID;
|
||||
if (checkIfHardcodedClientIdIsValid(dl)) {
|
||||
return clientId;
|
||||
}
|
||||
|
||||
String js = dl.download(jsElement.attr("src"));
|
||||
return clientId = Parser.matchGroup1(clientIdPattern, js);
|
||||
final DownloadResponse download = dl.get("https://soundcloud.com");
|
||||
String response = download.getResponseBody();
|
||||
final String clientIdPattern = ",client_id:\"(.*?)\"";
|
||||
|
||||
Document doc = Jsoup.parse(response);
|
||||
final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
|
||||
// The one containing the client id will likely be the last one
|
||||
Collections.reverse(possibleScripts);
|
||||
|
||||
final HashMap<String, String> headers = new HashMap<>();
|
||||
headers.put("Range", "bytes=0-16384");
|
||||
|
||||
for (Element element : possibleScripts) {
|
||||
final String srcUrl = element.attr("src");
|
||||
if (srcUrl != null && !srcUrl.isEmpty()) {
|
||||
try {
|
||||
return clientId = Parser.matchGroup1(clientIdPattern, dl.download(srcUrl, headers));
|
||||
} catch (RegexException ignored) {
|
||||
// Ignore it and proceed to try searching other script
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Officially give up
|
||||
throw new ExtractionException("Couldn't extract client id");
|
||||
}
|
||||
|
||||
static boolean checkIfHardcodedClientIdIsValid(Downloader dl) throws IOException, ReCaptchaException {
|
||||
final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID;
|
||||
// Should return 200 to indicate that the client id is valid, a 401 is returned otherwise.
|
||||
return dl.head(apiUrl).getResponseCode() == 200;
|
||||
}
|
||||
|
||||
public static String toDateString(String time) throws ParsingException {
|
||||
@ -79,7 +101,7 @@ public class SoundcloudParsingHelper {
|
||||
*
|
||||
* See https://developers.soundcloud.com/docs/api/reference#resolve
|
||||
*/
|
||||
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
|
||||
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
|
||||
String apiUrl = "https://api.soundcloud.com/resolve"
|
||||
+ "?url=" + URLEncoder.encode(url, "UTF-8")
|
||||
+ "&client_id=" + clientId();
|
||||
@ -113,7 +135,10 @@ public class SoundcloudParsingHelper {
|
||||
|
||||
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
|
||||
+ URLEncoder.encode(url, "UTF-8"));
|
||||
return Parser.matchGroup1(",\"id\":(.*?),", response);
|
||||
// handle playlists / sets different and get playlist id via uir field in JSON
|
||||
if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
|
||||
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
|
||||
return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -71,13 +71,15 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
||||
final String thumbnailUrl = item.getThumbnailUrl();
|
||||
if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
|
||||
|
||||
return thumbnailUrl;
|
||||
String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg");
|
||||
return thumbnailUrlBetterResolution;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
return artworkUrl;
|
||||
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
|
||||
return artworkUrlBetterResolution;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -32,7 +32,10 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
|
||||
// Over-engineering at its finest
|
||||
if (itemObject.isString(ARTWORK_URL_KEY)) {
|
||||
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
|
||||
if (!artworkUrl.isEmpty()) return artworkUrl;
|
||||
if (!artworkUrl.isEmpty()) {
|
||||
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
|
||||
return artworkUrlBetterResolution;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
@ -42,8 +45,11 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
|
||||
|
||||
// First look for track artwork url
|
||||
if (trackObject.isString(ARTWORK_URL_KEY)) {
|
||||
final String url = trackObject.getString(ARTWORK_URL_KEY, "");
|
||||
if (!url.isEmpty()) return url;
|
||||
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, "");
|
||||
if (!artworkUrl.isEmpty()) {
|
||||
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
|
||||
return artworkUrlBetterResolution;
|
||||
}
|
||||
}
|
||||
|
||||
// Then look for track creator avatar url
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
|
||||
@ -48,10 +49,10 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
|
||||
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new ParsingException("Could not encode query", e);
|
||||
} catch (IOException e) {
|
||||
throw new ParsingException("Could not get client id", e);
|
||||
} catch (ReCaptchaException e) {
|
||||
throw new ParsingException("ReCaptcha required", e);
|
||||
} catch (IOException | ExtractionException e) {
|
||||
throw new ParsingException("Could not get client id", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -95,6 +95,7 @@ public class SoundcloudService extends StreamingService {
|
||||
try {
|
||||
list.addKioskEntry(chartsFactory, h, "Top 50");
|
||||
list.addKioskEntry(chartsFactory, h, "New & hot");
|
||||
list.setDefaultKiosk("New & hot");
|
||||
} catch (Exception e) {
|
||||
throw new ExtractionException(e);
|
||||
}
|
||||
|
||||
@ -57,7 +57,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getThumbnailUrl() {
|
||||
return track.getString("artwork_url", "");
|
||||
String artworkUrl = track.getString("artwork_url", "");
|
||||
if (artworkUrl.isEmpty()) {
|
||||
artworkUrl = track.getObject("user").getString("avatar_url", "");
|
||||
}
|
||||
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
|
||||
return artworkUrlBetterResolution;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
||||
@ -52,7 +52,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() {
|
||||
return itemObject.getString("artwork_url");
|
||||
String artworkUrl = itemObject.getString("artwork_url", "");
|
||||
if (artworkUrl.isEmpty()) {
|
||||
artworkUrl = itemObject.getObject("user").getString("avatar_url");
|
||||
}
|
||||
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
|
||||
return artworkUrlBetterResolution;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.DownloadResponse;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
@ -14,6 +15,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
|
||||
@ -47,8 +49,9 @@ import java.util.ArrayList;
|
||||
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
|
||||
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
|
||||
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
|
||||
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en";
|
||||
|
||||
private Document doc;
|
||||
|
||||
@ -59,8 +62,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
||||
String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
|
||||
String pageContent = downloader.download(channelUrl);
|
||||
doc = Jsoup.parse(pageContent, channelUrl);
|
||||
final DownloadResponse response = downloader.get(channelUrl);
|
||||
doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -72,7 +75,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
try {
|
||||
return "https://www.youtube.com/channel/" + getId();
|
||||
return CHANNEL_URL_BASE + getId();
|
||||
} catch (ParsingException e) {
|
||||
return super.getUrl();
|
||||
}
|
||||
@ -81,6 +84,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
try {
|
||||
return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
|
||||
} catch (Exception ignored) {}
|
||||
|
||||
// fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
|
||||
try {
|
||||
Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
|
||||
if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
|
||||
@ -134,10 +142,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
|
||||
@Override
|
||||
public long getSubscriberCount() throws ParsingException {
|
||||
|
||||
final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
|
||||
if (el != null) {
|
||||
String elTitle = el.attr("title");
|
||||
try {
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
|
||||
return Utils.mixedNumberWordToLong(elTitle);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new ParsingException("Could not get subscriber count", e);
|
||||
}
|
||||
|
||||
@ -5,6 +5,9 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 12.02.17.
|
||||
*
|
||||
@ -53,8 +56,26 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
||||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
|
||||
.attr("abs:href");
|
||||
try {
|
||||
String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
|
||||
.attr("abs:data-href");
|
||||
|
||||
Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
|
||||
Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
|
||||
|
||||
if (match.matches()) {
|
||||
return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
|
||||
}
|
||||
} catch(Exception ignored) {}
|
||||
|
||||
// fallback method for channels without "Subscribe" button (or just in case yt changes things)
|
||||
// provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail
|
||||
try {
|
||||
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
|
||||
.attr("abs:href");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get channel url", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -38,7 +38,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
|
||||
try {
|
||||
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get author name", e);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
@ -95,7 +95,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
|
||||
try {
|
||||
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get author name", e);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
@ -104,7 +104,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
|
||||
try {
|
||||
return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get author endpoint", e);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParserException;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.DownloadResponse;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
@ -35,8 +36,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
||||
String pageContent = downloader.download(getUrl());
|
||||
doc = Jsoup.parse(pageContent, getUrl());
|
||||
final String url = getUrl();
|
||||
final DownloadResponse response = downloader.get(url);
|
||||
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -50,7 +52,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
try {
|
||||
return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist name");
|
||||
throw new ParsingException("Could not get playlist name", e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -59,7 +61,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
try {
|
||||
return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist thumbnail");
|
||||
throw new ParsingException("Could not get playlist thumbnail", e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -72,9 +74,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
try {
|
||||
return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
|
||||
return YoutubeChannelExtractor.CHANNEL_URL_BASE +
|
||||
doc.select("button[class*=\"yt-uix-subscription-button\"]")
|
||||
.first().attr("data-channel-external-id");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist uploader name");
|
||||
throw new ParsingException("Could not get playlist uploader url", e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -83,7 +87,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
try {
|
||||
return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist uploader name");
|
||||
throw new ParsingException("Could not get playlist uploader name", e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -92,7 +96,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
try {
|
||||
return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playlist uploader avatar");
|
||||
throw new ParsingException("Could not get playlist uploader avatar", e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -248,6 +252,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
// this url is not always in the form "/channel/..."
|
||||
// sometimes Youtube provides urls in the from "/user/..."
|
||||
return getUploaderLink().attr("abs:href");
|
||||
}
|
||||
|
||||
|
||||
@ -49,10 +49,11 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
try {
|
||||
final Element div = el.select("div[class=\"yt-lockup-meta\"]").first();
|
||||
final Element a = el.select("div[class=\"yt-lockup-meta\"]")
|
||||
.select("ul[class=\"yt-lockup-meta-info\"]")
|
||||
.select("li").select("a").first();
|
||||
|
||||
if(div != null) {
|
||||
final Element a = div.select("a").first();
|
||||
if(a != null) {
|
||||
return a.attr("abs:href");
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.DownloadResponse;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
@ -12,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
@ -52,13 +54,9 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
||||
final String site;
|
||||
final String url = getUrl();
|
||||
//String url = builder.build().toString();
|
||||
//if we've been passed a valid language code, append it to the URL
|
||||
site = downloader.download(url, getLocalization());
|
||||
|
||||
doc = Jsoup.parse(site, url);
|
||||
final DownloadResponse response = downloader.get(url, getLocalization());
|
||||
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.*;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
@ -30,11 +31,13 @@ import java.io.UnsupportedEncodingException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 06.08.15.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
|
||||
* Copyright (C) Christian Schabesberger 2019 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeStreamExtractor.java is part of NewPipe.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
@ -64,12 +67,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
public class GemaException extends ContentNotAvailableException {
|
||||
GemaException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
public class SubtitlesException extends ContentNotAvailableException {
|
||||
SubtitlesException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
@ -83,6 +80,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
private JsonObject playerArgs;
|
||||
@Nonnull
|
||||
private final Map<String, String> videoInfoPage = new HashMap<>();
|
||||
private JsonObject playerResponse;
|
||||
|
||||
@Nonnull
|
||||
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
|
||||
@ -162,14 +160,54 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
// onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;"
|
||||
// :00 is NOT recognized as a timestamp in description or comments.
|
||||
// 0:00 is recognized in both description and comments.
|
||||
// https://www.youtube.com/watch?v=4cccfDXu1vA
|
||||
private final static Pattern DESCRIPTION_TIMESTAMP_ONCLICK_REGEX = Pattern.compile(
|
||||
"seekTo\\("
|
||||
+ "(?:(\\d+)\\*3600\\+)?" // hours?
|
||||
+ "(\\d+)\\*60\\+" // minutes
|
||||
+ "(\\d+)" // seconds
|
||||
+ "\\)");
|
||||
|
||||
@SafeVarargs
|
||||
private static <T> T coalesce(T... args) {
|
||||
for (T arg : args) {
|
||||
if (arg != null) return arg;
|
||||
}
|
||||
throw new IllegalArgumentException("all arguments to coalesce() were null");
|
||||
}
|
||||
|
||||
private String parseHtmlAndGetFullLinks(String descriptionHtml)
|
||||
throws MalformedURLException, UnsupportedEncodingException, ParsingException {
|
||||
final Document description = Jsoup.parse(descriptionHtml, getUrl());
|
||||
for(Element a : description.select("a")) {
|
||||
final URL redirectLink = new URL(
|
||||
a.attr("abs:href"));
|
||||
final String queryString = redirectLink.getQuery();
|
||||
if(queryString != null) {
|
||||
final String rawUrl = a.attr("abs:href");
|
||||
final URL redirectLink = new URL(rawUrl);
|
||||
|
||||
final Matcher onClickTimestamp;
|
||||
final String queryString;
|
||||
if ((onClickTimestamp = DESCRIPTION_TIMESTAMP_ONCLICK_REGEX.matcher(a.attr("onclick")))
|
||||
.find()) {
|
||||
a.removeAttr("onclick");
|
||||
|
||||
String hours = coalesce(onClickTimestamp.group(1), "0");
|
||||
String minutes = onClickTimestamp.group(2);
|
||||
String seconds = onClickTimestamp.group(3);
|
||||
|
||||
int timestamp = 0;
|
||||
timestamp += Integer.parseInt(hours) * 3600;
|
||||
timestamp += Integer.parseInt(minutes) * 60;
|
||||
timestamp += Integer.parseInt(seconds);
|
||||
|
||||
String setTimestamp = "&t=" + timestamp;
|
||||
|
||||
// Even after clicking https://youtu.be/...?t=6,
|
||||
// getUrl() is https://www.youtube.com/watch?v=..., never youtu.be, never &t=.
|
||||
a.attr("href", getUrl() + setTimestamp);
|
||||
|
||||
} else if((queryString = redirectLink.getQuery()) != null) {
|
||||
// if the query string is null we are not dealing with a redirect link,
|
||||
// so we don't need to override it.
|
||||
final String link =
|
||||
@ -179,11 +217,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
// if link is null the a tag is a hashtag.
|
||||
// They refer to the youtube search. We do not handle them.
|
||||
a.text(link);
|
||||
a.attr("href", link);
|
||||
} else if(redirectLink.toString().contains("https://www.youtube.com/")) {
|
||||
a.text(redirectLink.toString());
|
||||
a.attr("href", redirectLink.toString());
|
||||
}
|
||||
} else if(redirectLink.toString().contains("https://www.youtube.com/")) {
|
||||
descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString());
|
||||
a.text(redirectLink.toString());
|
||||
a.attr("href", redirectLink.toString());
|
||||
}
|
||||
}
|
||||
return description.select("body").first().html();
|
||||
@ -206,29 +248,26 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
@Override
|
||||
public long getLength() throws ParsingException {
|
||||
assertPageFetched();
|
||||
if(playerArgs != null) {
|
||||
try {
|
||||
long returnValue = Long.parseLong(playerArgs.get("length_seconds") + "");
|
||||
if (returnValue >= 0) return returnValue;
|
||||
} catch (Exception ignored) {
|
||||
// Try other method...
|
||||
}
|
||||
}
|
||||
|
||||
String lengthString = videoInfoPage.get("length_seconds");
|
||||
// try getting duration from playerargs
|
||||
try {
|
||||
return Long.parseLong(lengthString);
|
||||
} catch (Exception ignored) {
|
||||
// Try other method...
|
||||
}
|
||||
|
||||
// TODO: 25.11.17 Implement a way to get the length for age restricted videos #44
|
||||
try {
|
||||
// Fallback to HTML method
|
||||
return Long.parseLong(doc.select("div[class~=\"ytp-progress-bar\"][role=\"slider\"]").first()
|
||||
.attr("aria-valuemax"));
|
||||
String durationMs = playerResponse
|
||||
.getObject("streamingData")
|
||||
.getArray("formats")
|
||||
.getObject(0)
|
||||
.getString("approxDurationMs");
|
||||
return Long.parseLong(durationMs)/1000;
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get video length", e);
|
||||
}
|
||||
|
||||
//try getting value from age gated video
|
||||
try {
|
||||
String duration = playerResponse
|
||||
.getObject("videoDetails")
|
||||
.getString("lengthSeconds");
|
||||
return Long.parseLong(duration);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Every methode to get the duration has failed: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -385,31 +424,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
@Override
|
||||
public String getHlsUrl() throws ParsingException {
|
||||
assertPageFetched();
|
||||
try {
|
||||
String hlsvp = "";
|
||||
if (playerArgs != null) {
|
||||
if( playerArgs.isString("hlsvp") ) {
|
||||
hlsvp = playerArgs.getString("hlsvp", "");
|
||||
}else {
|
||||
hlsvp = JsonParser.object()
|
||||
.from(playerArgs.getString("player_response", "{}"))
|
||||
.getObject("streamingData", new JsonObject())
|
||||
.getString("hlsManifestUrl", "");
|
||||
}
|
||||
}
|
||||
|
||||
return hlsvp;
|
||||
try {
|
||||
return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get hls manifest url", e);
|
||||
if (playerArgs != null && playerArgs.isString("hlsvp")) {
|
||||
return playerArgs.getString("hlsvp");
|
||||
} else {
|
||||
throw new ParsingException("Could not get hls manifest url", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
||||
public List<AudioStream> getAudioStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
List<AudioStream> audioStreams = new ArrayList<>();
|
||||
try {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) {
|
||||
ItagItem itag = entry.getValue();
|
||||
|
||||
AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate);
|
||||
@ -425,11 +457,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
||||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
List<VideoStream> videoStreams = new ArrayList<>();
|
||||
try {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
|
||||
ItagItem itag = entry.getValue();
|
||||
|
||||
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString);
|
||||
@ -449,7 +481,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
assertPageFetched();
|
||||
List<VideoStream> videoOnlyStreams = new ArrayList<>();
|
||||
try {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
|
||||
ItagItem itag = entry.getValue();
|
||||
|
||||
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true);
|
||||
@ -486,7 +518,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
assertPageFetched();
|
||||
try {
|
||||
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
|
||||
playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
|
||||
(!playerResponse.getObject("streamingData").has(FORMATS)))) {
|
||||
return StreamType.LIVE_STREAM;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
@ -505,7 +537,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
if (watch.size() < 1) {
|
||||
return null;// prevent the snackbar notification "report error" on age-restricted videos
|
||||
}
|
||||
|
||||
|
||||
collector.commit(extractVideoPreviewInfo(watch.first().select("li").first()));
|
||||
return collector.getItems().get(0);
|
||||
} catch (Exception e) {
|
||||
@ -538,21 +570,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
*/
|
||||
@Override
|
||||
public String getErrorMessage() {
|
||||
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
|
||||
StringBuilder errorReason;
|
||||
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
|
||||
|
||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||
if (errorElement == null) {
|
||||
errorReason = null;
|
||||
} else if (errorMessage.contains("GEMA")) {
|
||||
// Gema sometimes blocks youtube music content in germany:
|
||||
// https://www.gema.de/en/
|
||||
// Detailed description:
|
||||
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
||||
errorReason = new StringBuilder("GEMA");
|
||||
} else {
|
||||
errorReason = new StringBuilder(errorMessage);
|
||||
errorReason.append(" ");
|
||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||
String errorMessage = errorElement.text();
|
||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||
errorReason = null;
|
||||
} else {
|
||||
errorReason = new StringBuilder(errorMessage);
|
||||
errorReason.append(" ");
|
||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||
}
|
||||
}
|
||||
|
||||
return errorReason != null ? errorReason.toString() : null;
|
||||
@ -562,8 +593,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
// Fetch page
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
|
||||
private static final String ADAPTIVE_FMTS = "adaptive_fmts";
|
||||
private static final String FORMATS = "formats";
|
||||
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
|
||||
private static final String HTTPS = "https:";
|
||||
private static final String CONTENT = "content";
|
||||
private static final String DECRYPTION_FUNC_NAME = "decrypt";
|
||||
@ -571,7 +602,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
|
||||
|
||||
private final static String DECYRYPTION_SIGNATURE_FUNCTION_REGEX =
|
||||
"(\\w+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
|
||||
"([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
|
||||
private final static String DECRYPTION_AKAMAIZED_STRING_REGEX =
|
||||
"yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(";
|
||||
private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX =
|
||||
@ -581,22 +612,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
|
||||
private String pageHtml = null;
|
||||
|
||||
private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
|
||||
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
|
||||
if (pageHtml == null) {
|
||||
pageHtml = downloader.download(verifiedUrl);
|
||||
}
|
||||
return pageHtml;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
||||
final String pageContent = getPageHtml(downloader);
|
||||
doc = Jsoup.parse(pageContent, getUrl());
|
||||
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
|
||||
final DownloadResponse response = downloader.get(verifiedUrl);
|
||||
pageHtml = response.getResponseBody();
|
||||
doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
|
||||
|
||||
final String playerUrl;
|
||||
// Check if the video is age restricted
|
||||
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
|
||||
if (!doc.select("meta[property=\"og:restrictions:age\"").isEmpty()) {
|
||||
final EmbeddedInfo info = getEmbeddedInfo();
|
||||
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
|
||||
final String infoPageResponse = downloader.download(videoInfoUrl);
|
||||
@ -604,11 +629,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
playerUrl = info.url;
|
||||
isAgeRestricted = true;
|
||||
} else {
|
||||
final JsonObject ytPlayerConfig = getPlayerConfig(pageContent);
|
||||
final JsonObject ytPlayerConfig = getPlayerConfig();
|
||||
playerArgs = getPlayerArgs(ytPlayerConfig);
|
||||
playerUrl = getPlayerUrl(ytPlayerConfig);
|
||||
isAgeRestricted = false;
|
||||
}
|
||||
playerResponse = getPlayerResponse();
|
||||
|
||||
if (decryptionCode.isEmpty()) {
|
||||
decryptionCode = loadDecryptionCode(playerUrl);
|
||||
@ -619,15 +645,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
private JsonObject getPlayerConfig(String pageContent) throws ParsingException {
|
||||
private JsonObject getPlayerConfig() throws ParsingException {
|
||||
try {
|
||||
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
|
||||
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml);
|
||||
return JsonParser.object().from(ytPlayerConfigRaw);
|
||||
} catch (Parser.RegexException e) {
|
||||
String errorReason = getErrorMessage();
|
||||
switch (errorReason) {
|
||||
case "GEMA":
|
||||
throw new GemaException(errorReason);
|
||||
case "":
|
||||
throw new ContentNotAvailableException("Content not available: player config empty", e);
|
||||
default:
|
||||
@ -670,6 +694,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
private JsonObject getPlayerResponse() throws ParsingException {
|
||||
try {
|
||||
String playerResponseStr;
|
||||
if(playerArgs != null) {
|
||||
playerResponseStr = playerArgs.getString("player_response");
|
||||
} else {
|
||||
playerResponseStr = videoInfoPage.get("player_response");
|
||||
}
|
||||
return JsonParser.object().from(playerResponseStr);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not parse yt player response", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
|
||||
try {
|
||||
@ -685,16 +723,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
playerUrl = HTTPS + playerUrl;
|
||||
}
|
||||
|
||||
// Get embed sts
|
||||
final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
|
||||
final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
|
||||
try {
|
||||
// Get embed sts
|
||||
final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
|
||||
final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
|
||||
return new EmbeddedInfo(playerUrl, sts);
|
||||
} catch (Exception i) {
|
||||
// if it failes we simply reply with no sts as then it does not seem to be necessary
|
||||
return new EmbeddedInfo(playerUrl, "");
|
||||
}
|
||||
|
||||
return new EmbeddedInfo(playerUrl, sts);
|
||||
} catch (IOException e) {
|
||||
throw new ParsingException(
|
||||
"Could load decryption code form restricted video for the Youtube service.", e);
|
||||
} catch (ReCaptchaException e) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||
}
|
||||
}
|
||||
|
||||
@ -708,15 +749,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
|
||||
final String playerCode = downloader.download(playerUrl);
|
||||
final String decryptionFunctionName = getDecryptionFuncName(playerCode);
|
||||
|
||||
final String decryptionFunctionName;
|
||||
if (Parser.isMatch(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode)) {
|
||||
decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
|
||||
} else if (Parser.isMatch(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode)) {
|
||||
decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
|
||||
} else {
|
||||
decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
|
||||
}
|
||||
final String functionPattern = "("
|
||||
+ decryptionFunctionName.replace("$", "\\$")
|
||||
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
|
||||
@ -757,30 +791,38 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
return result == null ? "" : result.toString();
|
||||
}
|
||||
|
||||
private String getDecryptionFuncName(String playerCode) throws DecryptException {
|
||||
String decryptionFunctionName;
|
||||
// Cascading things in catch is ugly, but its faster than running a match before getting the actual name
|
||||
// to se if the function can actually be found with the given regex.
|
||||
// However if this cascading should propably be cleaned up somehow as it looks a bit weird.
|
||||
try {
|
||||
decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
|
||||
} catch (Parser.RegexException re) {
|
||||
try {
|
||||
decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
|
||||
} catch (Parser.RegexException re2) {
|
||||
try {
|
||||
decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
|
||||
} catch (Parser.RegexException re3) {
|
||||
throw new DecryptException("Could not find decrypt function with any of the given patterns.", re);
|
||||
}
|
||||
}
|
||||
}
|
||||
return decryptionFunctionName;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesException {
|
||||
// If the video is age restricted getPlayerConfig will fail
|
||||
if(isAgeRestricted) return Collections.emptyList();
|
||||
|
||||
final JsonObject playerConfig;
|
||||
try {
|
||||
playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
|
||||
} catch (IOException | ExtractionException e) {
|
||||
throw new SubtitlesException("Unable to download player configs", e);
|
||||
}
|
||||
final String playerResponse = playerConfig.getObject("args", new JsonObject())
|
||||
.getString("player_response");
|
||||
|
||||
final JsonObject captions;
|
||||
try {
|
||||
if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) {
|
||||
// Captions does not exist
|
||||
return Collections.emptyList();
|
||||
}
|
||||
captions = JsonParser.object().from(playerResponse).getObject("captions");
|
||||
} catch (JsonParserException e) {
|
||||
throw new SubtitlesException("Unable to parse subtitles listing", e);
|
||||
if (!playerResponse.has("captions")) {
|
||||
// Captions does not exist
|
||||
return Collections.emptyList();
|
||||
}
|
||||
captions = playerResponse.getObject("captions");
|
||||
|
||||
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
|
||||
final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
|
||||
@ -849,39 +891,36 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
"&sts=" + sts + "&ps=default&gl=US&hl=en";
|
||||
}
|
||||
|
||||
private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
|
||||
private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
|
||||
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
|
||||
|
||||
String encodedUrlMap = "";
|
||||
if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) {
|
||||
encodedUrlMap = playerArgs.getString(encodedUrlMapKey, "");
|
||||
} else if (videoInfoPage.containsKey(encodedUrlMapKey)) {
|
||||
encodedUrlMap = videoInfoPage.get(encodedUrlMapKey);
|
||||
JsonObject streamingData = playerResponse.getObject("streamingData");
|
||||
if (!streamingData.has(streamingDataKey)) {
|
||||
return urlAndItags;
|
||||
}
|
||||
|
||||
for (String url_data_str : encodedUrlMap.split(",")) {
|
||||
try {
|
||||
// This loop iterates through multiple streams, therefore tags
|
||||
// is related to one and the same stream at a time.
|
||||
Map<String, String> tags = Parser.compatParseMap(
|
||||
org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
|
||||
JsonArray formats = streamingData.getArray(streamingDataKey);
|
||||
for (int i = 0; i != formats.size(); ++i) {
|
||||
JsonObject formatData = formats.getObject(i);
|
||||
int itag = formatData.getInt("itag");
|
||||
|
||||
int itag = Integer.parseInt(tags.get("itag"));
|
||||
|
||||
if (ItagItem.isSupported(itag)) {
|
||||
if (ItagItem.isSupported(itag)) {
|
||||
try {
|
||||
ItagItem itagItem = ItagItem.getItag(itag);
|
||||
if (itagItem.itagType == itagTypeWanted) {
|
||||
String streamUrl = tags.get("url");
|
||||
// if video has a signature: decrypt it and add it to the url
|
||||
if (tags.get("s") != null) {
|
||||
streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
|
||||
String streamUrl;
|
||||
if (formatData.has("url")) {
|
||||
streamUrl = formatData.getString("url");
|
||||
} else {
|
||||
// this url has an encrypted signature
|
||||
Map<String, String> cipher = Parser.compatParseMap(formatData.getString("cipher"));
|
||||
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode);
|
||||
}
|
||||
|
||||
urlAndItags.put(streamUrl, itagItem);
|
||||
}
|
||||
} catch (UnsupportedEncodingException ignored) {
|
||||
|
||||
}
|
||||
} catch (DecryptException e) {
|
||||
throw e;
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
@ -954,4 +993,61 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<Frameset> getFrames() throws ExtractionException {
|
||||
try {
|
||||
final String script = doc.select("#player-api").first().siblingElements().select("script").html();
|
||||
int p = script.indexOf("ytplayer.config");
|
||||
if (p == -1) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
p = script.indexOf('{', p);
|
||||
int e = script.indexOf("ytplayer.load", p);
|
||||
if (e == -1) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
|
||||
final String resp = jo.getObject("args").getString("player_response");
|
||||
jo = JsonParser.object().from(resp);
|
||||
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
|
||||
final String url = spec[0];
|
||||
final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
|
||||
for (int i = 1; i < spec.length; ++i) {
|
||||
final String[] parts = spec[i].split("#");
|
||||
if (parts.length != 8) {
|
||||
continue;
|
||||
}
|
||||
final int frameWidth = Integer.parseInt(parts[0]);
|
||||
final int frameHeight = Integer.parseInt(parts[1]);
|
||||
final int totalCount = Integer.parseInt(parts[2]);
|
||||
final int framesPerPageX = Integer.parseInt(parts[3]);
|
||||
final int framesPerPageY = Integer.parseInt(parts[4]);
|
||||
final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
|
||||
final List<String> urls;
|
||||
if (baseUrl.contains("$M")) {
|
||||
final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
|
||||
urls = new ArrayList<>(totalPages);
|
||||
for (int j = 0; j < totalPages; j++) {
|
||||
urls.add(baseUrl.replace("$M", String.valueOf(j)));
|
||||
}
|
||||
} else {
|
||||
urls = Collections.singletonList(baseUrl);
|
||||
}
|
||||
result.add(new Frameset(
|
||||
urls,
|
||||
frameWidth,
|
||||
frameHeight,
|
||||
totalCount,
|
||||
framesPerPageX,
|
||||
framesPerPageY
|
||||
));
|
||||
}
|
||||
result.trimToSize();
|
||||
return result;
|
||||
} catch (Exception e) {
|
||||
throw new ExtractionException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -61,7 +61,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
try {
|
||||
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
|
||||
Element el = item.select("div[class*=\"yt-lockup-video\"]").first();
|
||||
Element dl = el.select("h3").first().select("a").first();
|
||||
return dl.attr("abs:href");
|
||||
} catch (Exception e) {
|
||||
@ -72,7 +72,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
try {
|
||||
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
|
||||
Element el = item.select("div[class*=\"yt-lockup-video\"]").first();
|
||||
Element dl = el.select("h3").first().select("a").first();
|
||||
return dl.text();
|
||||
} catch (Exception e) {
|
||||
@ -107,6 +107,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
// this url is not always in the form "/channel/..."
|
||||
// sometimes Youtube provides urls in the from "/user/..."
|
||||
try {
|
||||
try {
|
||||
return item.select("div[class=\"yt-lockup-byline\"]").first()
|
||||
@ -119,7 +121,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
.text().split(" - ")[0];
|
||||
} catch (Exception e) {
|
||||
System.out.println(item.html());
|
||||
throw new ParsingException("Could not get uploader", e);
|
||||
throw new ParsingException("Could not get uploader url", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -63,16 +63,10 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
|
||||
String title = outline.attr("title");
|
||||
String xmlUrl = outline.attr("abs:xmlUrl");
|
||||
|
||||
if (title.isEmpty() || xmlUrl.isEmpty()) {
|
||||
throw new InvalidSourceException("document has invalid entries");
|
||||
}
|
||||
|
||||
try {
|
||||
String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
|
||||
result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
|
||||
} catch (Parser.RegexException e) {
|
||||
throw new InvalidSourceException("document has invalid entries", e);
|
||||
}
|
||||
} catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ }
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@ -24,12 +24,14 @@ import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.schabi.newpipe.extractor.DownloadResponse;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
@ -56,8 +58,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
|
||||
url += "?gl=" + contentCountry;
|
||||
}
|
||||
|
||||
String pageContent = downloader.download(url);
|
||||
doc = Jsoup.parse(pageContent, url);
|
||||
final DownloadResponse response = downloader.get(url);
|
||||
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -126,6 +128,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
|
||||
}
|
||||
|
||||
private Element getUploaderLink() {
|
||||
// this url is not always in the form "/channel/..."
|
||||
// sometimes Youtube provides urls in the from "/user/..."
|
||||
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
|
||||
return uploaderEl.select("a").first();
|
||||
}
|
||||
|
||||
@ -46,7 +46,8 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
|
||||
URL urlObj = Utils.stringToURL(url);
|
||||
String path = urlObj.getPath();
|
||||
|
||||
if (!(YoutubeParsingHelper.isYoutubeURL(urlObj) || urlObj.getHost().equalsIgnoreCase("hooktube.com"))) {
|
||||
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) ||
|
||||
YoutubeParsingHelper.isInvidioURL(urlObj) || YoutubeParsingHelper.isHooktubeURL(urlObj))) {
|
||||
throw new ParsingException("the URL given is not a Youtube-URL");
|
||||
}
|
||||
|
||||
|
||||
@ -1,7 +1,11 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
|
||||
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.schabi.newpipe.extractor.DownloadResponse;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
|
||||
import java.net.URL;
|
||||
|
||||
@ -30,40 +34,42 @@ public class YoutubeParsingHelper {
|
||||
private YoutubeParsingHelper() {
|
||||
}
|
||||
|
||||
private static boolean isHTTP(URL url) {
|
||||
// make sure its http or https
|
||||
String protocol = url.getProtocol();
|
||||
if (!protocol.equals("http") && !protocol.equals("https")) {
|
||||
return false;
|
||||
private static final String[] RECAPTCHA_DETECTION_SELECTORS = {
|
||||
"form[action*=\"/das_captcha\"]",
|
||||
"input[name*=\"action_recaptcha_verify\"]"
|
||||
};
|
||||
|
||||
public static Document parseAndCheckPage(final String url, final DownloadResponse response) throws ReCaptchaException {
|
||||
final Document document = Jsoup.parse(response.getResponseBody(), url);
|
||||
|
||||
for (String detectionSelector : RECAPTCHA_DETECTION_SELECTORS) {
|
||||
if (!document.select(detectionSelector).isEmpty()) {
|
||||
throw new ReCaptchaException("reCAPTCHA challenge requested (detected with selector: \"" + detectionSelector + "\")", url);
|
||||
}
|
||||
}
|
||||
|
||||
boolean usesDefaultPort = url.getPort() == url.getDefaultPort();
|
||||
boolean setsNoPort = url.getPort() == -1;
|
||||
|
||||
return setsNoPort || usesDefaultPort;
|
||||
return document;
|
||||
}
|
||||
|
||||
public static boolean isYoutubeURL(URL url) {
|
||||
// make sure its http or https
|
||||
if (!isHTTP(url))
|
||||
return false;
|
||||
|
||||
// make sure its a known youtube url
|
||||
String host = url.getHost();
|
||||
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|
||||
|| host.equalsIgnoreCase("m.youtube.com");
|
||||
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com");
|
||||
}
|
||||
|
||||
public static boolean isYoutubeALikeURL(URL url) {
|
||||
// make sure its http or https
|
||||
if (!isHTTP(url))
|
||||
return false;
|
||||
|
||||
// make sure its a known youtube url
|
||||
public static boolean isYoutubeServiceURL(URL url) {
|
||||
String host = url.getHost();
|
||||
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|
||||
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("www.youtube-nocookie.com")
|
||||
|| host.equalsIgnoreCase("youtu.be") || host.equalsIgnoreCase("hooktube.com");
|
||||
return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be");
|
||||
}
|
||||
|
||||
public static boolean isHooktubeURL(URL url) {
|
||||
String host = url.getHost();
|
||||
return host.equalsIgnoreCase("hooktube.com");
|
||||
}
|
||||
|
||||
public static boolean isInvidioURL(URL url) {
|
||||
String host = url.getHost();
|
||||
return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") || host.equalsIgnoreCase("invidious.snopyta.org") || host.equalsIgnoreCase("de.invidious.snopyta.org") || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("vid.wxzm.sx") || host.equalsIgnoreCase("invidious.kabi.tk") || host.equalsIgnoreCase("invidiou.sh") || host.equalsIgnoreCase("www.invidiou.sh") || host.equalsIgnoreCase("no.invidiou.sh") || host.equalsIgnoreCase("invidious.enkirton.net") || host.equalsIgnoreCase("tube.poal.co") || host.equalsIgnoreCase("invidious.13ad.de") || host.equalsIgnoreCase("yt.elukerio.org");
|
||||
}
|
||||
|
||||
public static long parseDurationString(String input)
|
||||
|
||||
@ -25,10 +25,16 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
|
||||
try {
|
||||
URL urlObj = Utils.stringToURL(url);
|
||||
|
||||
if (!YoutubeParsingHelper.isYoutubeURL(urlObj)) {
|
||||
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|
||||
|| YoutubeParsingHelper.isInvidioURL(urlObj))) {
|
||||
throw new ParsingException("the url given is not a Youtube-URL");
|
||||
}
|
||||
|
||||
String path = urlObj.getPath();
|
||||
if (!path.equals("/watch" ) && !path.equals("/playlist")) {
|
||||
throw new ParsingException("the url given is neither a video nor a playlist URL");
|
||||
}
|
||||
|
||||
String listID = Utils.getQueryValue(urlObj, "list");
|
||||
|
||||
if (listID == null) {
|
||||
|
||||
@ -60,7 +60,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
||||
URI uri = new URI(urlString);
|
||||
String scheme = uri.getScheme();
|
||||
|
||||
if (scheme != null && scheme.equals("vnd.youtube")) {
|
||||
if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
|
||||
String schemeSpecificPart = uri.getSchemeSpecificPart();
|
||||
if (schemeSpecificPart.startsWith("//")) {
|
||||
urlString = "https:" + schemeSpecificPart;
|
||||
@ -85,7 +85,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
||||
path = path.substring(1);
|
||||
}
|
||||
|
||||
if (!YoutubeParsingHelper.isYoutubeALikeURL(url)) {
|
||||
if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) ||
|
||||
YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) ||
|
||||
YoutubeParsingHelper.isInvidioURL(url))) {
|
||||
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
|
||||
throw new FoundAdException("Error found ad: " + urlString);
|
||||
}
|
||||
@ -112,7 +114,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
||||
|
||||
case "YOUTUBE.COM":
|
||||
case "WWW.YOUTUBE.COM":
|
||||
case "M.YOUTUBE.COM": {
|
||||
case "M.YOUTUBE.COM":
|
||||
case "MUSIC.YOUTUBE.COM": {
|
||||
if (path.equals("attribution_link")) {
|
||||
String uQueryValue = Utils.getQueryValue(url, "u");
|
||||
|
||||
@ -147,6 +150,34 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
||||
}
|
||||
|
||||
case "HOOKTUBE.COM": {
|
||||
if (path.startsWith("v/")) {
|
||||
String id = path.substring("v/".length());
|
||||
|
||||
return assertIsID(id);
|
||||
}
|
||||
if (path.startsWith("watch/")) {
|
||||
String id = path.substring("watch/".length());
|
||||
|
||||
return assertIsID(id);
|
||||
}
|
||||
// there is no break-statement here on purpose so the next code-block gets also run for hooktube
|
||||
}
|
||||
|
||||
case "WWW.INVIDIO.US":
|
||||
case "DEV.INVIDIO.US":
|
||||
case "INVIDIO.US":
|
||||
case "INVIDIOUS.SNOPYTA.ORG":
|
||||
case "DE.INVIDIOUS.SNOPYTA.ORG":
|
||||
case "FI.INVIDIOUS.SNOPYTA.ORG":
|
||||
case "VID.WXZM.SX":
|
||||
case "INVIDIOUS.KABI.TK":
|
||||
case "INVIDIOU.SH":
|
||||
case "WWW.INVIDIOU.SH":
|
||||
case "NO.INVIDIOU.SH":
|
||||
case "INVIDIOUS.ENKIRTON.NET":
|
||||
case "TUBE.POAL.CO":
|
||||
case "INVIDIOUS.13AD.DE":
|
||||
case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances
|
||||
if (path.equals("watch")) {
|
||||
String viewQueryValue = Utils.getQueryValue(url, "v");
|
||||
if (viewQueryValue != null) {
|
||||
@ -158,19 +189,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
||||
|
||||
return assertIsID(id);
|
||||
}
|
||||
if (path.startsWith("v/")) {
|
||||
String id = path.substring("v/".length());
|
||||
|
||||
return assertIsID(id);
|
||||
}
|
||||
if (path.startsWith("watch/")) {
|
||||
String id = path.substring("watch/".length());
|
||||
|
||||
return assertIsID(id);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
throw new ParsingException("Error no suitable url: " + urlString);
|
||||
|
||||
@ -48,6 +48,6 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
|
||||
}
|
||||
|
||||
String urlPath = urlObj.getPath();
|
||||
return YoutubeParsingHelper.isYoutubeURL(urlObj) && urlPath.equals("/feed/trending");
|
||||
return Utils.isHTTP(urlObj) && (YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj)) && urlPath.equals("/feed/trending");
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,66 @@
|
||||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
public final class Frameset {
|
||||
|
||||
private List<String> urls;
|
||||
private int frameWidth;
|
||||
private int frameHeight;
|
||||
private int totalCount;
|
||||
private int framesPerPageX;
|
||||
private int framesPerPageY;
|
||||
|
||||
public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
|
||||
this.urls = urls;
|
||||
this.totalCount = totalCount;
|
||||
this.frameWidth = frameWidth;
|
||||
this.frameHeight = frameHeight;
|
||||
this.framesPerPageX = framesPerPageX;
|
||||
this.framesPerPageY = framesPerPageY;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list of urls to images with frames
|
||||
*/
|
||||
public List<String> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return total count of frames
|
||||
*/
|
||||
public int getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return maximum frames count by x
|
||||
*/
|
||||
public int getFramesPerPageX() {
|
||||
return framesPerPageX;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return maximum frames count by y
|
||||
*/
|
||||
public int getFramesPerPageY() {
|
||||
return framesPerPageY;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return width of a one frame, in pixels
|
||||
*/
|
||||
public int getFrameWidth() {
|
||||
return frameWidth;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return height of a one frame, in pixels
|
||||
*/
|
||||
public int getFrameHeight() {
|
||||
return frameHeight;
|
||||
}
|
||||
}
|
||||
@ -30,7 +30,10 @@ import org.schabi.newpipe.extractor.utils.Localization;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
@ -256,7 +259,18 @@ public abstract class StreamExtractor extends Extractor {
|
||||
public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException;
|
||||
|
||||
/**
|
||||
* Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block)
|
||||
* Should return a list of Frameset object that contains preview of stream frames
|
||||
* @return list of preview frames or empty list if frames preview is not supported or not found for specified stream
|
||||
* @throws IOException
|
||||
* @throws ExtractionException
|
||||
*/
|
||||
@Nonnull
|
||||
public List<Frameset> getFrames() throws IOException, ExtractionException {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Should analyse the webpage's document and extracts any error message there might be.
|
||||
*
|
||||
* @return Error message; null if there is no error message.
|
||||
*/
|
||||
|
||||
@ -123,8 +123,6 @@ public class DashMpdParser {
|
||||
dashDoc = downloader.download(streamInfo.getDashMpdUrl());
|
||||
} catch (IOException ioe) {
|
||||
throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe);
|
||||
} catch (ReCaptchaException e) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge needed");
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
@ -27,6 +27,38 @@ public class Utils {
|
||||
return toRemove.replaceAll("\\D+", "");
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Convert a mixed number word to a long.</p>
|
||||
* <p>Examples:</p>
|
||||
* <ul>
|
||||
* <li>123 -> 123</li>
|
||||
* <li>1.23K -> 1230</li>
|
||||
* <li>1.23M -> 1230000</li>
|
||||
* </ul>
|
||||
* @param numberWord string to be converted to a long
|
||||
* @return a long
|
||||
* @throws NumberFormatException
|
||||
* @throws ParsingException
|
||||
*/
|
||||
public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException {
|
||||
String multiplier = "";
|
||||
try {
|
||||
multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
|
||||
} catch(ParsingException ignored) {}
|
||||
double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord)
|
||||
.replace(",", "."));
|
||||
switch (multiplier.toUpperCase()) {
|
||||
case "K":
|
||||
return (long) (count * 1e3);
|
||||
case "M":
|
||||
return (long) (count * 1e6);
|
||||
case "B":
|
||||
return (long) (count * 1e9);
|
||||
default:
|
||||
return (long) (count);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the url matches the pattern.
|
||||
*
|
||||
@ -120,6 +152,19 @@ public class Utils {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public static boolean isHTTP(URL url) {
|
||||
// make sure its http or https
|
||||
String protocol = url.getProtocol();
|
||||
if (!protocol.equals("http") && !protocol.equals("https")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean usesDefaultPort = url.getPort() == url.getDefaultPort();
|
||||
boolean setsNoPort = url.getPort() == -1;
|
||||
|
||||
return setsNoPort || usesDefaultPort;
|
||||
}
|
||||
|
||||
public static String removeUTF8BOM(String s) {
|
||||
if (s.startsWith("\uFEFF")) {
|
||||
|
||||
@ -16,6 +16,8 @@ import org.schabi.newpipe.extractor.DownloadResponse;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 28.01.16.
|
||||
*
|
||||
@ -129,7 +131,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
* request See : https://github.com/rg3/youtube-dl/issues/5138
|
||||
*/
|
||||
if (con.getResponseCode() == 429) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
|
||||
}
|
||||
|
||||
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
|
||||
@ -172,6 +174,36 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
return dl(con);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException {
|
||||
final HttpsURLConnection con = (HttpsURLConnection) new URL(siteUrl).openConnection();
|
||||
|
||||
try {
|
||||
con.setRequestMethod("HEAD");
|
||||
setDefaults(con);
|
||||
} catch (Exception e) {
|
||||
/*
|
||||
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
|
||||
* request See : https://github.com/rg3/youtube-dl/issues/5138
|
||||
*/
|
||||
if (con.getResponseCode() == 429) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
|
||||
}
|
||||
|
||||
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
|
||||
}
|
||||
|
||||
return new DownloadResponse(con.getResponseCode(), null, con.getHeaderFields());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse get(String siteUrl, Localization localization) throws IOException, ReCaptchaException {
|
||||
final Map<String, List<String>> requestHeaders = new HashMap<>();
|
||||
requestHeaders.put("Accept-Language", singletonList(localization.getLanguage()));
|
||||
|
||||
return get(siteUrl, new DownloadRequest(null, requestHeaders));
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse get(String siteUrl, DownloadRequest request)
|
||||
throws IOException, ReCaptchaException {
|
||||
@ -183,7 +215,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
}
|
||||
}
|
||||
String responseBody = dl(con);
|
||||
return new DownloadResponse(responseBody, con.getHeaderFields());
|
||||
return new DownloadResponse(con.getResponseCode(), responseBody, con.getHeaderFields());
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -219,6 +251,6 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
sb.append(inputLine);
|
||||
}
|
||||
}
|
||||
return new DownloadResponse(sb.toString(), con.getHeaderFields());
|
||||
return new DownloadResponse(con.getResponseCode(), sb.toString(), con.getHeaderFields());
|
||||
}
|
||||
}
|
||||
|
||||
@ -36,7 +36,7 @@ public class MediaCCCOggTest {
|
||||
@Test
|
||||
public void getAudioStreamsContainOgg() throws Exception {
|
||||
for(AudioStream stream : extractor.getAudioStreams()) {
|
||||
System.out.println(stream.getFormat());
|
||||
assertEquals("OGG", stream.getFormat().toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -25,7 +25,7 @@ import org.schabi.newpipe.extractor.utils.Localization;
|
||||
* Test for {@link PeertubeChannelExtractor}
|
||||
*/
|
||||
public class PeertubeChannelExtractorTest {
|
||||
public static class LilUzi implements BaseChannelExtractorTest {
|
||||
public static class KDE implements BaseChannelExtractorTest {
|
||||
private static PeertubeChannelExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
@ -34,7 +34,7 @@ public class PeertubeChannelExtractorTest {
|
||||
// setting instance might break test when running in parallel
|
||||
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
|
||||
extractor = (PeertubeChannelExtractor) PeerTube
|
||||
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org");
|
||||
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@ -49,22 +49,22 @@ public class PeertubeChannelExtractorTest {
|
||||
|
||||
@Test
|
||||
public void testName() throws ParsingException {
|
||||
assertEquals("Noureddine HADDAG", extractor.getName());
|
||||
assertEquals("The KDE Community", extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testId() throws ParsingException {
|
||||
assertEquals("root@tube.openalgeria.org", extractor.getId());
|
||||
assertEquals("kde", extractor.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUrl() throws ParsingException {
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getUrl());
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOriginalUrl() throws ParsingException {
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getOriginalUrl());
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getOriginalUrl());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
@ -112,7 +112,7 @@ public class PeertubeChannelExtractorTest {
|
||||
}
|
||||
}
|
||||
|
||||
public static class DubMatix implements BaseChannelExtractorTest {
|
||||
public static class Booteille implements BaseChannelExtractorTest {
|
||||
private static PeertubeChannelExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
@ -121,7 +121,7 @@ public class PeertubeChannelExtractorTest {
|
||||
// setting instance might break test when running in parallel
|
||||
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
|
||||
extractor = (PeertubeChannelExtractor) PeerTube
|
||||
.getChannelExtractor("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch");
|
||||
.getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@ -146,22 +146,22 @@ public class PeertubeChannelExtractorTest {
|
||||
|
||||
@Test
|
||||
public void testName() throws ParsingException {
|
||||
assertEquals("France Inter", extractor.getName());
|
||||
assertEquals("booteille", extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testId() throws ParsingException {
|
||||
assertEquals("franceinter@tube.kdy.ch", extractor.getId());
|
||||
assertEquals("booteille", extractor.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUrl() throws ParsingException {
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/franceinter@tube.kdy.ch", extractor.getUrl());
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOriginalUrl() throws ParsingException {
|
||||
assertEquals("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch", extractor.getOriginalUrl());
|
||||
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
@ -205,7 +205,7 @@ public class PeertubeChannelExtractorTest {
|
||||
|
||||
@Test
|
||||
public void testSubscriberCount() throws ParsingException {
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 75);
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -32,7 +32,7 @@ public class PeertubeStreamExtractorDefaultTest {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
// setting instance might break test when running in parallel
|
||||
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
|
||||
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
|
||||
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@ -44,39 +44,39 @@ public class PeertubeStreamExtractorDefaultTest {
|
||||
|
||||
@Test
|
||||
public void testGetTitle() throws ParsingException {
|
||||
assertEquals(extractor.getName(), "The Internet's Own Boy");
|
||||
assertEquals(extractor.getName(), "Power Corrupts the Best");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDescription() throws ParsingException {
|
||||
assertEquals(extractor.getDescription(), "The story of programming prodigy and information activist Aaron Swartz, who took his own life at the age of 26.");
|
||||
assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderName() throws ParsingException {
|
||||
assertEquals(extractor.getUploaderName(), "root");
|
||||
assertEquals(extractor.getUploaderName(), "Rowsedower");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertEquals(extractor.getLength(), 6299);
|
||||
assertEquals(extractor.getLength(), 269);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetViewCount() throws ParsingException {
|
||||
assertTrue(Long.toString(extractor.getViewCount()),
|
||||
extractor.getViewCount() > 700);
|
||||
extractor.getViewCount() > 10);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploadDate() throws ParsingException {
|
||||
assertEquals("2017-10-17", extractor.getUploadDate());
|
||||
assertEquals("2018-09-30", extractor.getUploadDate());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderUrl() throws ParsingException {
|
||||
assertIsSecureUrl(extractor.getUploaderUrl());
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@peertube2.cpy.re", extractor.getUploaderUrl());
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@ -25,23 +25,21 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
// setting instance might break test when running in parallel
|
||||
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
|
||||
extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet's own boy");
|
||||
extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde");
|
||||
extractor.fetchPage();
|
||||
itemsPage = extractor.getInitialPage();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSecondPageUrl() throws Exception {
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet%27s+own+boy&start=12&count=12", extractor.getNextPageUrl());
|
||||
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=kde&start=12&count=12", extractor.getNextPageUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResultList_FirstElement() {
|
||||
InfoItem firstInfoItem = itemsPage.getItems().get(0);
|
||||
|
||||
// THe channel should be the first item
|
||||
assertEquals("name", "The Internet's Own Boy", firstInfoItem.getName());
|
||||
assertEquals("url","https://peertube.mastodon.host/api/v1/videos/04af977f-4201-4697-be67-a8d8cae6fa7a", firstInfoItem.getUrl());
|
||||
|
||||
assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -83,11 +81,11 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB
|
||||
|
||||
@Test
|
||||
public void testId() throws Exception {
|
||||
assertEquals("internet's own boy", extractor.getId());
|
||||
assertEquals("kde", extractor.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testName() {
|
||||
assertEquals("internet's own boy", extractor.getName());
|
||||
assertEquals("kde", extractor.getName());
|
||||
}
|
||||
}
|
||||
|
||||
@ -36,7 +36,6 @@ public class SoundcloudChartsExtractorTest {
|
||||
assertNotNull(NewPipe.getDownloader());
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testGetName() throws Exception {
|
||||
assertEquals(extractor.getName(), "Top 50");
|
||||
|
||||
@ -1,18 +1,24 @@
|
||||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.junit.*;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class SoundcloudParsingHelperTest {
|
||||
@BeforeClass
|
||||
public static void setUp() {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void assertThatHardcodedClientIdIsValid() throws Exception {
|
||||
assertTrue("Hardcoded client id is not valid anymore",
|
||||
SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid(Downloader.getInstance()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void resolveUrlWithEmbedPlayerTest() throws Exception {
|
||||
Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import org.hamcrest.CoreMatchers;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
@ -119,14 +120,14 @@ public class SoundcloudPlaylistExtractorTest {
|
||||
}
|
||||
}
|
||||
|
||||
public static class RandomHouseDanceMusic implements BasePlaylistExtractorTest {
|
||||
public static class RandomHouseMusic implements BasePlaylistExtractorTest {
|
||||
private static SoundcloudPlaylistExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (SoundcloudPlaylistExtractor) SoundCloud
|
||||
.getPlaylistExtractor("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2");
|
||||
.getPlaylistExtractor("https://soundcloud.com/micky96/sets/house");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@ -141,22 +142,22 @@ public class SoundcloudPlaylistExtractorTest {
|
||||
|
||||
@Test
|
||||
public void testName() {
|
||||
assertEquals("House, Electro , Dance Music 2", extractor.getName());
|
||||
assertEquals("House", extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testId() {
|
||||
assertEquals("310980722", extractor.getId());
|
||||
assertEquals("123062856", extractor.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUrl() throws Exception {
|
||||
assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getUrl());
|
||||
assertEquals("https://soundcloud.com/micky96/sets/house", extractor.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOriginalUrl() throws Exception {
|
||||
assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getOriginalUrl());
|
||||
assertEquals("https://soundcloud.com/micky96/sets/house", extractor.getOriginalUrl());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
@ -182,7 +183,7 @@ public class SoundcloudPlaylistExtractorTest {
|
||||
assertIsSecureUrl(extractor.getThumbnailUrl());
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Ignore("not implemented")
|
||||
@Test
|
||||
public void testBannerUrl() {
|
||||
assertIsSecureUrl(extractor.getBannerUrl());
|
||||
@ -192,12 +193,12 @@ public class SoundcloudPlaylistExtractorTest {
|
||||
public void testUploaderUrl() {
|
||||
final String uploaderUrl = extractor.getUploaderUrl();
|
||||
assertIsSecureUrl(uploaderUrl);
|
||||
assertTrue(uploaderUrl, uploaderUrl.contains("hunter-leader"));
|
||||
assertThat(uploaderUrl, CoreMatchers.containsString("micky96"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUploaderName() {
|
||||
assertEquals("Gosu", extractor.getUploaderName());
|
||||
assertEquals("_mickyyy", extractor.getUploaderName());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -226,6 +227,7 @@ public class SoundcloudPlaylistExtractorTest {
|
||||
// Additional Testing
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testGetPageInNewExtractor() throws Exception {
|
||||
final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
|
||||
@ -265,11 +267,14 @@ public class SoundcloudPlaylistExtractorTest {
|
||||
// ListExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testRelatedItems() throws Exception {
|
||||
defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
|
||||
}
|
||||
|
||||
//TODO: FUCK THIS: This triggers a 500 at sever
|
||||
@Ignore
|
||||
@Test
|
||||
public void testMoreRelatedItems() throws Exception {
|
||||
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId());
|
||||
@ -284,6 +289,7 @@ public class SoundcloudPlaylistExtractorTest {
|
||||
// PlaylistExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testThumbnailUrl() {
|
||||
assertIsSecureUrl(extractor.getThumbnailUrl());
|
||||
|
||||
@ -39,27 +39,27 @@ public class SoundcloudStreamExtractorDefaultTest {
|
||||
@Test
|
||||
public void testGetValidTimeStamp() throws IOException, ExtractionException {
|
||||
StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69");
|
||||
assertEquals(extractor.getTimeStamp() + "", "69");
|
||||
assertEquals("69", extractor.getTimeStamp() + "");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTitle() throws ParsingException {
|
||||
assertEquals(extractor.getName(), "Do What I Want [Produced By Maaly Raw + Don Cannon]");
|
||||
assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDescription() throws ParsingException {
|
||||
assertEquals(extractor.getDescription(), "The Perfect LUV Tape®️");
|
||||
assertEquals("The Perfect LUV Tape®️", extractor.getDescription());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderName() throws ParsingException {
|
||||
assertEquals(extractor.getUploaderName(), "LIL UZI VERT");
|
||||
assertEquals("LIL UZI VERT", extractor.getUploaderName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertEquals(extractor.getLength(), 175);
|
||||
assertEquals(175, extractor.getLength());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@ -109,6 +109,7 @@ public class YoutubeChannelExtractorTest {
|
||||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
|
||||
assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 4e6);
|
||||
}
|
||||
}
|
||||
|
||||
@ -199,6 +200,7 @@ public class YoutubeChannelExtractorTest {
|
||||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
|
||||
assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 10e6);
|
||||
}
|
||||
|
||||
}
|
||||
@ -394,6 +396,100 @@ public class YoutubeChannelExtractorTest {
|
||||
}
|
||||
}
|
||||
|
||||
// this channel has no "Subscribe" button
|
||||
public static class EminemVEVO implements BaseChannelExtractorTest {
|
||||
private static YoutubeChannelExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeChannelExtractor) YouTube
|
||||
.getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Extractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testServiceId() {
|
||||
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testName() throws Exception {
|
||||
assertEquals("EminemVEVO", extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testId() throws Exception {
|
||||
assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOriginalUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// ListExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testRelatedItems() throws Exception {
|
||||
defaultTestRelatedItems(extractor, YouTube.getServiceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMoreRelatedItems() throws Exception {
|
||||
defaultTestMoreItems(extractor, YouTube.getServiceId());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// ChannelExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testDescription() throws Exception {
|
||||
final String description = extractor.getDescription();
|
||||
assertTrue(description, description.contains("Eminem on Vevo"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAvatarUrl() throws Exception {
|
||||
String avatarUrl = extractor.getAvatarUrl();
|
||||
assertIsSecureUrl(avatarUrl);
|
||||
assertTrue(avatarUrl, avatarUrl.contains("yt3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBannerUrl() throws Exception {
|
||||
String bannerUrl = extractor.getBannerUrl();
|
||||
assertIsSecureUrl(bannerUrl);
|
||||
assertTrue(bannerUrl, bannerUrl.contains("yt3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFeedUrl() throws Exception {
|
||||
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
// there is no "Subscribe" button
|
||||
long subscribers = extractor.getSubscriberCount();
|
||||
assertEquals("Wrong subscriber count", -1, subscribers);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static class RandomChannel implements BaseChannelExtractorTest {
|
||||
private static YoutubeChannelExtractor extractor;
|
||||
|
||||
@ -485,8 +581,9 @@ public class YoutubeChannelExtractorTest {
|
||||
|
||||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
|
||||
long subscribers = extractor.getSubscriberCount();
|
||||
assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -37,6 +37,12 @@ public class YoutubeChannelLinkHandlerFactoryTest {
|
||||
|
||||
assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA"));
|
||||
assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
|
||||
|
||||
assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Gronkh"));
|
||||
assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Netzkino/videos"));
|
||||
|
||||
assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA"));
|
||||
assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -53,5 +59,11 @@ public class YoutubeChannelLinkHandlerFactoryTest {
|
||||
|
||||
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId());
|
||||
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
|
||||
|
||||
assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidio.us/user/Gronkh").getId());
|
||||
assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidio.us/user/Netzkino/videos").getId());
|
||||
|
||||
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId());
|
||||
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
|
||||
}
|
||||
}
|
||||
|
||||
@ -27,18 +27,18 @@ public class YoutubeCommentsExtractorTest {
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeCommentsExtractor) YouTube
|
||||
.getCommentsExtractor("https://www.youtube.com/watch?v=ehn8mJ8vvsI");
|
||||
.getCommentsExtractor("https://www.youtube.com/watch?v=D00Au7k3i6o");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetComments() throws IOException, ExtractionException {
|
||||
boolean result = false;
|
||||
boolean result;
|
||||
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
|
||||
result = findInComments(comments, "Tsuki blyat");
|
||||
result = findInComments(comments, "s1ck m3m3");
|
||||
|
||||
while (comments.hasNextPage() && !result) {
|
||||
comments = extractor.getPage(comments.getNextPageUrl());
|
||||
result = findInComments(comments, "Tsuki blyat");
|
||||
result = findInComments(comments, "s1ck m3m3");
|
||||
}
|
||||
|
||||
assertTrue(result);
|
||||
@ -47,14 +47,14 @@ public class YoutubeCommentsExtractorTest {
|
||||
@Test
|
||||
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
|
||||
boolean result = false;
|
||||
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=ehn8mJ8vvsI");
|
||||
assertTrue("the dark side of YouTube...".equals(commentsInfo.getName()));
|
||||
result = findInComments(commentsInfo.getRelatedItems(), "Tsuki blyat");
|
||||
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=D00Au7k3i6o");
|
||||
assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName()));
|
||||
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
|
||||
|
||||
String nextPage = commentsInfo.getNextPageUrl();
|
||||
while (!StringUtil.isBlank(nextPage) && !result) {
|
||||
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
|
||||
result = findInComments(moreItems.getItems(), "Tsuki blyat");
|
||||
result = findInComments(moreItems.getItems(), "s1ck m3m3");
|
||||
nextPage = moreItems.getNextPageUrl();
|
||||
}
|
||||
|
||||
|
||||
@ -100,7 +100,7 @@ public class YoutubePlaylistExtractorTest {
|
||||
|
||||
@Test
|
||||
public void testUploaderUrl() throws Exception {
|
||||
assertTrue(extractor.getUploaderUrl().contains("youtube.com"));
|
||||
assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -185,8 +185,8 @@ public class YoutubePlaylistExtractorTest {
|
||||
public void testMoreRelatedItems() throws Exception {
|
||||
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage
|
||||
= defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
|
||||
// Test for 2 more levels
|
||||
|
||||
// test for 2 more levels
|
||||
for (int i = 0; i < 2; i++) {
|
||||
currentPage = extractor.getPage(currentPage.getNextPageUrl());
|
||||
defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors());
|
||||
@ -214,7 +214,7 @@ public class YoutubePlaylistExtractorTest {
|
||||
|
||||
@Test
|
||||
public void testUploaderUrl() throws Exception {
|
||||
assertTrue(extractor.getUploaderUrl().contains("youtube.com"));
|
||||
assertEquals("https://www.youtube.com/channel/UCHSPWoY1J5fbDVbcnyeqwdw", extractor.getUploaderUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@ -0,0 +1,107 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
/**
|
||||
* Test for {@link YoutubePlaylistLinkHandlerFactory}
|
||||
*/
|
||||
public class YoutubePlaylistLinkHandlerFactoryTest {
|
||||
private static YoutubePlaylistLinkHandlerFactory linkHandler;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() {
|
||||
linkHandler = YoutubePlaylistLinkHandlerFactory.getInstance();
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void getIdWithNullAsUrl() throws ParsingException {
|
||||
linkHandler.fromId(null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getIdfromYt() throws Exception {
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://m.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
|
||||
assertEquals("OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM", linkHandler.fromUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM").getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAcceptYtUrl() throws ParsingException {
|
||||
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI"));
|
||||
assertTrue(linkHandler.acceptUrl("HTTPS://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("http://www.youtube.com/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("https://m.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeniesInvalidYtUrl() throws ParsingException {
|
||||
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("ftp://www.youtube.com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("www.youtube.com:22/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("youtube . com/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAcceptInvidioUrl() throws ParsingException {
|
||||
assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertTrue(linkHandler.acceptUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI"));
|
||||
assertTrue(linkHandler.acceptUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeniesInvalidInvidioUrl() throws ParsingException {
|
||||
assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("ftp:/invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("invidio.us:22/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("invidio . us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertFalse(linkHandler.acceptUrl("?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetInvidioIdfromUrl() throws ParsingException {
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("HTTPS://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("http://www.invidio.us/watch?v=0JFM3PRZH-k&index=8&list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
|
||||
}
|
||||
}
|
||||
@ -80,6 +80,7 @@ public class YoutubeStreamLinkHandlerFactoryTest {
|
||||
assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
|
||||
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
|
||||
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
|
||||
assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -98,13 +99,14 @@ public class YoutubeStreamLinkHandlerFactoryTest {
|
||||
assertTrue(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
|
||||
assertTrue(linkHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
|
||||
|
||||
assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
|
||||
assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI"));
|
||||
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAcceptHookUrl() throws ParsingException {
|
||||
assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU"));
|
||||
assertTrue(linkHandler.acceptUrl("http://hooktube.com/watch?v=TglNG-yjabU"));
|
||||
assertTrue(linkHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE"));
|
||||
assertTrue(linkHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
|
||||
assertTrue(linkHandler.acceptUrl("hooktube.com/watch/3msbfr6pBNE"));
|
||||
@ -115,10 +117,31 @@ public class YoutubeStreamLinkHandlerFactoryTest {
|
||||
@Test
|
||||
public void testGetHookIdfromUrl() throws ParsingException {
|
||||
assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId());
|
||||
assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://hooktube.com/watch?v=TglNG-yjabU").getId());
|
||||
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch?v=3msbfr6pBNE").getId());
|
||||
assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
|
||||
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch/3msbfr6pBNE").getId());
|
||||
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId());
|
||||
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAcceptInvidioUrl() throws ParsingException {
|
||||
assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU"));
|
||||
assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=TglNG-yjabU"));
|
||||
assertTrue(linkHandler.acceptUrl("http://invidio.us/watch?v=TglNG-yjabU"));
|
||||
assertTrue(linkHandler.acceptUrl("invidio.us/watch?v=3msbfr6pBNE"));
|
||||
assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
|
||||
assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetInvidioIdfromUrl() throws ParsingException {
|
||||
assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId());
|
||||
assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://www.invidio.us/watch?v=TglNG-yjabU").getId());
|
||||
assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://invidio.us/watch?v=TglNG-yjabU").getId());
|
||||
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/watch?v=3msbfr6pBNE").getId());
|
||||
assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
|
||||
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/embed/3msbfr6pBNE").getId());
|
||||
}
|
||||
}
|
||||
@ -59,15 +59,38 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
assertTrue(items.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
|
||||
String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA";
|
||||
String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
|
||||
"<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_id=" + channelId + "\" />" +
|
||||
"</outline></body></opml>";
|
||||
|
||||
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
|
||||
assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1);
|
||||
assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty());
|
||||
assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSubscriptionWithInvalidUrlInSource() throws Exception {
|
||||
String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
|
||||
"<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/>" +
|
||||
"<outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/>" +
|
||||
"<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"\"/>" +
|
||||
"<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"\"/>" +
|
||||
"</outline></body></opml>";
|
||||
|
||||
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
|
||||
assertTrue(items.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidSourceException() {
|
||||
List<String> invalidList = Arrays.asList(
|
||||
"<xml><notvalid></notvalid></xml>",
|
||||
"<opml><notvalid></notvalid></opml>",
|
||||
"<opml><body></body></opml>",
|
||||
"<opml><body><outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/></outline></body></opml>",
|
||||
"<opml><body><outline><outline text=\"invalid\" title=\"url\" type=\"rss\"" +
|
||||
" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/></outline></body></opml>",
|
||||
"",
|
||||
null,
|
||||
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
|
||||
@ -78,11 +101,11 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
if (invalidContent != null) {
|
||||
byte[] bytes = invalidContent.getBytes("UTF-8");
|
||||
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
|
||||
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
|
||||
} else {
|
||||
subscriptionExtractor.fromInputStream(null);
|
||||
fail("Extracting from null String didn't throw an exception");
|
||||
}
|
||||
|
||||
fail("didn't throw exception");
|
||||
} catch (Exception e) {
|
||||
// System.out.println(" -> " + e);
|
||||
boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;
|
||||
|
||||
@ -69,6 +69,10 @@ public class YoutubeTrendingLinkHandlerFactoryTest {
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("https://youtube.com/feed/trending"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("m.youtube.com/feed/trending"));
|
||||
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidio.us/feed/trending"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("https://invidio.us/feed/trending"));
|
||||
assertTrue(LinkHandlerFactory.acceptUrl("invidio.us/feed/trending"));
|
||||
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("https://youtu.be/feed/trending"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("kdskjfiiejfia"));
|
||||
assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/bullshit/feed/trending"));
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.search;
|
||||
|
||||
import org.hamcrest.CoreMatchers;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
@ -53,6 +55,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
|
||||
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl());
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testOnlyContainChannels() {
|
||||
for(InfoItem item : itemsPage.getItems()) {
|
||||
@ -61,4 +64,19 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testChannelUrl() {
|
||||
for(InfoItem item : itemsPage.getItems()) {
|
||||
if (item instanceof ChannelInfoItem) {
|
||||
ChannelInfoItem channel = (ChannelInfoItem) item;
|
||||
|
||||
if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie
|
||||
assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl());
|
||||
} else {
|
||||
assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -73,7 +73,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
|
||||
assertTrue((firstInfoItem instanceof ChannelInfoItem)
|
||||
|| (secondInfoItem instanceof ChannelInfoItem));
|
||||
assertEquals("name", "PewDiePie", channelItem.getName());
|
||||
assertEquals("url","https://www.youtube.com/user/PewDiePie", channelItem.getUrl());
|
||||
assertEquals("url", "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", channelItem.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
@ -71,10 +71,9 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
|
||||
assertFalse(extractor.getUploaderName().isEmpty());
|
||||
}
|
||||
|
||||
@Ignore // Currently there is no way get the length from restricted videos
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertTrue(extractor.getLength() > 0);
|
||||
assertEquals(1789, extractor.getLength());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -97,8 +96,6 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
|
||||
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
|
||||
}
|
||||
|
||||
// FIXME: 25.11.17 Are there no streams or are they not listed?
|
||||
@Ignore
|
||||
@Test
|
||||
public void testGetAudioStreams() throws IOException, ExtractionException {
|
||||
// audio streams are not always necessary
|
||||
@ -1,4 +1,4 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
@ -71,10 +71,9 @@ public class YoutubeStreamExtractorControversialTest {
|
||||
assertFalse(extractor.getUploaderName().isEmpty());
|
||||
}
|
||||
|
||||
@Ignore // Currently there is no way get the length from restricted videos
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertTrue(extractor.getLength() > 0);
|
||||
assertEquals(219, extractor.getLength());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -97,8 +96,6 @@ public class YoutubeStreamExtractorControversialTest {
|
||||
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
|
||||
}
|
||||
|
||||
// FIXME: 25.11.17 Are there no streams or are they not listed?
|
||||
@Ignore
|
||||
@Test
|
||||
public void testGetAudioStreams() throws IOException, ExtractionException {
|
||||
// audio streams are not always necessary
|
||||
@ -113,17 +110,15 @@ public class YoutubeStreamExtractorControversialTest {
|
||||
assertTrue(streams.size() > 0);
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
|
||||
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
|
||||
assertTrue(!extractor.getSubtitlesDefault().isEmpty());
|
||||
assertFalse(extractor.getSubtitlesDefault().isEmpty());
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testGetSubtitlesList() throws IOException, ExtractionException {
|
||||
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
|
||||
assertTrue(!extractor.getSubtitles(MediaFormat.TTML).isEmpty());
|
||||
assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
|
||||
}
|
||||
}
|
||||
@ -1,8 +1,9 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.ExtractorAsserts;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.utils.Localization;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
@ -53,7 +55,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeStreamExtractor) YouTube
|
||||
.getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
|
||||
.getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@ -81,8 +83,8 @@ public class YoutubeStreamExtractorDefaultTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetFullLinksInDescriptlion() throws ParsingException {
|
||||
assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt"));
|
||||
public void testGetFullLinksInDescription() throws ParsingException {
|
||||
assertTrue(extractor.getDescription().contains("http://adele.com"));
|
||||
assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi..."));
|
||||
}
|
||||
|
||||
@ -95,7 +97,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
||||
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertTrue(extractor.getLength() > 0);
|
||||
assertEquals(366, extractor.getLength());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -111,7 +113,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
||||
|
||||
@Test
|
||||
public void testGetUploaderUrl() throws ParsingException {
|
||||
assertTrue(extractor.getUploaderUrl().length() > 0);
|
||||
assertEquals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw", extractor.getUploaderUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -231,4 +233,29 @@ public class YoutubeStreamExtractorDefaultTest {
|
||||
assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
|
||||
}
|
||||
}
|
||||
|
||||
public static class FramesTest {
|
||||
private static YoutubeStreamExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeStreamExtractor) YouTube
|
||||
.getStreamExtractor("https://www.youtube.com/watch?v=HoK9shIJ2xQ");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetFrames() throws ExtractionException {
|
||||
final List<Frameset> frames = extractor.getFrames();
|
||||
assertNotNull(frames);
|
||||
assertFalse(frames.isEmpty());
|
||||
for (final Frameset f : frames) {
|
||||
for (final String url : f.getUrls()) {
|
||||
ExtractorAsserts.assertIsValidUrl(url);
|
||||
ExtractorAsserts.assertIsSecureUrl(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,138 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
|
||||
public class YoutubeStreamExtractorLivestreamTest {
|
||||
private static YoutubeStreamExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeStreamExtractor) YouTube
|
||||
.getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetInvalidTimeStamp() throws ParsingException {
|
||||
assertTrue(extractor.getTimeStamp() + "",
|
||||
extractor.getTimeStamp() <= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTitle() throws ParsingException {
|
||||
assertFalse(extractor.getName().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDescription() throws ParsingException {
|
||||
assertNotNull(extractor.getDescription());
|
||||
assertFalse(extractor.getDescription().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetFullLinksInDescription() throws ParsingException {
|
||||
assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/"));
|
||||
assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba..."));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderName() throws ParsingException {
|
||||
assertNotNull(extractor.getUploaderName());
|
||||
assertFalse(extractor.getUploaderName().isEmpty());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertEquals(0, extractor.getLength());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetViewCount() throws ParsingException {
|
||||
long count = extractor.getViewCount();
|
||||
assertTrue(Long.toString(count), count >= 7148995);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploadDate() throws ParsingException {
|
||||
assertTrue(extractor.getUploadDate().length() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetThumbnailUrl() throws ParsingException {
|
||||
assertIsSecureUrl(extractor.getThumbnailUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderAvatarUrl() throws ParsingException {
|
||||
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAudioStreams() throws ExtractionException {
|
||||
assertFalse(extractor.getAudioStreams().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetVideoStreams() throws ExtractionException {
|
||||
for (VideoStream s : extractor.getVideoStreams()) {
|
||||
assertIsSecureUrl(s.url);
|
||||
assertTrue(s.resolution.length() > 0);
|
||||
assertTrue(Integer.toString(s.getFormatId()),
|
||||
0 <= s.getFormatId() && s.getFormatId() <= 0x100);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStreamType() throws ParsingException {
|
||||
assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDashMpd() throws ParsingException {
|
||||
// we dont expect this particular video to have a DASH file. For this purpouse we use a different test class.
|
||||
assertTrue(extractor.getDashMpdUrl(), extractor.getDashMpdUrl().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetRelatedVideos() throws ExtractionException, IOException {
|
||||
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
|
||||
Utils.printErrors(relatedVideos.getErrors());
|
||||
assertFalse(relatedVideos.getItems().isEmpty());
|
||||
assertTrue(relatedVideos.getErrors().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
|
||||
assertTrue(extractor.getSubtitlesDefault().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSubtitlesList() throws IOException, ExtractionException {
|
||||
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
package org.schabi.newpipe.extractor.utils;
|
||||
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class UtilsTest {
|
||||
@Test
|
||||
public void testMixedNumberWordToLong() throws JsonParserException, ParsingException {
|
||||
assertEquals(10, Utils.mixedNumberWordToLong("10"));
|
||||
assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0);
|
||||
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0);
|
||||
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10,5M"), 0.0);
|
||||
assertEquals(1.5e9, Utils.mixedNumberWordToLong("1,5B"), 0.0);
|
||||
}
|
||||
}
|
||||
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
Binary file not shown.
3
gradle/wrapper/gradle-wrapper.properties
vendored
3
gradle/wrapper/gradle-wrapper.properties
vendored
@ -1,6 +1,5 @@
|
||||
#Fri Jan 18 11:51:40 CET 2019
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-5.5.1-bin.zip
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-all.zip
|
||||
|
||||
18
gradlew
vendored
18
gradlew
vendored
@ -1,5 +1,21 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
#
|
||||
# Copyright 2015 the original author or authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
##
|
||||
## Gradle start up script for UN*X
|
||||
@ -28,7 +44,7 @@ APP_NAME="Gradle"
|
||||
APP_BASE_NAME=`basename "$0"`
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS=""
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD="maximum"
|
||||
|
||||
18
gradlew.bat
vendored
18
gradlew.bat
vendored
@ -1,3 +1,19 @@
|
||||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%" == "" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS=
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
Loading…
Reference in New Issue
Block a user