mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2024-12-15 14:50:33 +05:30
fix:invidious: getID and onAccpetURl for comments
This commit is contained in:
B0pol
parent
4aaf123f24
commit
a12c0e2a37
@ -1,13 +1,15 @@
|
|||||||
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
|
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
|
||||||
|
|
||||||
import java.io.UnsupportedEncodingException;
|
|
||||||
import java.net.URLDecoder;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
|
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
|
||||||
import org.schabi.newpipe.extractor.utils.Parser;
|
import org.schabi.newpipe.extractor.utils.Utils;
|
||||||
|
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
|
public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
|
||||||
|
|
||||||
@ -18,75 +20,164 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
|
|||||||
return instance;
|
return instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static String assertIsID(String id) throws ParsingException {
|
||||||
|
if (id == null || !id.matches("[a-zA-Z0-9_-]{11}")) {
|
||||||
|
throw new ParsingException("The given string is not a Youtube-Video-ID");
|
||||||
|
}
|
||||||
|
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getUrl(String id) {
|
public String getUrl(String id) {
|
||||||
return "https://m.youtube.com/watch?v=" + id;
|
return "https://m.youtube.com/watch?v=" + id;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getId(String url) throws ParsingException, IllegalArgumentException {
|
public String getId(String urlString) throws ParsingException, IllegalArgumentException {
|
||||||
if (url.isEmpty()) {
|
|
||||||
throw new IllegalArgumentException("The url parameter should not be empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
String id;
|
|
||||||
String lowercaseUrl = url.toLowerCase();
|
|
||||||
if (lowercaseUrl.contains("youtube")) {
|
|
||||||
if (url.contains("attribution_link")) {
|
|
||||||
try {
|
try {
|
||||||
String escapedQuery = Parser.matchGroup1("u=(.[^&|$]*)", url);
|
URI uri = new URI(urlString);
|
||||||
String query = URLDecoder.decode(escapedQuery, "UTF-8");
|
String scheme = uri.getScheme();
|
||||||
id = Parser.matchGroup1("v=" + ID_PATTERN, query);
|
|
||||||
} catch (UnsupportedEncodingException uee) {
|
if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
|
||||||
throw new ParsingException("Could not parse attribution_link", uee);
|
String schemeSpecificPart = uri.getSchemeSpecificPart();
|
||||||
}
|
if (schemeSpecificPart.startsWith("//")) {
|
||||||
} else if (url.contains("vnd.youtube")) {
|
urlString = "https:" + schemeSpecificPart;
|
||||||
id = Parser.matchGroup1(ID_PATTERN, url);
|
|
||||||
} else if (url.contains("embed")) {
|
|
||||||
id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
|
|
||||||
} else if (url.contains("googleads")) {
|
|
||||||
throw new FoundAdException("Error found add: " + url);
|
|
||||||
} else {
|
} else {
|
||||||
id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
|
return assertIsID(schemeSpecificPart);
|
||||||
}
|
}
|
||||||
} else if (lowercaseUrl.contains("youtu.be")) {
|
|
||||||
if (url.contains("v=")) {
|
|
||||||
id = Parser.matchGroup1("v=" + ID_PATTERN, url);
|
|
||||||
} else {
|
|
||||||
id = Parser.matchGroup1("[Yy][Oo][Uu][Tt][Uu]\\.[Bb][Ee]/" + ID_PATTERN, url);
|
|
||||||
}
|
}
|
||||||
} else if(lowercaseUrl.contains("hooktube")) {
|
} catch (URISyntaxException ignored) {
|
||||||
if(lowercaseUrl.contains("&v=")
|
|
||||||
|| lowercaseUrl.contains("?v=")) {
|
|
||||||
id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
|
|
||||||
} else if (url.contains("/embed/")) {
|
|
||||||
id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
|
|
||||||
} else if (url.contains("/v/")) {
|
|
||||||
id = Parser.matchGroup1("v/" + ID_PATTERN, url);
|
|
||||||
} else if (url.contains("/watch/")) {
|
|
||||||
id = Parser.matchGroup1("watch/" + ID_PATTERN, url);
|
|
||||||
} else {
|
|
||||||
throw new ParsingException("Error no suitable url: " + url);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw new ParsingException("Error no suitable url: " + url);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
URL url;
|
||||||
if (!id.isEmpty()) {
|
try {
|
||||||
return id;
|
url = Utils.stringToURL(urlString);
|
||||||
} else {
|
} catch (MalformedURLException e) {
|
||||||
throw new ParsingException("Error could not parse url: " + url);
|
throw new IllegalArgumentException("The given URL is not valid");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
String host = url.getHost();
|
||||||
|
String path = url.getPath();
|
||||||
|
// remove leading "/" of URL-path if URL-path is given
|
||||||
|
if (!path.isEmpty()) {
|
||||||
|
path = path.substring(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) ||
|
||||||
|
YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) ||
|
||||||
|
YoutubeParsingHelper.isInvidioURL(url))) {
|
||||||
|
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
|
||||||
|
throw new FoundAdException("Error found ad: " + urlString);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new ParsingException("The url is not a Youtube-URL");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (YoutubePlaylistLinkHandlerFactory.getInstance().acceptUrl(urlString)) {
|
||||||
|
throw new ParsingException("Error no suitable url: " + urlString);
|
||||||
|
}
|
||||||
|
|
||||||
|
// using uppercase instead of lowercase, because toLowercase replaces some unicode characters
|
||||||
|
// with their lowercase ASCII equivalent. Using toLowercase could result in faultily matching unicode urls.
|
||||||
|
switch (host.toUpperCase()) {
|
||||||
|
case "WWW.YOUTUBE-NOCOOKIE.COM": {
|
||||||
|
if (path.startsWith("embed/")) {
|
||||||
|
String id = path.split("/")[1];
|
||||||
|
|
||||||
|
return assertIsID(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case "YOUTUBE.COM":
|
||||||
|
case "WWW.YOUTUBE.COM":
|
||||||
|
case "M.YOUTUBE.COM":
|
||||||
|
case "MUSIC.YOUTUBE.COM": {
|
||||||
|
if (path.equals("attribution_link")) {
|
||||||
|
String uQueryValue = Utils.getQueryValue(url, "u");
|
||||||
|
|
||||||
|
URL decodedURL;
|
||||||
|
try {
|
||||||
|
decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue);
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
throw new ParsingException("Error no suitable url: " + urlString);
|
||||||
|
}
|
||||||
|
|
||||||
|
String viewQueryValue = Utils.getQueryValue(decodedURL, "v");
|
||||||
|
return assertIsID(viewQueryValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (path.startsWith("embed/")) {
|
||||||
|
String id = path.split("/")[1];
|
||||||
|
|
||||||
|
return assertIsID(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
String viewQueryValue = Utils.getQueryValue(url, "v");
|
||||||
|
return assertIsID(viewQueryValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
case "YOUTU.BE": {
|
||||||
|
String viewQueryValue = Utils.getQueryValue(url, "v");
|
||||||
|
if (viewQueryValue != null) {
|
||||||
|
return assertIsID(viewQueryValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
return assertIsID(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
case "HOOKTUBE.COM": {
|
||||||
|
if (path.startsWith("v/")) {
|
||||||
|
String id = path.substring("v/".length());
|
||||||
|
|
||||||
|
return assertIsID(id);
|
||||||
|
}
|
||||||
|
if (path.startsWith("watch/")) {
|
||||||
|
String id = path.substring("watch/".length());
|
||||||
|
|
||||||
|
return assertIsID(id);
|
||||||
|
}
|
||||||
|
// there is no break-statement here on purpose so the next code-block gets also run for hooktube
|
||||||
|
}
|
||||||
|
|
||||||
|
case "WWW.INVIDIO.US":
|
||||||
|
case "DEV.INVIDIO.US":
|
||||||
|
case "INVIDIO.US":
|
||||||
|
case "INVIDIOUS.SNOPYTA.ORG":
|
||||||
|
case "DE.INVIDIOUS.SNOPYTA.ORG":
|
||||||
|
case "FI.INVIDIOUS.SNOPYTA.ORG":
|
||||||
|
case "VID.WXZM.SX":
|
||||||
|
case "INVIDIOUS.KABI.TK":
|
||||||
|
case "INVIDIOU.SH":
|
||||||
|
case "WWW.INVIDIOU.SH":
|
||||||
|
case "NO.INVIDIOU.SH":
|
||||||
|
case "INVIDIOUS.ENKIRTON.NET":
|
||||||
|
case "TUBE.POAL.CO":
|
||||||
|
case "INVIDIOUS.13AD.DE":
|
||||||
|
case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances
|
||||||
|
if (path.equals("watch")) {
|
||||||
|
String viewQueryValue = Utils.getQueryValue(url, "v");
|
||||||
|
if (viewQueryValue != null) {
|
||||||
|
return assertIsID(viewQueryValue);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (path.startsWith("embed/")) {
|
||||||
|
String id = path.substring("embed/".length());
|
||||||
|
|
||||||
|
return assertIsID(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new ParsingException("Error no suitable url: " + urlString);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean onAcceptUrl(final String url) throws FoundAdException {
|
public boolean onAcceptUrl(final String url) throws FoundAdException {
|
||||||
final String lowercaseUrl = url.toLowerCase();
|
|
||||||
if (lowercaseUrl.contains("youtube")
|
|
||||||
|| lowercaseUrl.contains("youtu.be")
|
|
||||||
|| lowercaseUrl.contains("hooktube")) {
|
|
||||||
// bad programming I know
|
|
||||||
try {
|
try {
|
||||||
getId(url);
|
getId(url);
|
||||||
return true;
|
return true;
|
||||||
@ -95,9 +186,6 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
|
|||||||
} catch (ParsingException e) {
|
} catch (ParsingException e) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@ -27,7 +27,7 @@ public class YoutubeCommentsExtractorTest {
|
|||||||
public static void setUp() throws Exception {
|
public static void setUp() throws Exception {
|
||||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||||
extractor = (YoutubeCommentsExtractor) YouTube
|
extractor = (YoutubeCommentsExtractor) YouTube
|
||||||
.getCommentsExtractor("https://www.youtube.com/watch?v=D00Au7k3i6o");
|
.getCommentsExtractor("https://www.invidio.us/watch?v=D00Au7k3i6o");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -47,7 +47,7 @@ public class YoutubeCommentsExtractorTest {
|
|||||||
@Test
|
@Test
|
||||||
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
|
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
|
||||||
boolean result = false;
|
boolean result = false;
|
||||||
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=D00Au7k3i6o");
|
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.invidio.us/watch?v=D00Au7k3i6o");
|
||||||
assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName()));
|
assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName()));
|
||||||
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
|
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user