Merge branch 'dev' into dev

.travis.yml

@@ -1,7 +1,10 @@
 language: java
+jdk:
+  - openjdk8
 
-script: ./gradlew check
+script:
-after_success: ./gradlew aggregatedJavadocs
+  - ./gradlew check
+  - ./gradlew aggregatedJavadocs
 
 deploy:
   provider: pages

build.gradle

@@ -5,7 +5,7 @@ allprojects {
     sourceCompatibility = 1.7
     targetCompatibility = 1.7
 
-    version 'v0.13.0'
+    version 'v0.18.0'
     group 'com.github.TeamNewPipe'
 
     repositories {
@@ -43,7 +43,8 @@ task aggregatedJavadocs(type: Javadoc, group: 'Documentation') {
     title = "$project.name $version"
     // options.memberLevel = JavadocMemberLevel.PRIVATE
     options.links 'https://docs.oracle.com/javase/7/docs/api/'
-    
+    options.encoding 'UTF-8'
+
     subprojects.each { project ->
         project.tasks.withType(Javadoc).each { javadocTask ->
             source += javadocTask.source

extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -56,7 +56,7 @@ public abstract class Extractor{
     }
 
     protected void assertPageFetched() {
-        if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()");
+        if (!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()");
     }
 
     protected boolean isPageFetched() {

extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.feed.FeedExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
@@ -24,6 +25,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
+import javax.annotation.Nullable;
+
 /*
  * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
  * StreamingService.java is part of NewPipe.
@@ -65,7 +68,7 @@ public abstract class StreamingService {
         public String getName() {
             return name;
         }
-        
+
         public List<MediaCapability> getMediaCapabilities() {
             return mediaCapabilities;
         }
@@ -116,7 +119,7 @@ public abstract class StreamingService {
     public String toString() {
         return serviceId + ":" + serviceInfo.getName();
     }
-    
+
     public abstract String getBaseUrl();
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -173,6 +176,19 @@ public abstract class StreamingService {
      */
     public abstract SubscriptionExtractor getSubscriptionExtractor();
 
+    /**
+     * This method decides which strategy will be chosen to fetch the feed. In YouTube, for example, a separate feed
+     * exists which is lightweight and made specifically to be used like this.
+     * <p>
+     * In services which there's no other way to retrieve them, null should be returned.
+     *
+     * @return a {@link FeedExtractor} instance or null.
+     */
+    @Nullable
+    public FeedExtractor getFeedExtractor(String url) throws ExtractionException {
+        return null;
+    }
+
     /**
      * Must create a new instance of a KioskList implementation.
      * @return a new KioskList instance
@@ -258,7 +274,7 @@ public abstract class StreamingService {
         }
         return getCommentsExtractor(llhf.fromUrl(url));
     }
-    
+
     /*//////////////////////////////////////////////////////////////////////////
     // Utils
     //////////////////////////////////////////////////////////////////////////*/

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java

@@ -5,9 +5,7 @@ import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 
@@ -35,8 +33,8 @@ import java.io.IOException;
 
 public class ChannelInfo extends ListInfo<StreamInfoItem> {
 
-    public ChannelInfo(int serviceId, ListLinkHandler linkHandler, String name) throws ParsingException {
+    public ChannelInfo(int serviceId, String id, String url, String originalUrl, String name, ListLinkHandler listLinkHandler) {
-        super(serviceId, linkHandler, name);
+        super(serviceId, id, url, originalUrl, name, listLinkHandler.getContentFilters(), listLinkHandler.getSortFilter());
     }
 
     public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
@@ -57,15 +55,14 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
 
     public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
 
-        ChannelInfo info = new ChannelInfo(extractor.getServiceId(),
+        final int serviceId = extractor.getServiceId();
-                extractor.getLinkHandler(),
+        final String id = extractor.getId();
-                extractor.getName());
+        final String url = extractor.getUrl();
+        final String originalUrl = extractor.getOriginalUrl();
+        final String name = extractor.getName();
+
+        final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name, extractor.getLinkHandler());
 
-        try {
-            info.setOriginalUrl(extractor.getOriginalUrl());
-        } catch (Exception e) {
-            info.addError(e);
-        }
         try {
             info.setAvatarUrl(extractor.getAvatarUrl());
         } catch (Exception e) {

extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedExtractor.java

@@ -0,0 +1,17 @@
+package org.schabi.newpipe.extractor.feed;
+
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+
+/**
+ * This class helps to extract items from lightweight feeds that the services may provide.
+ * <p>
+ * YouTube is an example of a service that has this alternative available.
+ */
+public abstract class FeedExtractor extends ListExtractor<StreamInfoItem> {
+    public FeedExtractor(StreamingService service, ListLinkHandler listLinkHandler) {
+        super(service, listLinkHandler);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java

@@ -0,0 +1,52 @@
+package org.schabi.newpipe.extractor.feed;
+
+import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
+import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.utils.ExtractorHelper;
+
+import java.io.IOException;
+import java.util.List;
+
+public class FeedInfo extends ListInfo<StreamInfoItem> {
+
+    public FeedInfo(int serviceId, String id, String url, String originalUrl, String name, List<String> contentFilter, String sortFilter) {
+        super(serviceId, id, url, originalUrl, name, contentFilter, sortFilter);
+    }
+
+    public static FeedInfo getInfo(String url) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url);
+    }
+
+    public static FeedInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
+        final FeedExtractor extractor = service.getFeedExtractor(url);
+
+        if (extractor == null) {
+            throw new IllegalArgumentException("Service \"" + service.getServiceInfo().getName() + "\" doesn't support FeedExtractor.");
+        }
+
+        extractor.fetchPage();
+        return getInfo(extractor);
+    }
+
+    public static FeedInfo getInfo(FeedExtractor extractor) throws IOException, ExtractionException {
+        extractor.fetchPage();
+
+        final int serviceId = extractor.getServiceId();
+        final String id = extractor.getId();
+        final String url = extractor.getUrl();
+        final String originalUrl = extractor.getOriginalUrl();
+        final String name = extractor.getName();
+
+        final FeedInfo info = new FeedInfo(serviceId, id, url, originalUrl, name, null, null);
+
+        final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
+        info.setRelatedItems(itemsPage.getItems());
+        info.setNextPageUrl(itemsPage.getNextPageUrl());
+
+        return info;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java

@@ -34,7 +34,7 @@ public abstract class LinkHandlerFactory {
     public abstract String getUrl(String id) throws ParsingException;
     public abstract boolean onAcceptUrl(final String url) throws ParsingException;
 
-    public String getUrl(String id, String baseUrl) throws ParsingException{
+    public String getUrl(String id, String baseUrl) throws ParsingException {
        return getUrl(id);
     }
     
@@ -43,13 +43,14 @@ public abstract class LinkHandlerFactory {
     ///////////////////////////////////
 
     public LinkHandler fromUrl(String url) throws ParsingException {
+        if (url == null) throw new IllegalArgumentException("url can not be null");
         final String baseUrl = Utils.getBaseUrl(url);
         return fromUrl(url, baseUrl);
     }
     
     public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
-        if(url == null) throw new IllegalArgumentException("url can not be null");
+        if (url == null) throw new IllegalArgumentException("url can not be null");
-        if(!acceptUrl(url)) {
+        if (!acceptUrl(url)) {
             throw new ParsingException("Malformed unacceptable url: " + url);
         }
 
@@ -58,13 +59,13 @@ public abstract class LinkHandlerFactory {
     }
     
     public LinkHandler fromId(String id) throws ParsingException {
-        if(id == null) throw new IllegalArgumentException("id can not be null");
+        if (id == null) throw new IllegalArgumentException("id can not be null");
         final String url = getUrl(id);
         return new LinkHandler(url, url, id);
     }
 
     public LinkHandler fromId(String id, String baseUrl) throws ParsingException {
-        if(id == null) throw new IllegalArgumentException("id can not be null");
+        if (id == null) throw new IllegalArgumentException("id can not be null");
         final String url = getUrl(id, baseUrl);
         return new LinkHandler(url, url, id);
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.feed.FeedExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
@@ -24,14 +25,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
+import org.schabi.newpipe.extractor.services.youtube.extractors.*;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
@@ -42,6 +36,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
+import javax.annotation.Nonnull;
+
 /*
  * Created by Christian Schabesberger on 23.08.15.
  *
@@ -72,7 +68,7 @@ public class YoutubeService extends StreamingService {
     public String getBaseUrl() {
         return "https://youtube.com";
     }
-    
+
     @Override
     public LinkHandlerFactory getStreamLHFactory() {
         return YoutubeStreamLinkHandlerFactory.getInstance();
@@ -147,6 +143,12 @@ public class YoutubeService extends StreamingService {
         return new YoutubeSubscriptionExtractor(this);
     }
 
+    @Nonnull
+    @Override
+    public FeedExtractor getFeedExtractor(final String channelUrl) throws ExtractionException {
+        return new YoutubeFeedExtractor(this, getChannelLHFactory().fromUrl(channelUrl));
+    }
+
     @Override
     public ListLinkHandlerFactory getCommentsLHFactory() {
         return YoutubeCommentsLinkHandlerFactory.getInstance();

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -46,7 +46,6 @@ import java.io.IOException;
 @SuppressWarnings("WeakerAccess")
 public class YoutubeChannelExtractor extends ChannelExtractor {
     /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
-    private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
     private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
 
     private Document doc;
@@ -130,7 +129,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getFeedUrl() throws ParsingException {
         try {
-            return CHANNEL_FEED_BASE + getId();
+            return YoutubeParsingHelper.getFeedUrlFrom(getId());
         } catch (Exception e) {
             throw new ParsingException("Could not get feed url", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java

@@ -0,0 +1,82 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.feed.FeedExtractor;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+public class YoutubeFeedExtractor extends FeedExtractor {
+    public YoutubeFeedExtractor(StreamingService service, ListLinkHandler linkHandler) {
+        super(service, linkHandler);
+    }
+
+    private Document document;
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        final String channelIdOrUser = getLinkHandler().getId();
+        final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser);
+
+        final Response response = downloader.get(feedUrl);
+        document = Jsoup.parse(response.responseBody());
+    }
+
+    @Nonnull
+    @Override
+    public ListExtractor.InfoItemsPage<StreamInfoItem> getInitialPage() {
+        final Elements entries = document.select("feed > entry");
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+
+        for (Element entryElement : entries) {
+            collector.commit(new YoutubeFeedInfoItemExtractor(entryElement));
+        }
+
+        return new InfoItemsPage<>(collector, null);
+    }
+
+    @Nonnull
+    @Override
+    public String getId() {
+        return document.getElementsByTag("yt:channelId").first().text();
+    }
+
+    @Nonnull
+    @Override
+    public String getUrl() {
+        return document.select("feed > author > uri").first().text();
+    }
+
+    @Nonnull
+    @Override
+    public String getName() {
+        return document.select("feed > author > name").first().text();
+    }
+
+    @Override
+    public String getNextPageUrl() {
+        return null;
+    }
+
+    @Override
+    public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) {
+        return null;
+    }
+
+    @Override
+    public boolean hasNextPage() {
+        return false;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java

@@ -0,0 +1,94 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.StreamType;
+
+import javax.annotation.Nullable;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.TimeZone;
+
+public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
+    private final Element entryElement;
+
+    public YoutubeFeedInfoItemExtractor(Element entryElement) {
+        this.entryElement = entryElement;
+    }
+
+    @Override
+    public StreamType getStreamType() {
+        // It is not possible to determine the stream type using the feed endpoint.
+        // All entries are considered a video stream.
+        return StreamType.VIDEO_STREAM;
+    }
+
+    @Override
+    public boolean isAd() {
+        return false;
+    }
+
+    @Override
+    public long getDuration() {
+        // Not available when fetching through the feed endpoint.
+        return -1;
+    }
+
+    @Override
+    public long getViewCount() {
+        return Long.parseLong(entryElement.getElementsByTag("media:statistics").first().attr("views"));
+    }
+
+    @Override
+    public String getUploaderName() {
+        return entryElement.select("author > name").first().text();
+    }
+
+    @Override
+    public String getUploaderUrl() {
+        return entryElement.select("author > uri").first().text();
+    }
+
+    @Nullable
+    @Override
+    public String getTextualUploadDate() {
+        return entryElement.getElementsByTag("published").first().text();
+    }
+
+    @Nullable
+    @Override
+    public DateWrapper getUploadDate() throws ParsingException {
+        final Date date;
+        try {
+            final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00");
+            dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+            date = dateFormat.parse(getTextualUploadDate());
+        } catch (ParseException e) {
+            throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e);
+        }
+
+        final Calendar calendar = Calendar.getInstance();
+        calendar.setTime(date);
+
+        return new DateWrapper(calendar);
+    }
+
+    @Override
+    public String getName() {
+        return entryElement.getElementsByTag("title").first().text();
+    }
+
+    @Override
+    public String getUrl() {
+        return entryElement.getElementsByTag("link").first().attr("href");
+    }
+
+    @Override
+    public String getThumbnailUrl() {
+        return entryElement.getElementsByTag("media:thumbnail").first().attr("url");
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -22,8 +22,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
-import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.*;
@@ -106,19 +106,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public String getName() throws ParsingException {
         assertPageFetched();
-        String name = getStringFromMetaData("title");
+        try {
-        if(name == null) {
+            return playerResponse.getObject("videoDetails").getString("title");
-            // Fallback to HTML method
+
+        } catch (Exception e) {
+            // fallback HTML method
+            String name = null;
             try {
                 name = doc.select("meta[name=title]").attr(CONTENT);
-            } catch (Exception e) {
+            } catch (Exception ignored) {}
-                throw new ParsingException("Could not get the title", e);
+
+            if (name == null) {
+                throw new ParsingException("Could not get name", e);
             }
+            return name;
         }
-        if(name == null || name.isEmpty()) {
-            throw new ParsingException("Could not get the title");
-        }
-        return name;
     }
 
     @Override
@@ -128,9 +130,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
 
         try {
-            return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
+            return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate");
-        } catch (Exception e) {//todo: add fallback method
+        } catch (Exception e) {
-            throw new ParsingException("Could not get upload date", e);
+            String uploadDate = null;
+            try {
+                uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT);
+            } catch (Exception ignored) {}
+
+            if (uploadDate == null) {
+                throw new ParsingException("Could not get upload date", e);
+            }
+            return uploadDate;
         }
     }
 
@@ -142,31 +152,30 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             return null;
         }
 
-        return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate));
+        return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate), true);
     }
 
     @Nonnull
     @Override
     public String getThumbnailUrl() throws ParsingException {
         assertPageFetched();
-        // Try to get high resolution thumbnail first, if it fails, use low res from the player instead
         try {
-            return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
+            JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
-        } catch (Exception ignored) {
+            // the last thumbnail is the one with the highest resolution
-            // Try other method...
+            return thumbnails.getObject(thumbnails.size() - 1).getString("url");
-        }
 
-        try {
-            if (playerArgs != null && playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url");
-        } catch (Exception ignored) {
-            // Try other method...
-        }
-
-        try {
-            return videoInfoPage.get("thumbnail_url");
         } catch (Exception e) {
-            throw new ParsingException("Could not get thumbnail url", e);
+            String url = null;
+            try {
+                url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
+            } catch (Exception ignored) {}
+
+            if (url == null) {
+                throw new ParsingException("Could not get thumbnail url", e);
+            }
+            return url;
         }
+
     }
 
     @Nonnull
@@ -174,9 +183,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getDescription() throws ParsingException {
         assertPageFetched();
         try {
+            // first try to get html-formatted description
             return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html());
         } catch (Exception e) {
-            throw new ParsingException("Could not get the description", e);
+            try {
+                // fallback to raw non-html description
+                return playerResponse.getObject("videoDetails").getString("shortDescription");
+            } catch (Exception ignored) {
+                throw new ParsingException("Could not get the description", e);
+            }
         }
     }
 
@@ -269,25 +284,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public long getLength() throws ParsingException {
         assertPageFetched();
 
-        // try getting duration from playerargs
-        try {
-            String durationMs = playerResponse
-                    .getObject("streamingData")
-                    .getArray("formats")
-                    .getObject(0)
-                    .getString("approxDurationMs");
-            return Long.parseLong(durationMs)/1000;
-        } catch (Exception e) {
-        }
-
-        //try getting value from age gated video
         try {
             String duration = playerResponse
                     .getObject("videoDetails")
                     .getString("lengthSeconds");
             return Long.parseLong(duration);
         } catch (Exception e) {
-            throw new ParsingException("Every methode to get the duration has failed: ", e);
+            try {
+                String durationMs = playerResponse
+                        .getObject("streamingData")
+                        .getArray("formats")
+                        .getObject(0)
+                        .getString("approxDurationMs");
+                return Math.round(Long.parseLong(durationMs) / 1000f);
+            } catch (Exception ignored) {
+                throw new ParsingException("Could not get duration", e);
+            }
         }
     }
 
@@ -307,11 +319,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         try {
             if (getStreamType().equals(StreamType.LIVE_STREAM)) {
                 return getLiveStreamWatchingCount();
+            } else {
+                return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount"));
+            }
+        } catch (Exception e) {
+            try {
+                return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
+            } catch (Exception ignored) {
+                throw new ParsingException("Could not get view count", e);
             }
-
-            return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
-        } catch (Exception e) {//todo: find fallback method
-            throw new ParsingException("Could not get number of views", e);
         }
     }
 
@@ -373,7 +389,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             try {
                 likesString = button.select("span.yt-uix-button-content").first().text();
             } catch (NullPointerException e) {
-                //if this kicks in our button has no content and therefore likes/dislikes are disabled
+                //if this kicks in our button has no content and therefore ratings must be disabled
+                if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
+                    throw new ParsingException("Ratings are enabled even though the like button is missing", e);
+                }
                 return -1;
             }
             return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
@@ -393,7 +412,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             try {
                 dislikesString = button.select("span.yt-uix-button-content").first().text();
             } catch (NullPointerException e) {
-                //if this kicks in our button has no content and therefore likes/dislikes are disabled
+                //if this kicks in our button has no content and therefore ratings must be disabled
+                if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
+                    throw new ParsingException("Ratings are enabled even though the dislike button is missing", e);
+                }
                 return -1;
             }
             return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
@@ -409,60 +431,59 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getUploaderUrl() throws ParsingException {
         assertPageFetched();
         try {
-            return doc.select("div[class=\"yt-user-info\"]").first().children()
+            return "https://www.youtube.com/channel/" +
-                    .select("a").first().attr("abs:href");
+                    playerResponse.getObject("videoDetails").getString("channelId");
         } catch (Exception e) {
-            throw new ParsingException("Could not get channel link", e);
+            String uploaderUrl = null;
-        }
+            try {
-    }
+                uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children()
+                        .select("a").first().attr("abs:href");
+            } catch (Exception ignored) {}
 
-
+            if (uploaderUrl == null) {
-    @Nullable
+                throw new ParsingException("Could not get channel link", e);
-    private String getStringFromMetaData(String field) {
+            }
-        assertPageFetched();
+            return uploaderUrl;
-        String value = null;
-        if(playerArgs != null) {
-            // This can not fail
-            value = playerArgs.getString(field);
         }
-        if(value == null) {
-            // This can not fail too
-            value = videoInfoPage.get(field);
-        }
-        return value;
     }
 
     @Nonnull
     @Override
     public String getUploaderName() throws ParsingException {
         assertPageFetched();
-        String name = getStringFromMetaData("author");
+        try {
-
+            return playerResponse.getObject("videoDetails").getString("author");
-        if(name == null) {
+        } catch (Exception e) {
+            String name = null;
             try {
-                // Fallback to HTML method
                 name = doc.select("div.yt-user-info").first().text();
-            } catch (Exception e) {
+            } catch (Exception ignored) {}
-                throw new ParsingException("Could not get uploader name", e);
+
+            if (name == null) {
+                throw new ParsingException("Could not get uploader name");
             }
+            return name;
         }
-        if(name == null || name.isEmpty()) {
-            throw new ParsingException("Could not get uploader name");
-        }
-        return name;
     }
 
     @Nonnull
     @Override
     public String getUploaderAvatarUrl() throws ParsingException {
         assertPageFetched();
+
+        String uploaderAvatarUrl = null;
         try {
-            return doc.select("a[class*=\"yt-user-photo\"]").first()
+            uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first()
                     .select("img").first()
                     .attr("abs:data-thumb");
         } catch (Exception e) {//todo: add fallback method
-            throw new ParsingException("Could not get uploader thumbnail URL.", e);
+            throw new ParsingException("Could not get uploader avatar url", e);
         }
+
+        if (uploaderAvatarUrl == null) {
+            throw new ParsingException("Could not get uploader avatar url");
+        }
+        return uploaderAvatarUrl;
     }
 
     @Nonnull
@@ -590,12 +611,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public StreamType getStreamType() throws ParsingException {
         assertPageFetched();
         try {
-            if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
+            if (!playerResponse.getObject("streamingData").has(FORMATS) ||
-                    (!playerResponse.getObject("streamingData").has(FORMATS)))) {
+                    (playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) {
                 return StreamType.LIVE_STREAM;
             }
         } catch (Exception e) {
-            throw new ParsingException("Could not get hls manifest url", e);
+            throw new ParsingException("Could not get stream type", e);
         }
         return StreamType.VIDEO_STREAM;
     }
@@ -677,8 +698,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
 
-    private final static String DECYRYPTION_SIGNATURE_FUNCTION_REGEX =
+    private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX =
             "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
+    private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 =
+            "\\b([\\w$]{2})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
     private final static String DECRYPTION_AKAMAIZED_STRING_REGEX =
             "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(";
     private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX =
@@ -773,7 +796,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private JsonObject getPlayerResponse() throws ParsingException {
         try {
             String playerResponseStr;
-            if(playerArgs != null) {
+            if (playerArgs != null) {
                 playerResponseStr = playerArgs.getString("player_response");
             } else {
                 playerResponseStr = videoInfoPage.get("player_response");
@@ -805,7 +828,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
                 return new EmbeddedInfo(playerUrl, sts);
             } catch (Exception i) {
-                // if it failes we simply reply with no sts as then it does not seem to be necessary
+                // if it fails we simply reply with no sts as then it does not seem to be necessary
                 return new EmbeddedInfo(playerUrl, "");
             }
 
@@ -868,30 +891,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     private String getDecryptionFuncName(String playerCode) throws DecryptException {
-        String decryptionFunctionName;
+        String[] decryptionFuncNameRegexes = {
-        // Cascading things in catch is ugly, but its faster than running a match before getting the actual name
+                DECRYPTION_SIGNATURE_FUNCTION_REGEX_2,
-        // to se if the function can actually be found with the given regex.
+                DECRYPTION_SIGNATURE_FUNCTION_REGEX,
-        // However if this cascading should propably be cleaned up somehow as it looks a bit weird.
+                DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX,
-        try {
+                DECRYPTION_AKAMAIZED_STRING_REGEX
-            decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
+        };
-        } catch (Parser.RegexException re) {
+        Parser.RegexException exception = null;
+        for (String regex : decryptionFuncNameRegexes) {
             try {
-                decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
+                return Parser.matchGroup1(regex, playerCode);
-            } catch (Parser.RegexException re2) {
+            } catch (Parser.RegexException re) {
-                try {
+                if (exception == null)
-                    decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
+                    exception = re;
-                } catch (Parser.RegexException re3) {
-                    throw new DecryptException("Could not find decrypt function with any of the given patterns.", re);
-                }
             }
         }
-        return decryptionFunctionName;
+        throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception);
     }
 
     @Nonnull
     private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesException {
         // If the video is age restricted getPlayerConfig will fail
-        if(isAgeRestricted) return Collections.emptyList();
+        if (isAgeRestricted) return Collections.emptyList();
 
         final JsonObject captions;
         if (!playerResponse.has("captions")) {
@@ -908,7 +929,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         // This check is necessary since there may be cases where subtitles metadata do not contain caption track info
         // e.g. https://www.youtube.com/watch?v=-Vpwatutnko
         final int captionsSize = captionsArray.size();
-        if(captionsSize == 0) return Collections.emptyList();
+        if (captionsSize == 0) return Collections.emptyList();
 
         List<SubtitlesInfo> result = new ArrayList<>();
         for (int i = 0; i < captionsSize; i++) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java

@@ -51,7 +51,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
                 throw new ParsingException("the URL given is not a Youtube-URL");
             }
 
-            if (!path.startsWith("/user/") && !path.startsWith("/channel/")) {
+            if (!path.startsWith("/user/") && !path.startsWith("/channel/") && !path.startsWith("/c/")) {
                 throw new ParsingException("the URL given is neither a channel nor an user");
             }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java

@@ -1,18 +1,14 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.util.List;
-
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
-import org.schabi.newpipe.extractor.utils.Parser;
+
+import java.util.List;
 
 public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final YoutubeCommentsLinkHandlerFactory instance = new YoutubeCommentsLinkHandlerFactory();
-    private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})";
 
     public static YoutubeCommentsLinkHandlerFactory getInstance() {
         return instance;
@@ -24,78 +20,18 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
     }
 
     @Override
-    public String getId(String url) throws ParsingException, IllegalArgumentException {
+    public String getId(String urlString) throws ParsingException, IllegalArgumentException {
-        if (url.isEmpty()) {
+        return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString); //we need the same id, avoids duplicate code
-            throw new IllegalArgumentException("The url parameter should not be empty");
-        }
-
-        String id;
-        String lowercaseUrl = url.toLowerCase();
-        if (lowercaseUrl.contains("youtube")) {
-            if (url.contains("attribution_link")) {
-                try {
-                    String escapedQuery = Parser.matchGroup1("u=(.[^&|$]*)", url);
-                    String query = URLDecoder.decode(escapedQuery, "UTF-8");
-                    id = Parser.matchGroup1("v=" + ID_PATTERN, query);
-                } catch (UnsupportedEncodingException uee) {
-                    throw new ParsingException("Could not parse attribution_link", uee);
-                }
-            } else if (url.contains("vnd.youtube")) {
-                id = Parser.matchGroup1(ID_PATTERN, url);
-            } else if (url.contains("embed")) {
-                id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
-            } else if (url.contains("googleads")) {
-                throw new FoundAdException("Error found add: " + url);
-            } else {
-                id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
-            }
-        } else if (lowercaseUrl.contains("youtu.be")) {
-            if (url.contains("v=")) {
-                id = Parser.matchGroup1("v=" + ID_PATTERN, url);
-            } else {
-                id = Parser.matchGroup1("[Yy][Oo][Uu][Tt][Uu]\\.[Bb][Ee]/" + ID_PATTERN, url);
-            }
-        } else if(lowercaseUrl.contains("hooktube")) {
-            if(lowercaseUrl.contains("&v=")
-                    || lowercaseUrl.contains("?v=")) {
-                id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
-            } else if (url.contains("/embed/")) {
-                id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
-            } else if (url.contains("/v/")) {
-                id = Parser.matchGroup1("v/" + ID_PATTERN, url);
-            } else if (url.contains("/watch/")) {
-                id = Parser.matchGroup1("watch/" + ID_PATTERN, url);
-            } else {
-                throw new ParsingException("Error no suitable url: " + url);
-            }
-        } else {
-            throw new ParsingException("Error no suitable url: " + url);
-        }
-
-
-        if (!id.isEmpty()) {
-            return id;
-        } else {
-            throw new ParsingException("Error could not parse url: " + url);
-        }
     }
 
     @Override
     public boolean onAcceptUrl(final String url) throws FoundAdException {
-        final String lowercaseUrl = url.toLowerCase();
+        try {
-        if (lowercaseUrl.contains("youtube")
+            getId(url);
-                || lowercaseUrl.contains("youtu.be")
+            return true;
-                || lowercaseUrl.contains("hooktube")) {
+        } catch (FoundAdException fe) {
-            // bad programming I know
+            throw fe;
-            try {
+        } catch (ParsingException e) {
-                getId(url);
-                return true;
-            } catch (FoundAdException fe) {
-                throw fe;
-            } catch (ParsingException e) {
-                return false;
-            }
-        } else {
             return false;
         }
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java

@@ -38,6 +38,9 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
+    private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
+    private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
+
     private static final String[] RECAPTCHA_DETECTION_SELECTORS = {
             "form[action*=\"/das_captcha\"]",
             "input[name*=\"action_recaptcha_verify\"]"
@@ -118,6 +121,16 @@ public class YoutubeParsingHelper {
                 + Long.parseLong(seconds);
     }
 
+    public static String getFeedUrlFrom(final String channelIdOrUser) {
+        if (channelIdOrUser.startsWith("user/")) {
+            return FEED_BASE_USER + channelIdOrUser.replace("user/", "");
+        } else if (channelIdOrUser.startsWith("channel/")) {
+            return FEED_BASE_CHANNEL_ID + channelIdOrUser.replace("channel/", "");
+        } else {
+            return FEED_BASE_CHANNEL_ID + channelIdOrUser;
+        }
+    }
+
     public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
         Date date;
         try {

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java

@@ -67,6 +67,7 @@ public class Parser {
         if (foundMatch) {
             return mat.group(group);
         } else {
+            // only pass input to exception message when it is not too long
             if (input.length() > 1024) {
                 throw new RegexException("failed to find pattern \"" + pat.pattern());
             } else {

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java

@@ -89,7 +89,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
 
     @Test
     public void testGetTextualUploadDate() throws ParsingException {
-        Assert.assertEquals("2018-05-11", extractor.getTextualUploadDate());
+        Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java

@@ -24,7 +24,7 @@ public class PeertubeChannelLinkHandlerFactoryTest {
     }
 
     @Test
-    public void acceptrUrlTest() throws ParsingException {
+    public void acceptUrlTest() throws ParsingException {
         assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net"));
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java

@@ -24,7 +24,7 @@ public class PeertubeCommentsLinkHandlerFactoryTest {
     }
 
     @Test
-    public void acceptrUrlTest() throws ParsingException {
+    public void acceptUrlTest() throws ParsingException {
         assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt"));
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java

@@ -24,7 +24,7 @@ public class PeertubePlaylistLinkHandlerFactoryTest {
     }
 
     @Test
-    public void acceptrUrlTest() throws ParsingException {
+    public void acceptUrlTest() throws ParsingException {
         assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos"));
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java

@@ -24,10 +24,12 @@ public class YoutubeChannelLinkHandlerFactoryTest {
     }
 
     @Test
-    public void acceptrUrlTest() throws ParsingException {
+    public void acceptUrlTest() throws ParsingException {
         assertTrue(linkHandler.acceptUrl("https://www.youtube.com/user/Gronkh"));
         assertTrue(linkHandler.acceptUrl("https://www.youtube.com/user/Netzkino/videos"));
 
+        assertTrue(linkHandler.acceptUrl("https://www.youtube.com/c/creatoracademy"));
+
         assertTrue(linkHandler.acceptUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA"));
         assertTrue(linkHandler.acceptUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
 
@@ -64,5 +66,8 @@ public class YoutubeChannelLinkHandlerFactoryTest {
     
         assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId());
         assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
+
+        assertEquals("c/creatoracademy", linkHandler.fromUrl("https://www.youtube.com/c/creatoracademy").getId());
+        assertEquals("c/YouTubeCreators", linkHandler.fromUrl("https://www.youtube.com/c/YouTubeCreators").getId());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -4,7 +4,6 @@ import org.jsoup.helper.StringUtil;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.comments.CommentsInfo;
@@ -21,17 +20,32 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class YoutubeCommentsExtractorTest {
 
-    private static YoutubeCommentsExtractor extractor;
+    private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o";
+    private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o";
+    private static final String urlInvidioush = "https://invidiou.sh/watch?v=D00Au7k3i6o";
+    private static YoutubeCommentsExtractor extractorYT;
+    private static YoutubeCommentsExtractor extractorInvidious;
+    private static YoutubeCommentsExtractor extractorInvidioush;
 
     @BeforeClass
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
-        extractor = (YoutubeCommentsExtractor) YouTube
+        extractorYT = (YoutubeCommentsExtractor) YouTube
-                .getCommentsExtractor("https://www.youtube.com/watch?v=D00Au7k3i6o");
+                .getCommentsExtractor(urlYT);
+        extractorInvidious = (YoutubeCommentsExtractor) YouTube
+                .getCommentsExtractor(urlInvidious);
+        extractorInvidioush = (YoutubeCommentsExtractor) YouTube
+                .getCommentsExtractor(urlInvidioush);
     }
 
     @Test
     public void testGetComments() throws IOException, ExtractionException {
+        assertTrue(getCommentsHelper(extractorYT));
+        assertTrue(getCommentsHelper(extractorInvidious));
+        assertTrue(getCommentsHelper(extractorInvidioush));
+    }
+
+    private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
         boolean result;
         InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
         result = findInComments(comments, "s1ck m3m3");
@@ -41,14 +55,20 @@ public class YoutubeCommentsExtractorTest {
             result = findInComments(comments, "s1ck m3m3");
         }
 
-        assertTrue(result);
+        return result;
     }
 
     @Test
     public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
+        assertTrue(getCommentsFromCommentsInfoHelper(urlYT));
+        assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious));
+        assertTrue(getCommentsFromCommentsInfoHelper(urlInvidioush));
+    }
+
+    private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
         boolean result = false;
-        CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=D00Au7k3i6o");
+        CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
-        assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName()));
+        assertEquals("what the fuck am i doing with my life", commentsInfo.getName());
         result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
 
         String nextPage = commentsInfo.getNextPageUrl();
@@ -57,16 +77,15 @@ public class YoutubeCommentsExtractorTest {
             result = findInComments(moreItems.getItems(), "s1ck m3m3");
             nextPage = moreItems.getNextPageUrl();
         }
-
+        return result;
-        assertTrue(result);
     }
-    
+
     @Test
     public void testGetCommentsAllData() throws IOException, ExtractionException {
-        InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
+        InfoItemsPage<CommentsInfoItem> comments = extractorYT.getInitialPage();
 
         DefaultTests.defaultTestListOfItems(YouTube.getServiceId(), comments.getItems(), comments.getErrors());
-        for(CommentsInfoItem c: comments.getItems()) {
+        for (CommentsInfoItem c : comments.getItems()) {
             assertFalse(StringUtil.isBlank(c.getAuthorEndpoint()));
             assertFalse(StringUtil.isBlank(c.getAuthorName()));
             assertFalse(StringUtil.isBlank(c.getAuthorThumbnail()));
@@ -86,8 +105,8 @@ public class YoutubeCommentsExtractorTest {
     }
 
     private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
-        for(CommentsInfoItem c: comments) {
+        for (CommentsInfoItem c : comments) {
-            if(c.getCommentText().contains(comment)) {
+            if (c.getCommentText().contains(comment)) {
                 return true;
             }
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java

@@ -0,0 +1,72 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
+
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
+
+public class YoutubeFeedExtractorTest {
+    public static class Kurzgesagt implements BaseListExtractorTest {
+        private static YoutubeFeedExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (YoutubeFeedExtractor) YouTube
+                    .getFeedExtractor("https://www.youtube.com/user/Kurzgesagt");
+            extractor.fetchPage();
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testServiceId() {
+            assertEquals(YouTube.getServiceId(), extractor.getServiceId());
+        }
+
+        @Test
+        public void testName() {
+            String name = extractor.getName();
+            assertTrue(name, name.startsWith("Kurzgesagt"));
+        }
+
+        @Test
+        public void testId() {
+            assertEquals("UCsXVk37bltHxD1rDPwtNM8Q", extractor.getId());
+        }
+
+        @Test
+        public void testUrl() {
+            assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getUrl());
+        }
+
+        @Test
+        public void testOriginalUrl() throws ParsingException {
+            assertEquals("https://www.youtube.com/user/Kurzgesagt", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ListExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+        }
+
+        @Test
+        public void testMoreRelatedItems() {
+            assertFalse(extractor.hasNextPage());
+            assertNull(extractor.getNextPageUrl());
+        }
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java

@@ -77,8 +77,8 @@ public class YoutubeStreamLinkHandlerFactoryTest {
         assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId());
         assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId());
         assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
-        assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
+        assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI", "youtube.com").getId());
-        assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
+        assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI", "youtube.com").getId());
         assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java

@@ -1,36 +0,0 @@
-package org.schabi.newpipe.extractor.services.youtube.search;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
-
-import static java.util.Collections.singletonList;
-import static junit.framework.TestCase.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-
-/**
- * Test for {@link YoutubeSearchExtractor}
- */
-public class YoutubeSearchCountTest {
-    public static class YoutubeChannelViewCountTest extends YoutubeSearchExtractorBaseTest {
-        @BeforeClass
-        public static void setUpClass() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie",
-                    singletonList(YoutubeSearchQueryHandlerFactory.CHANNELS), null);
-            extractor.fetchPage();
-            itemsPage = extractor.getInitialPage();
-        }
-
-        @Test
-        public void testViewCount() {
-            ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0);
-            assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()),
-                    69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 103043316);
-        }
-    }
-}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java

@@ -39,7 +39,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
         boolean equals = true;
         for (int i = 0; i < secondPage.getItems().size()
                 && i < itemsPage.getItems().size(); i++) {
-            if(!secondPage.getItems().get(i).getUrl().equals(
+            if (!secondPage.getItems().get(i).getUrl().equals(
                     itemsPage.getItems().get(i).getUrl())) {
                 equals = false;
             }
@@ -58,7 +58,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
     @Test
     public void testOnlyContainChannels() {
         for(InfoItem item : itemsPage.getItems()) {
-            if(!(item instanceof ChannelInfoItem)) {
+            if (!(item instanceof ChannelInfoItem)) {
                 fail("The following item is no channel item: " + item.toString());
             }
         }
@@ -78,4 +78,11 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
             }
         }
     }
+
+    @Test
+    public void testStreamCount() {
+        ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0);
+        assertTrue("Stream count does not fit: " + ci.getStreamCount(),
+                4000 < ci.getStreamCount() && ci.getStreamCount() < 5500);
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -75,7 +75,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
 
     @Test
     public void testGetLength() throws ParsingException {
-        assertEquals(1789, extractor.getLength());
+        assertEquals(1790, extractor.getLength());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest {
     @Test
     public void testGetDescription() throws ParsingException {
         assertNotNull(extractor.getDescription());
-//        assertFalse(extractor.getDescription().isEmpty());
+        assertFalse(extractor.getDescription().isEmpty());
     }
 
     @Test
@@ -122,13 +122,13 @@ public class YoutubeStreamExtractorControversialTest {
 
     @Test
     public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
-        // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
+        // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions
         assertFalse(extractor.getSubtitlesDefault().isEmpty());
     }
 
     @Test
     public void testGetSubtitlesList() throws IOException, ExtractionException {
-        // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
+        // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions
         assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -101,7 +101,7 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @Test
         public void testGetLength() throws ParsingException {
-            assertEquals(366, extractor.getLength());
+            assertEquals(367, extractor.getLength());
         }
 
         @Test
@@ -124,7 +124,11 @@ public class YoutubeStreamExtractorDefaultTest {
 
         @Test
         public void testGetUploaderUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw", extractor.getUploaderUrl());
+            String url = extractor.getUploaderUrl();
+            if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") &&
+                !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) {
+                fail("Uploader url is neither the music channel one nor the Vevo one");
+            }
         }
 
         @Test
@@ -183,6 +187,18 @@ public class YoutubeStreamExtractorDefaultTest {
             // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
             assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
         }
+
+        @Test
+        public void testGetLikeCount() throws ParsingException {
+            long likeCount = extractor.getLikeCount();
+            assertTrue("" + likeCount, likeCount >= 15000000);
+        }
+
+        @Test
+        public void testGetDislikeCount() throws ParsingException {
+            long dislikeCount = extractor.getDislikeCount();
+            assertTrue("" + dislikeCount, dislikeCount >= 818000);
+        }
     }
 
     public static class DescriptionTestPewdiepie {
@@ -245,6 +261,29 @@ public class YoutubeStreamExtractorDefaultTest {
         }
     }
 
+    public static class RatingsDisabledTest {
+        private static YoutubeStreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (YoutubeStreamExtractor) YouTube
+                    .getStreamExtractor("https://www.youtube.com/watch?v=HRKu0cvrr_o");
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetLikeCount() throws ParsingException {
+            assertEquals(-1, extractor.getLikeCount());
+        }
+
+        @Test
+        public void testGetDislikeCount() throws ParsingException {
+            assertEquals(-1, extractor.getDislikeCount());
+        }
+
+    }
+
     public static class FramesTest {
         private static YoutubeStreamExtractor extractor;