Merge TNP/dev into fynngodau/dev

README.md

@@ -11,7 +11,9 @@ NewPipe Extractor is available at JitPack's Maven repo.
 If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
 
 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
-2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.19.7'`the `dependencies` in your `build.gradle`. Replace `v0.19.7` with the latest release.
+2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.3'`the `dependencies` in your `build.gradle`. Replace `v0.20.3` with the latest release.
+
+**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
 
 ### Testing changes
 

build.gradle

@@ -2,10 +2,10 @@ allprojects {
     apply plugin: 'java-library'
     apply plugin: 'maven'
 
-    sourceCompatibility = 1.7
+    sourceCompatibility = 1.8
-    targetCompatibility = 1.7
+    targetCompatibility = 1.8
 
-    version 'v0.19.7'
+    version 'v0.20.3'
     group 'com.github.TeamNewPipe'
 
     repositories {

extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nullable;
 import java.util.Collections;
@@ -277,18 +278,19 @@ public abstract class StreamingService {
      * Figures out where the link is pointing to (a channel, a video, a playlist, etc.)
      * @param url the url on which it should be decided of which link type it is
      * @return the link type of url
-     * @throws ParsingException
      */
-    public final LinkType getLinkTypeByUrl(String url) throws ParsingException {
+    public final LinkType getLinkTypeByUrl(final String url) throws ParsingException {
-        LinkHandlerFactory sH = getStreamLHFactory();
+        final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
-        LinkHandlerFactory cH = getChannelLHFactory();
-        LinkHandlerFactory pH = getPlaylistLHFactory();
 
-        if (sH != null && sH.acceptUrl(url)) {
+        final LinkHandlerFactory sH = getStreamLHFactory();
+        final LinkHandlerFactory cH = getChannelLHFactory();
+        final LinkHandlerFactory pH = getPlaylistLHFactory();
+
+        if (sH != null && sH.acceptUrl(polishedUrl)) {
             return LinkType.STREAM;
-        } else if (cH != null && cH.acceptUrl(url)) {
+        } else if (cH != null && cH.acceptUrl(polishedUrl)) {
             return LinkType.CHANNEL;
-        } else if (pH != null && pH.acceptUrl(url)) {
+        } else if (pH != null && pH.acceptUrl(polishedUrl)) {
             return LinkType.PLAYLIST;
         } else {
             return LinkType.NONE;

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java

@@ -42,12 +42,29 @@ public abstract class LinkHandlerFactory {
     // Logic
     ///////////////////////////////////
 
-    public LinkHandler fromUrl(String url) throws ParsingException {
+    /**
-        if (url == null) throw new IllegalArgumentException("url can not be null");
+     * Builds a {@link LinkHandler} from a url.<br>
-        final String baseUrl = Utils.getBaseUrl(url);
+     * Be sure to call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding
-        return fromUrl(url, baseUrl);
+     * this function.
+     * @param url the url to extract path and id from
+     * @return a {@link LinkHandler} complete with information
+     */
+    public LinkHandler fromUrl(final String url) throws ParsingException {
+        final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
+        final String baseUrl = Utils.getBaseUrl(polishedUrl);
+        return fromUrl(polishedUrl, baseUrl);
     }
 
+    /**
+     * Builds a {@link LinkHandler} from a url and a base url. The url is expected to be already
+     * polished from google search redirects (otherwise how could {@code baseUrl} have been
+     * extracted?).<br>
+     * So do not call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding
+     * this function, since that should be done in {@link #fromUrl(String)}.
+     * @param url the url without google search redirects to extract id from
+     * @param baseUrl the base url
+     * @return a {@link LinkHandler} complete with information
+     */
     public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
         if (url == null) throw new IllegalArgumentException("url can not be null");
         if (!acceptUrl(url)) {

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java

@@ -31,9 +31,10 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
     ///////////////////////////////////
 
     @Override
-    public ListLinkHandler fromUrl(String url) throws ParsingException {
+    public ListLinkHandler fromUrl(final String url) throws ParsingException {
-        String baseUrl = Utils.getBaseUrl(url);
+        final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
-        return fromUrl(url, baseUrl);
+        final String baseUrl = Utils.getBaseUrl(polishedUrl);
+        return fromUrl(polishedUrl, baseUrl);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java

@@ -3,30 +3,60 @@ package org.schabi.newpipe.extractor.localization;
 import edu.umd.cs.findbugs.annotations.NonNull;
 
 import java.io.Serializable;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
 import java.util.Calendar;
+import java.util.GregorianCalendar;
 
 /**
- * A wrapper class that provides a field to describe if the date is precise or just an approximation.
+ * A wrapper class that provides a field to describe if the date/time is precise or just an approximation.
  */
 public class DateWrapper implements Serializable {
-    @NonNull private final Calendar date;
+    @NonNull private final OffsetDateTime offsetDateTime;
     private final boolean isApproximation;
 
-    public DateWrapper(@NonNull Calendar date) {
+    /**
-        this(date, false);
+     * @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead.
+     */
+    @Deprecated
+    public DateWrapper(@NonNull Calendar calendar) {
+        this(calendar, false);
     }
 
-    public DateWrapper(@NonNull Calendar date, boolean isApproximation) {
+    /**
-        this.date = date;
+     * @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead.
+     */
+    @Deprecated
+    public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) {
+        this(OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC), isApproximation);
+    }
+
+    public DateWrapper(@NonNull OffsetDateTime offsetDateTime) {
+        this(offsetDateTime, false);
+    }
+
+    public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) {
+        this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
         this.isApproximation = isApproximation;
     }
 
     /**
-     * @return the wrapped date.
+     * @return the wrapped date/time as a {@link Calendar}.
+     *
+     * @deprecated use {@link #offsetDateTime()} instead.
      */
+    @Deprecated
     @NonNull
     public Calendar date() {
-        return date;
+        return GregorianCalendar.from(offsetDateTime.toZonedDateTime());
+    }
+
+    /**
+     * @return the wrapped date/time.
+     */
+    @NonNull
+    public OffsetDateTime offsetDateTime() {
+        return offsetDateTime;
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java

@@ -2,10 +2,11 @@ package org.schabi.newpipe.extractor.localization;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.timeago.PatternsHolder;
-import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
 import org.schabi.newpipe.extractor.utils.Parser;
 
-import java.util.Calendar;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoUnit;
 import java.util.Collection;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -16,7 +17,7 @@ import java.util.regex.Pattern;
  */
 public class TimeAgoParser {
     private final PatternsHolder patternsHolder;
-    private final Calendar consistentNow;
+    private final OffsetDateTime now;
 
     /**
      * Creates a helper to parse upload dates in the format '2 days ago'.
@@ -28,7 +29,7 @@ public class TimeAgoParser {
      */
     public TimeAgoParser(PatternsHolder patternsHolder) {
         this.patternsHolder = patternsHolder;
-        consistentNow = Calendar.getInstance();
+        now = OffsetDateTime.now(ZoneOffset.UTC);
     }
 
     /**
@@ -42,14 +43,14 @@ public class TimeAgoParser {
      * @throws ParsingException if the time unit could not be recognized
      */
     public DateWrapper parse(String textualDate) throws ParsingException {
-        for (Map.Entry<TimeAgoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
+        for (Map.Entry<ChronoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
-            final TimeAgoUnit timeAgoUnit = caseUnitEntry.getKey();
+            final ChronoUnit chronoUnit = caseUnitEntry.getKey();
             for (Map.Entry<String, Integer> caseMapToAmountEntry : caseUnitEntry.getValue().entrySet()) {
                 final String caseText = caseMapToAmountEntry.getKey();
                 final Integer caseAmount = caseMapToAmountEntry.getValue();
 
                 if (textualDateMatches(textualDate, caseText)) {
-                    return getResultFor(caseAmount, timeAgoUnit);
+                    return getResultFor(caseAmount, chronoUnit);
                 }
             }
         }
@@ -63,8 +64,8 @@ public class TimeAgoParser {
             timeAgoAmount = 1;
         }
 
-        final TimeAgoUnit timeAgoUnit = parseTimeAgoUnit(textualDate);
+        final ChronoUnit chronoUnit = parseChronoUnit(textualDate);
-        return getResultFor(timeAgoAmount, timeAgoUnit);
+        return getResultFor(timeAgoAmount, chronoUnit);
     }
 
     private int parseTimeAgoAmount(String textualDate) throws NumberFormatException {
@@ -72,13 +73,13 @@ public class TimeAgoParser {
         return Integer.parseInt(timeValueStr);
     }
 
-    private TimeAgoUnit parseTimeAgoUnit(String textualDate) throws ParsingException {
+    private ChronoUnit parseChronoUnit(String textualDate) throws ParsingException {
-        for (Map.Entry<TimeAgoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
+        for (Map.Entry<ChronoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
-            final TimeAgoUnit timeAgoUnit = entry.getKey();
+            final ChronoUnit chronoUnit = entry.getKey();
 
             for (String agoPhrase : entry.getValue()) {
                 if (textualDateMatches(textualDate, agoPhrase)) {
-                    return timeAgoUnit;
+                    return chronoUnit;
                 }
             }
         }
@@ -112,65 +113,35 @@ public class TimeAgoParser {
         }
     }
 
-    private DateWrapper getResultFor(int timeAgoAmount, TimeAgoUnit timeAgoUnit) {
+    private DateWrapper getResultFor(int timeAgoAmount, ChronoUnit chronoUnit) {
-        final Calendar calendarTime = getNow();
+        OffsetDateTime offsetDateTime = now;
         boolean isApproximation = false;
 
-        switch (timeAgoUnit) {
+        switch (chronoUnit) {
             case SECONDS:
-                calendarTime.add(Calendar.SECOND, -timeAgoAmount);
-                break;
-
             case MINUTES:
-                calendarTime.add(Calendar.MINUTE, -timeAgoAmount);
-                break;
-
             case HOURS:
-                calendarTime.add(Calendar.HOUR_OF_DAY, -timeAgoAmount);
+                offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
                 break;
 
             case DAYS:
-                calendarTime.add(Calendar.DAY_OF_MONTH, -timeAgoAmount);
-                isApproximation = true;
-                break;
-
             case WEEKS:
-                calendarTime.add(Calendar.WEEK_OF_YEAR, -timeAgoAmount);
-                isApproximation = true;
-                break;
-
             case MONTHS:
-                calendarTime.add(Calendar.MONTH, -timeAgoAmount);
+                offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
                 isApproximation = true;
                 break;
 
             case YEARS:
-                calendarTime.add(Calendar.YEAR, -timeAgoAmount);
+                // minusDays is needed to prevent `PrettyTime` from showing '12 months ago'.
-                // Prevent `PrettyTime` from showing '12 months ago'.
+                offsetDateTime = offsetDateTime.minusYears(timeAgoAmount).minusDays(1);
-                calendarTime.add(Calendar.DAY_OF_MONTH, -1);
                 isApproximation = true;
                 break;
         }
 
         if (isApproximation) {
-            markApproximatedTime(calendarTime);
+            offsetDateTime = offsetDateTime.truncatedTo(ChronoUnit.HOURS);
         }
 
-        return new DateWrapper(calendarTime, isApproximation);
+        return new DateWrapper(offsetDateTime, isApproximation);
-    }
-
-    private Calendar getNow() {
-        return (Calendar) consistentNow.clone();
-    }
-
-    /**
-     * Marks the time as approximated by setting minutes, seconds and milliseconds to 0.
-     *
-     * @param calendarTime Time to be marked as approximated
-     */
-    private void markApproximatedTime(Calendar calendarTime) {
-        calendarTime.set(Calendar.MINUTE, 0);
-        calendarTime.set(Calendar.SECOND, 0);
-        calendarTime.set(Calendar.MILLISECOND, 0);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java

@@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
@@ -71,8 +72,8 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() {
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        JsonArray events = conferenceData.getArray("events");
+        final JsonArray events = conferenceData.getArray("events");
         for (int i = 0; i < events.size(); i++) {
             collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
         }
@@ -87,10 +88,11 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
+        final String conferenceUrl = MediaCCCConferenceLinkHandlerFactory.CONFERENCE_API_ENDPOINT + getId();
         try {
-            conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody());
+            conferenceData = JsonParser.object().from(downloader.get(conferenceUrl).responseBody());
         } catch (JsonParserException jpe) {
-            throw new ExtractionException("Could not parse json returnd by url: " + getUrl());
+            throw new ExtractionException("Could not parse json returnd by url: " + conferenceUrl);
         }
     }
 
@@ -99,10 +101,4 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
     public String getName() throws ParsingException {
         return conferenceData.getString("title");
     }
-
-    @Nonnull
-    @Override
-    public String getOriginalUrl() {
-        return "https://media.ccc.de/c/" + conferenceData.getString("acronym");
-    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java

@@ -2,25 +2,17 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
-import java.text.ParseException;
+import java.time.OffsetDateTime;
-import java.text.SimpleDateFormat;
+import java.time.format.DateTimeParseException;
-import java.util.Calendar;
-import java.util.Date;
 
 public final class MediaCCCParsingHelper {
     private MediaCCCParsingHelper() { }
 
-    public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
+    public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
-        Date date;
         try {
-            date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
+            return OffsetDateTime.parse(textualUploadDate);
-        } catch (ParseException e) {
+        } catch (DateTimeParseException e) {
             throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
         }
-
-        final Calendar uploadDate = Calendar.getInstance();
-        uploadDate.setTime(date);
-        return uploadDate;
     }
-
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -19,14 +19,18 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 public class MediaCCCStreamExtractor extends StreamExtractor {
     private JsonObject data;
@@ -93,7 +97,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getUploaderUrl() {
-        return data.getString("conference_url");
+        return MediaCCCConferenceLinkHandlerFactory.CONFERENCE_PATH + getUploaderName();
     }
 
     @Nonnull
@@ -111,25 +115,25 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getSubChannelUrl() throws ParsingException {
+    public String getSubChannelUrl() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getSubChannelName() throws ParsingException {
+    public String getSubChannelName() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getSubChannelAvatarUrl() throws ParsingException {
+    public String getSubChannelAvatarUrl() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getDashMpdUrl() throws ParsingException {
+    public String getDashMpdUrl() {
         return "";
     }
 
@@ -194,7 +198,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
 
     @Override
     public List<VideoStream> getVideoOnlyStreams() {
-        return null;
+        return Collections.emptyList();
     }
 
     @Nonnull
@@ -214,9 +218,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
         return StreamType.VIDEO_STREAM;
     }
 
+    @Nullable
     @Override
     public StreamInfoItemsCollector getRelatedStreams() {
-        return new StreamInfoItemsCollector(getServiceId());
+        return null;
     }
 
     @Override
@@ -227,14 +232,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
+        final String videoUrl = MediaCCCStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId();
         try {
-            data = JsonParser.object().from(
+            data = JsonParser.object().from(downloader.get(videoUrl).responseBody());
-                    downloader.get(getLinkHandler().getUrl()).responseBody());
             conferenceData = JsonParser.object()
-                    .from(downloader.get(getUploaderUrl()).responseBody());
+                    .from(downloader.get(data.getString("conference_url")).responseBody());
         } catch (JsonParserException jpe) {
-            throw new ExtractionException("Could not parse json returned by url: "
+            throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe);
-                    + getLinkHandler().getUrl(), jpe);
         }
     }
 
@@ -250,21 +254,25 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
         return data.getString("frontend_link");
     }
 
+    @Nonnull
     @Override
     public String getHost() {
         return "";
     }
 
+    @Nonnull
     @Override
     public String getPrivacy() {
         return "";
     }
 
+    @Nonnull
     @Override
     public String getCategory() {
         return "";
     }
 
+    @Nonnull
     @Override
     public String getLicence() {
         return "";
@@ -278,7 +286,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<String> getTags() {
-        return new ArrayList<>();
+        return Arrays.asList(data.getArray("tags").toArray(new String[0]));
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java

@@ -7,30 +7,26 @@ import org.schabi.newpipe.extractor.utils.Parser;
 import java.util.List;
 
 public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
+    public static final String CONFERENCE_API_ENDPOINT = "https://api.media.ccc.de/public/conferences/";
+    public static final String CONFERENCE_PATH = "https://media.ccc.de/c/";
+    private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/conferences/)|(?:media\\.ccc\\.de/[bc]/))([^/?&#]*)";
+
     @Override
-    public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
+    public String getUrl(final String id,
-            throws ParsingException {
+                         final List<String> contentFilter,
-        return "https://media.ccc.de/public/conferences/" + id;
+                         final String sortFilter) throws ParsingException {
+        return CONFERENCE_PATH + id;
     }
 
     @Override
     public String getId(final String url) throws ParsingException {
-        if (url.startsWith("https://media.ccc.de/public/conferences/")
+        return Parser.matchGroup1(ID_PATTERN, url);
-                || url.startsWith("https://api.media.ccc.de/public/conferences/")) {
-            return url.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
-        } else if (url.startsWith("https://media.ccc.de/c/")) {
-            return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url);
-        } else if (url.startsWith("https://media.ccc.de/b/")) {
-            return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url);
-        }
-        throw new ParsingException("Could not get id from url: " + url);
     }
 
     @Override
     public boolean onAcceptUrl(final String url) {
         try {
-            getId(url);
+            return getId(url) != null;
-            return true;
         } catch (ParsingException e) {
             return false;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java

@@ -2,54 +2,27 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
-import org.schabi.newpipe.extractor.utils.Utils;
+import org.schabi.newpipe.extractor.utils.Parser;
-
-import java.net.MalformedURLException;
-import java.net.URL;
 
 public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
+    public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
+    private static final String VIDEO_PATH = "https://media.ccc.de/v/";
+    private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
+
     @Override
-    public String getId(final String urlString) throws ParsingException {
+    public String getId(final String url) throws ParsingException {
-        if (urlString.startsWith("https://media.ccc.de/public/events/")
+        return Parser.matchGroup1(ID_PATTERN, url);
-                && !urlString.contains("?q=")) {
-            return urlString.substring(35); //remove …/public/events part
-        }
-
-        if (urlString.startsWith("https://api.media.ccc.de/public/events/")
-                && !urlString.contains("?q=")) {
-            return urlString.substring(39); //remove api…/public/events part
-        }
-
-        URL url;
-        try {
-            url = Utils.stringToURL(urlString);
-        } catch (MalformedURLException e) {
-            throw new IllegalArgumentException("The given URL is not valid");
-        }
-
-        String path = url.getPath();
-        // remove leading "/" of URL-path if URL-path is given
-        if (!path.isEmpty()) {
-            path = path.substring(1);
-        }
-
-        if (path.startsWith("v/")) {
-            return path.substring(2);
-        }
-
-        throw new ParsingException("Could not get id from url: " + url);
     }
 
     @Override
     public String getUrl(final String id) throws ParsingException {
-        return "https://media.ccc.de/public/events/" + id;
+        return VIDEO_PATH + id;
     }
 
     @Override
     public boolean onAcceptUrl(final String url) {
         try {
-            getId(url);
+            return getId(url) != null;
-            return true;
         } catch (ParsingException e) {
             return false;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java

@@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.peertube;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-
 import org.schabi.newpipe.extractor.InfoItemsCollector;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -12,11 +11,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.text.ParseException;
+import java.time.Instant;
-import java.text.SimpleDateFormat;
+import java.time.OffsetDateTime;
-import java.util.Calendar;
+import java.time.ZoneOffset;
-import java.util.Date;
+import java.time.format.DateTimeParseException;
-import java.util.TimeZone;
 
 public class PeertubeParsingHelper {
     public static final String START_KEY = "start";
@@ -34,19 +32,12 @@ public class PeertubeParsingHelper {
         }
     }
 
-    public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
+    public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
-        final Date date;
         try {
-            final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
+            return OffsetDateTime.ofInstant(Instant.parse(textualUploadDate), ZoneOffset.UTC);
-            sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+        } catch (DateTimeParseException e) {
-            date = sdf.parse(textualUploadDate);
-        } catch (ParseException e) {
             throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
         }
-
-        final Calendar uploadDate = Calendar.getInstance();
-        uploadDate.setTime(date);
-        return uploadDate;
     }
 
     public static Page getNextPage(final String prevPageUrl, final long total) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java

@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@@ -13,17 +12,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
+import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 
-import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
+import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
-import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
-import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class PeertubeAccountExtractor extends ChannelExtractor {
@@ -85,14 +83,16 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
         return "";
     }
 
+    @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
-        final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
+        return getPage(new Page(
-        return getPage(new Page(pageUrl));
+                baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE));
     }
 
     @Override
-    public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
+    public InfoItemsPage<StreamInfoItem> getPage(final Page page)
+            throws IOException, ExtractionException {
         if (page == null || isNullOrEmpty(page.getUrl())) {
             throw new IllegalArgumentException("Page doesn't contain an URL");
         }
@@ -122,8 +122,16 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
     }
 
     @Override
-    public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull final Downloader downloader)
-        final Response response = downloader.get(getUrl());
+            throws IOException, ExtractionException {
+        String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT;
+        if (getId().contains("accounts/")) {
+            accountUrl += getId();
+        } else {
+            accountUrl += "accounts/" + getId();
+        }
+
+        final Response response = downloader.get(accountUrl);
         if (response != null && response.responseBody() != null) {
             setInitialData(response.responseBody());
         } else {
@@ -140,13 +148,9 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
         if (json == null) throw new ExtractionException("Unable to extract PeerTube account data");
     }
 
+    @Nonnull
     @Override
     public String getName() throws ParsingException {
         return JsonUtils.getString(json, "displayName");
     }
-
-    @Override
-    public String getOriginalUrl() throws ParsingException {
-        return baseUrl + "/" + getId();
-    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java

@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@@ -13,17 +12,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
+import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 
-import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
+import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
-import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
-import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 
@@ -92,10 +90,11 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
         return baseUrl + value;
     }
 
+    @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
-        final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
+        return getPage(new Page(
-        return getPage(new Page(pageUrl));
+                baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE));
     }
 
     @Override
@@ -130,7 +129,8 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
 
     @Override
     public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
-        final Response response = downloader.get(getUrl());
+        final Response response = downloader.get(
+                baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId());
         if (response != null && response.responseBody() != null) {
             setInitialData(response.responseBody());
         } else {
@@ -147,13 +147,9 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
         if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
     }
 
+    @Nonnull
     @Override
     public String getName() throws ParsingException {
         return JsonUtils.getString(json, "displayName");
     }
-
-    @Override
-    public String getOriginalUrl() throws ParsingException {
-        return baseUrl + "/" + getId();
-    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
+import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.AudioStream;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.Stream;
@@ -37,11 +38,12 @@ import java.util.List;
 import java.util.Locale;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 public class PeertubeStreamExtractor extends StreamExtractor {
     private final String baseUrl;
     private JsonObject json;
-    private List<SubtitlesStream> subtitles = new ArrayList<>();
+    private final List<SubtitlesStream> subtitles = new ArrayList<>();
 
     public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) throws ParsingException {
         super(service, linkHandler);
@@ -64,11 +66,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
     }
 
+    @Nonnull
     @Override
     public String getThumbnailUrl() throws ParsingException {
         return baseUrl + JsonUtils.getString(json, "previewPath");
     }
 
+    @Nonnull
     @Override
     public Description getDescription() throws ParsingException {
         String text;
@@ -81,7 +85,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
             //if description is shortened, get full description
             final Downloader dl = NewPipe.getDownloader();
             try {
-                final Response response = dl.get(getUrl() + "/description");
+                final Response response = dl.get(baseUrl
+                        + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT
+                        + getId() + "/description");
                 final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
                 text = JsonUtils.getString(jsonObject, "description");
             } catch (ReCaptchaException | IOException | JsonParserException e) {
@@ -107,9 +113,16 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public long getTimeStamp() {
+    public long getTimeStamp() throws ParsingException {
-        //TODO fetch timestamp from url if present;
+        final long timestamp =
-        return 0;
+                getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
+
+        if (timestamp == -2) {
+            // regex for timestamp was not found
+            return 0;
+        } else {
+            return timestamp;
+        }
     }
 
     @Override
@@ -127,6 +140,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return json.getLong("dislikes");
     }
 
+    @Nonnull
     @Override
     public String getUploaderUrl() throws ParsingException {
         final String name = JsonUtils.getString(json, "account.name");
@@ -134,11 +148,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
     }
 
+    @Nonnull
     @Override
     public String getUploaderName() throws ParsingException {
         return JsonUtils.getString(json, "account.displayName");
     }
 
+    @Nonnull
     @Override
     public String getUploaderAvatarUrl() {
         String value;
@@ -150,6 +166,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return baseUrl + value;
     }
 
+    @Nonnull
     @Override
     public String getSubChannelUrl() throws ParsingException {
         return JsonUtils.getString(json, "channel.url");
@@ -173,11 +190,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return baseUrl + value;
     }
 
+    @Nonnull
     @Override
     public String getDashMpdUrl() {
         return "";
     }
 
+    @Nonnull
     @Override
     public String getHlsUrl() {
         return "";
@@ -185,7 +204,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
 
     @Override
     public List<AudioStream> getAudioStreams() {
-        return null;
+        return Collections.emptyList();
     }
 
     @Override
@@ -220,11 +239,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return Collections.emptyList();
     }
 
+    @Nonnull
     @Override
     public List<SubtitlesStream> getSubtitlesDefault() {
         return subtitles;
     }
 
+    @Nonnull
     @Override
     public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
         final List<SubtitlesStream> filteredSubs = new ArrayList<>();
@@ -241,21 +262,27 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return StreamType.VIDEO_STREAM;
     }
 
+    @Nullable
     @Override
     public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
-        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         final List<String> tags = getTags();
         final String apiUrl;
-        if (!tags.isEmpty()) {
+        if (tags.isEmpty()) {
-            apiUrl = getRelatedStreamsUrl(tags);
-
-        } else {
             apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
+        } else {
+            apiUrl = getRelatedStreamsUrl(tags);
+        }
+
+        if (Utils.isBlank(apiUrl)) {
+            return null;
+        } else {
+            final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+            getStreamsFromApi(collector, apiUrl);
+            return collector;
         }
-        if (!Utils.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
-        return collector;
     }
 
+    @Nonnull
     @Override
     public List<String> getTags() {
         try {
@@ -327,7 +354,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
 
     @Override
     public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
-        final Response response = downloader.get(getUrl());
+        final Response response = downloader.get(baseUrl + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId());
         if (response != null && response.responseBody() != null) {
             setInitialData(response.responseBody());
         } else {
@@ -343,14 +370,18 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         } catch (JsonParserException e) {
             throw new ExtractionException("Unable to extract PeerTube stream data", e);
         }
-        if (json == null) throw new ExtractionException("Unable to extract PeerTube stream data");
+        if (json == null) {
+            throw new ExtractionException("Unable to extract PeerTube stream data");
+        }
         PeertubeParsingHelper.validate(json);
     }
 
     private void loadSubtitles() {
         if (subtitles.isEmpty()) {
             try {
-                final Response response = getDownloader().get(getUrl() + "/captions");
+                final Response response = getDownloader().get(baseUrl
+                        + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT
+                        + getId() + "/captions");
                 final JsonObject captionsJson = JsonParser.object().from(response.responseBody());
                 final JsonArray captions = JsonUtils.getArray(captionsJson, "data");
                 for (final Object c : captions) {
@@ -370,31 +401,31 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         }
     }
 
+    @Nonnull
     @Override
     public String getName() throws ParsingException {
         return JsonUtils.getString(json, "name");
     }
 
-    @Override
+    @Nonnull
-    public String getOriginalUrl() throws ParsingException {
-        return baseUrl + "/videos/watch/" + getId();
-    }
-
     @Override
     public String getHost() throws ParsingException {
         return JsonUtils.getString(json, "account.host");
     }
 
+    @Nonnull
     @Override
     public String getPrivacy() throws ParsingException {
         return JsonUtils.getString(json, "privacy.label");
     }
 
+    @Nonnull
     @Override
     public String getCategory() throws ParsingException {
         return JsonUtils.getString(json, "category.label");
     }
 
+    @Nonnull
     @Override
     public String getLicence() throws ParsingException {
         return JsonUtils.getString(json, "licence.label");

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java

@@ -27,8 +27,7 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
-        final String value = JsonUtils.getString(item, "thumbnailPath");
+        return baseUrl + JsonUtils.getString(item, "thumbnailPath");
-        return baseUrl + value;
     }
 
     @Override
@@ -51,7 +50,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
         final String name = JsonUtils.getString(item, "account.name");
         final String host = JsonUtils.getString(item, "account.host");
 
-        return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
+        return ServiceList.PeerTube.getChannelLHFactory()
+                .fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java

@@ -11,7 +11,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
     private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
-    private static final String API_ENDPOINT = "/api/v1/";
+    public static final String API_ENDPOINT = "/api/v1/";
 
     public static PeertubeChannelLinkHandlerFactory getInstance() {
         return instance;
@@ -24,19 +24,17 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
 
     @Override
     public String getUrl(String id, List<String> contentFilters, String searchFilter) throws ParsingException {
-        String baseUrl = ServiceList.PeerTube.getBaseUrl();
+        return getUrl(id, contentFilters, searchFilter, ServiceList.PeerTube.getBaseUrl());
-        return getUrl(id, contentFilters, searchFilter, baseUrl);
     }
 
     @Override
     public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
             throws ParsingException {
-
         if (id.matches(ID_PATTERN)) {
-            return baseUrl + API_ENDPOINT + id;
+            return baseUrl + "/" + id;
         } else {
             // This is needed for compatibility with older versions were we didn't support video channels yet
-            return baseUrl + API_ENDPOINT + "accounts/" + id;
+            return baseUrl + "/accounts/" + id;
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java

@@ -10,7 +10,8 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
 
     private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory();
     private static final String ID_PATTERN = "/videos/(watch/|embed/)?([^/?&#]*)";
-    private static final String VIDEO_ENDPOINT = "/api/v1/videos/";
+    public static final String VIDEO_API_ENDPOINT = "/api/v1/videos/";
+    private static final String VIDEO_PATH = "/videos/watch/";
 
     private PeertubeStreamLinkHandlerFactory() {
     }
@@ -21,13 +22,12 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
 
     @Override
     public String getUrl(String id) {
-        String baseUrl = ServiceList.PeerTube.getBaseUrl();
+        return getUrl(id, ServiceList.PeerTube.getBaseUrl());
-        return getUrl(id, baseUrl);
     }
 
     @Override
     public String getUrl(String id, String baseUrl) {
-        return baseUrl + VIDEO_ENDPOINT + id;
+        return baseUrl + VIDEO_PATH + id;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -25,10 +25,15 @@ import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.net.URLEncoder;
-import java.text.ParseException;
+import java.time.OffsetDateTime;
-import java.text.SimpleDateFormat;
+import java.time.format.DateTimeFormatter;
-import java.util.*;
+import java.time.format.DateTimeParseException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
 
 import static java.util.Collections.singletonList;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@@ -93,23 +98,16 @@ public class SoundcloudParsingHelper {
         }
     }
 
-    public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
+    public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
-        Date date;
         try {
-            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+            return OffsetDateTime.parse(textualUploadDate);
-            sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+        } catch (DateTimeParseException e1) {
-            date = sdf.parse(textualUploadDate);
-        } catch (ParseException e1) {
             try {
-                date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate);
+                return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
-            } catch (ParseException e2) {
+            } catch (DateTimeParseException e2) {
                 throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
             }
         }
-
-        final Calendar uploadDate = Calendar.getInstance();
-        uploadDate.setTime(date);
-        return uploadDate;
     }
 
     /**
@@ -148,12 +146,21 @@ public class SoundcloudParsingHelper {
      *
      * @return the resolved id
      */
-    public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
+    public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
+        // Remove the tailing slash from URLs due to issues with the SoundCloud API
+        if (urlString.charAt(urlString.length() -1) == '/') urlString = urlString.substring(0, urlString.length()-1);
+
+        URL url;
+        try {
+            url = Utils.stringToURL(urlString);
+        } catch (MalformedURLException e){
+            throw new IllegalArgumentException("The given URL is not valid");
+        }
 
         String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
-                + URLEncoder.encode(url, "UTF-8"), SoundCloud.getLocalization()).responseBody();
+                + URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody();
         // handle playlists / sets different and get playlist id via uir field in JSON
-        if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
+        if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
             return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
         return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -33,6 +33,7 @@ import java.util.List;
 import java.util.Locale;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -46,7 +47,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
+        track = SoundcloudParsingHelper.resolveFor(downloader, getUrl());
 
         String policy = track.getString("policy", EMPTY_STRING);
         if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
@@ -68,8 +69,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getTextualUploadDate() throws ParsingException {
+    public String getTextualUploadDate() {
-        return track.getString("created_at").replace("T"," ").replace("Z", "");
+        return track.getString("created_at")
+                .replace("T"," ")
+                .replace("Z", "");
     }
 
     @Nonnull
@@ -85,10 +88,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         if (artworkUrl.isEmpty()) {
             artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING);
         }
-        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrl.replace("large.jpg", "crop.jpg");
-        return artworkUrlBetterResolution;
     }
 
+    @Nonnull
     @Override
     public Description getDescription() {
         return new Description(track.getString("description"), Description.PLAIN_TEXT);
@@ -144,19 +147,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getSubChannelUrl() throws ParsingException {
+    public String getSubChannelUrl() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getSubChannelName() throws ParsingException {
+    public String getSubChannelName() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getSubChannelAvatarUrl() throws ParsingException {
+    public String getSubChannelAvatarUrl() {
         return "";
     }
 
@@ -168,14 +171,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getHlsUrl() throws ParsingException {
+    public String getHlsUrl() {
         return "";
     }
 
     @Override
     public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
         List<AudioStream> audioStreams = new ArrayList<>();
-        Downloader dl = NewPipe.getDownloader();
+        final Downloader dl = NewPipe.getDownloader();
 
         // Streams can be streamable and downloadable - or explicitly not.
         // For playing the track, it is only necessary to have a streamable track.
@@ -183,12 +186,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         if (!track.getBoolean("streamable")) return audioStreams;
 
         try {
-            JsonArray transcodings = track.getObject("media").getArray("transcodings");
+            final JsonArray transcodings = track.getObject("media").getArray("transcodings");
 
             // get information about what stream formats are available
             for (Object transcoding : transcodings) {
 
-                JsonObject t = (JsonObject) transcoding;
+                final JsonObject t = (JsonObject) transcoding;
                 String url = t.getString("url");
 
                 if (!isNullOrEmpty(url)) {
@@ -200,7 +203,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
                         // This url points to the endpoint which generates a unique and short living url to the stream.
                         // TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream)
                         url += "?client_id=" + SoundcloudParsingHelper.clientId();
-                        String res = dl.get(url).responseBody();
+                        final String res = dl.get(url).responseBody();
 
                         try {
                             JsonObject mp3UrlObject = JsonParser.object().from(res);
@@ -234,24 +237,24 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
+    public List<VideoStream> getVideoStreams() {
-        return null;
+        return Collections.emptyList();
     }
 
     @Override
-    public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
+    public List<VideoStream> getVideoOnlyStreams() {
-        return null;
-    }
-
-    @Override
-    @Nonnull
-    public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
         return Collections.emptyList();
     }
 
     @Override
     @Nonnull
-    public List<SubtitlesStream> getSubtitles(MediaFormat format) throws IOException, ExtractionException {
+    public List<SubtitlesStream> getSubtitlesDefault() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    @Nonnull
+    public List<SubtitlesStream> getSubtitles(MediaFormat format) {
         return Collections.emptyList();
     }
 
@@ -260,12 +263,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         return StreamType.AUDIO_STREAM;
     }
 
+    @Nullable
     @Override
     public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
 
-        String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related"
+        final String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId())
-                + "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
+                + "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
 
         SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
         return collector;
@@ -276,40 +280,44 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         return null;
     }
 
+    @Nonnull
     @Override
-    public String getHost() throws ParsingException {
+    public String getHost() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getPrivacy() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getCategory() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getLicence() {
         return "";
     }
 
     @Override
-    public String getPrivacy() throws ParsingException {
+    public Locale getLanguageInfo() {
-        return "";
-    }
-
-    @Override
-    public String getCategory() throws ParsingException {
-        return "";
-    }
-
-    @Override
-    public String getLicence() throws ParsingException {
-        return "";
-    }
-
-    @Override
-    public Locale getLanguageInfo() throws ParsingException {
         return null;
     }
 
     @Nonnull
     @Override
-    public List<String> getTags() throws ParsingException {
+    public List<String> getTags() {
-        return new ArrayList<>();
+        return Collections.emptyList();
     }
 
     @Nonnull
     @Override
-    public String getSupportInfo() throws ParsingException {
+    public String getSupportInfo() {
         return "";
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java

@@ -94,7 +94,7 @@ public class ItagItem {
                 return item;
             }
         }
-        throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported");
+        throw new ParsingException("itag=" + itagId + " not supported");
     }
 
     /*//////////////////////////////////////////////////////////////////////////

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import com.grack.nanojson.JsonWriter;
-
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -22,13 +21,20 @@ import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
-import java.text.ParseException;
+import java.time.LocalDate;
-import java.text.SimpleDateFormat;
+import java.time.OffsetDateTime;
-import java.util.*;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeParseException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
 import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.utils.Utils.*;
+import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
+import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /*
  * Created by Christian Schabesberger on 02.03.16.
@@ -55,12 +61,6 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
-    /**
-     * The official youtube app supports intents in this format, where after the ':' is the videoId.
-     * Accordingly there are other apps sharing streams in this format.
-     */
-    public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube";
-
     private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
     private static String clientVersion;
 
@@ -182,23 +182,27 @@ public class YoutubeParsingHelper {
         }
     }
 
-    public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
+    public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
-        Date date;
         try {
-            date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
+            return OffsetDateTime.parse(textualUploadDate);
-        } catch (ParseException e) {
+        } catch (DateTimeParseException e) {
-            throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
+            try {
+                return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC);
+            } catch (DateTimeParseException e1) {
+                throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1);
+            }
         }
-
-        final Calendar uploadDate = Calendar.getInstance();
-        uploadDate.setTime(date);
-        return uploadDate;
     }
 
     public static JsonObject getInitialData(String html) throws ParsingException {
         try {
-            String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
+            try {
-            return JsonParser.object().from(initialData);
+                final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
+                return JsonParser.object().from(initialData);
+            } catch (Parser.RegexException e) {
+                final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
+                return JsonParser.object().from(initialData);
+            }
         } catch (JsonParserException | Parser.RegexException e) {
             throw new ParsingException("Could not get ytInitialData", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@@ -18,9 +17,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.IOException;
-
 import javax.annotation.Nonnull;
+import java.io.IOException;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
@@ -87,7 +85,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
             final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING);
 
-            if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) {
+            if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
+                    || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") && !browseId.isEmpty()) {
                 if (!browseId.startsWith("UC")) {
                     throw new ExtractionException("Redirected id is not pointing to a channel");
                 }
@@ -191,12 +190,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                 throw new ParsingException("Could not get subscriber count", e);
             }
         } else {
-            // If there's no subscribe button, the channel has the subscriber count disabled
+            return ITEM_COUNT_UNKNOWN;
-            if (c4TabbedHeaderRenderer.has("subscribeButton")) {
-                return 0;
-            } else {
-                return -1;
-            }
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -47,13 +47,13 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
 
     @Override
     public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
-        String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
+        final String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
-        String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
+        final String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
         return getPage(getNextPage(commentsToken));
     }
 
     private Page getNextPage(JsonObject ajaxJson) throws ParsingException {
-        JsonArray arr;
+        final JsonArray arr;
         try {
             arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
         } catch (Exception e) {
@@ -89,14 +89,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
             throw new IllegalArgumentException("Page doesn't contain an URL");
         }
 
-        String ajaxResponse = makeAjaxRequest(page.getUrl());
+        final String ajaxResponse = makeAjaxRequest(page.getUrl());
-        JsonObject ajaxJson;
+        final JsonObject ajaxJson;
         try {
             ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1);
         } catch (Exception e) {
             throw new ParsingException("Could not parse json data for comments", e);
         }
-        CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
+        final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
         collectCommentsFrom(collector, ajaxJson);
         return new InfoItemsPage<>(collector, getNextPage(ajaxJson));
     }
@@ -160,8 +160,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
     }
 
     private String findValue(String doc, String start, String end) {
-        int beginIndex = doc.indexOf(start) + start.length();
+        final int beginIndex = doc.indexOf(start) + start.length();
-        int endIndex = doc.indexOf(end, beginIndex);
+        final int endIndex = doc.indexOf(end, beginIndex);
         return doc.substring(beginIndex, endIndex);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -34,7 +34,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     @Override
     public String getThumbnailUrl() throws ParsingException {
         try {
-            JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
+            final JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
             return JsonUtils.getString(arr.getObject(2), "url");
         } catch (Exception e) {
             throw new ParsingException("Could not get thumbnail url", e);
@@ -82,7 +82,13 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     @Override
     public String getCommentText() throws ParsingException {
         try {
-            String commentText = getTextFromObject(JsonUtils.getObject(json, "contentText"));
+            final JsonObject contentText = JsonUtils.getObject(json, "contentText");
+            if (contentText.isEmpty()) {
+                // completely empty comments as described in
+                // https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
+                return "";
+            }
+            final String commentText = getTextFromObject(contentText);
             // youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
             return Utils.removeUTF8BOM(commentText);
         } catch (Exception e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java

@@ -7,11 +7,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
 import javax.annotation.Nullable;
-import java.text.ParseException;
+import java.time.OffsetDateTime;
-import java.text.SimpleDateFormat;
+import java.time.format.DateTimeParseException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.TimeZone;
 
 public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
     private final Element entryElement;
@@ -62,19 +59,11 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
     @Nullable
     @Override
     public DateWrapper getUploadDate() throws ParsingException {
-        final Date date;
         try {
-            final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00");
+            return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate()));
-            dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+        } catch (DateTimeParseException e) {
-            date = dateFormat.parse(getTextualUploadDate());
-        } catch (ParseException e) {
             throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e);
         }
-
-        final Calendar calendar = Calendar.getInstance();
-        calendar.setTime(date);
-
-        return new DateWrapper(calendar);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -3,7 +3,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
 
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptableObject;
@@ -36,22 +41,21 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
+import java.time.LocalDate;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
 import java.io.UnsupportedEncodingException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.Calendar;
 import java.util.Collections;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
@@ -84,27 +88,25 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     // Exceptions
     //////////////////////////////////////////////////////////////////////////*/
 
-    public class DecryptException extends ParsingException {
+    public static class DeobfuscateException extends ParsingException {
-        DecryptException(String message, Throwable cause) {
+        DeobfuscateException(String message, Throwable cause) {
             super(message, cause);
         }
     }
 
     /*//////////////////////////////////////////////////////////////////////////*/
 
+    @Nullable private static String cachedDeobfuscationCode = null;
+    @Nullable private String playerJsUrl = null;
+
     private JsonArray initialAjaxJson;
-    @Nullable
-    private JsonObject playerArgs;
-    @Nonnull
-    private final Map<String, String> videoInfoPage = new HashMap<>();
-    private JsonObject playerResponse;
     private JsonObject initialData;
+    @Nonnull private final Map<String, String> videoInfoPage = new HashMap<>();
+    private JsonObject playerResponse;
     private JsonObject videoPrimaryInfoRenderer;
     private JsonObject videoSecondaryInfoRenderer;
-    private int ageLimit;
+    private int ageLimit = -1;
-
+    @Nullable private List<SubtitlesStream> subtitles = null;
-    @Nonnull
-    private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
 
     public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
         super(service, linkHandler);
@@ -135,18 +137,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return title;
     }
 
+    @Nullable
     @Override
     public String getTextualUploadDate() throws ParsingException {
-        if (getStreamType().equals(StreamType.LIVE_STREAM)) {
+        final JsonObject micro =
-            return null;
+                playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
-        }
+        if (!micro.getString("uploadDate", EMPTY_STRING).isEmpty()) {
-
-        JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
-        if (micro.isString("uploadDate") && !micro.getString("uploadDate").isEmpty()) {
             return micro.getString("uploadDate");
-        }
+        } else if (!micro.getString("publishDate", EMPTY_STRING).isEmpty()) {
-        if (micro.isString("publishDate") && !micro.getString("publishDate").isEmpty()) {
             return micro.getString("publishDate");
+        } else {
+            final JsonObject liveDetails = micro.getObject("liveBroadcastDetails");
+            if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) {
+                // an ended live stream
+                return liveDetails.getString("endTimestamp");
+            } else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) {
+                // a running live stream
+                return liveDetails.getString("startTimestamp");
+            } else if (getStreamType() == StreamType.LIVE_STREAM) {
+                // this should never be reached, but a live stream without upload date is valid
+                return null;
+            }
         }
 
         if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
@@ -154,22 +165,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
             try { // Premiered 20 hours ago
                 TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
-                Calendar parsedTime = timeAgoParser.parse(time).date();
+                OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
-                return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
+                return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
-            } catch (Exception ignored) {}
+            } catch (Exception ignored) {
+            }
 
             try { // Premiered Feb 21, 2020
-                Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
+                final LocalDate localDate = LocalDate.parse(time,
-                return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
+                        DateTimeFormatter.ofPattern("MMM dd, yyyy", Locale.ENGLISH));
-            } catch (Exception ignored) {}
+                return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
+            } catch (Exception ignored) {
+            }
         }
 
         try {
             // TODO: this parses English formatted dates only, we need a better approach to parse the textual date
-            Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
+            LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")),
-                    getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
+                    DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
-            return new SimpleDateFormat("yyyy-MM-dd").format(d);
+            return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
-        } catch (Exception ignored) {}
+        } catch (Exception ignored) {
+        }
+
         throw new ParsingException("Could not get upload date");
     }
 
@@ -217,9 +233,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public int getAgeLimit() {
+    public int getAgeLimit() throws ParsingException {
-        if (isNullOrEmpty(initialData)) throw new IllegalStateException("initialData is not parsed yet");
+        if (ageLimit == -1) {
+            ageLimit = NO_AGE_LIMIT;
 
+            final JsonArray metadataRows = getVideoSecondaryInfoRenderer()
+                    .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
+                    .getArray("rows");
+            for (final Object metadataRow : metadataRows) {
+                final JsonArray contents = ((JsonObject) metadataRow)
+                        .getObject("metadataRowRenderer").getArray("contents");
+                for (final Object content : contents) {
+                    final JsonArray runs = ((JsonObject) content).getArray("runs");
+                    for (final Object run : runs) {
+                        final String rowText = ((JsonObject) run).getString("text", EMPTY_STRING);
+                        if (rowText.contains("Age-restricted")) {
+                            ageLimit = 18;
+                            return ageLimit;
+                        }
+                    }
+                }
+            }
+        }
         return ageLimit;
     }
 
@@ -253,7 +288,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
      */
     @Override
     public long getTimeStamp() throws ParsingException {
-        return getTimestampSeconds("((#|&|\\?)(t|start)=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
+        final long timestamp =
+                getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
+
+        if (timestamp == -2) {
+            // regex for timestamp was not found
+            return 0;
+        } else {
+            return timestamp;
+        }
     }
 
     @Override
@@ -298,8 +341,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         } catch (NumberFormatException nfe) {
             throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe);
         } catch (Exception e) {
-            if (ageLimit == 18) return -1;
+            if (getAgeLimit() == NO_AGE_LIMIT) {
-            throw new ParsingException("Could not get like count", e);
+                throw new ParsingException("Could not get like count", e);
+            }
+            return -1;
         }
     }
 
@@ -322,8 +367,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         } catch (NumberFormatException nfe) {
             throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe);
         } catch (Exception e) {
-            if (ageLimit == 18) return -1;
+            if (getAgeLimit() == NO_AGE_LIMIT) {
-            throw new ParsingException("Could not get dislike count", e);
+                throw new ParsingException("Could not get dislike count", e);
+            }
+            return -1;
         }
     }
 
@@ -360,7 +407,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         try {
             uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
                     .getObject("videoOwnerRenderer").getObject("title"));
-        } catch (ParsingException ignored) { }
+        } catch (ParsingException ignored) {
+        }
 
         if (isNullOrEmpty(uploaderName)) {
             uploaderName = playerResponse.getObject("videoDetails").getString("author");
@@ -386,8 +434,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
 
         if (isNullOrEmpty(url)) {
-            if (ageLimit == 18) return "";
+            if (ageLimit == NO_AGE_LIMIT) {
-            throw new ParsingException("Could not get uploader avatar URL");
+                throw new ParsingException("Could not get uploader avatar URL");
+            }
+            return "";
         }
 
         return fixThumbnailUrl(url);
@@ -395,19 +445,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getSubChannelUrl() throws ParsingException {
+    public String getSubChannelUrl() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getSubChannelName() throws ParsingException {
+    public String getSubChannelName() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getSubChannelAvatarUrl() throws ParsingException {
+    public String getSubChannelAvatarUrl() {
         return "";
     }
 
@@ -421,18 +471,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 return playerResponse.getObject("streamingData").getString("dashManifestUrl");
             } else if (videoInfoPage.containsKey("dashmpd")) {
                 dashManifestUrl = videoInfoPage.get("dashmpd");
-            } else if (playerArgs != null && playerArgs.isString("dashmpd")) {
-                dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING);
             } else {
                 return "";
             }
 
             if (!dashManifestUrl.contains("/signature/")) {
-                String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
+                String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
-                String decryptedSig;
+                String deobfuscatedSig;
 
-                decryptedSig = decryptSignature(encryptedSig, decryptionCode);
+                deobfuscatedSig = deobfuscateSignature(obfuscatedSig);
-                dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
+                dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig);
             }
 
             return dashManifestUrl;
@@ -449,11 +497,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         try {
             return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
         } catch (Exception e) {
-            if (playerArgs != null && playerArgs.isString("hlsvp")) {
+            throw new ParsingException("Could not get hls manifest url", e);
-                return playerArgs.getString("hlsvp");
-            } else {
-                throw new ParsingException("Could not get hls manifest url", e);
-            }
         }
     }
 
@@ -519,35 +563,57 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Override
     @Nonnull
-    public List<SubtitlesStream> getSubtitlesDefault() {
+    public List<SubtitlesStream> getSubtitlesDefault() throws ParsingException {
         return getSubtitles(MediaFormat.TTML);
     }
 
     @Override
     @Nonnull
-    public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
+    public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws ParsingException {
         assertPageFetched();
-        List<SubtitlesStream> subtitles = new ArrayList<>();
+        // If the video is age restricted getPlayerConfig will fail
-        for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) {
+        if (getAgeLimit() != NO_AGE_LIMIT) {
-            subtitles.add(subtitlesInfo.getSubtitle(format));
+            return Collections.emptyList();
         }
+        if (subtitles != null) {
+            // already calculated
+            return subtitles;
+        }
+
+        final JsonObject renderer = playerResponse.getObject("captions")
+                .getObject("playerCaptionsTracklistRenderer");
+        final JsonArray captionsArray = renderer.getArray("captionTracks");
+        // TODO: use this to apply auto translation to different language from a source language
+        // final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
+
+        subtitles = new ArrayList<>();
+        for (int i = 0; i < captionsArray.size(); i++) {
+            final String languageCode = captionsArray.getObject(i).getString("languageCode");
+            final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
+            final String vssId = captionsArray.getObject(i).getString("vssId");
+
+            if (languageCode != null && baseUrl != null && vssId != null) {
+                final boolean isAutoGenerated = vssId.startsWith("a.");
+                final String cleanUrl = baseUrl
+                        .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
+                        .replaceAll("&tlang=[^&]*", ""); // Remove translation language
+
+                subtitles.add(new SubtitlesStream(format, languageCode,
+                        cleanUrl + "&fmt=" + format.getSuffix(), isAutoGenerated));
+            }
+        }
+
         return subtitles;
     }
 
     @Override
-    public StreamType getStreamType() throws ParsingException {
+    public StreamType getStreamType() {
         assertPageFetched();
-        try {
+        return playerResponse.getObject("streamingData").has(FORMATS)
-            if (!playerResponse.getObject("streamingData").has(FORMATS) ||
+                ? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM;
-                    (playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) {
-                return StreamType.LIVE_STREAM;
-            }
-        } catch (Exception e) {
-            throw new ParsingException("Could not get stream type", e);
-        }
-        return StreamType.VIDEO_STREAM;
     }
 
+    @Nullable
     private StreamInfoItemExtractor getNextStream() throws ExtractionException {
         try {
             final JsonObject firstWatchNextItem = initialData.getObject("contents")
@@ -568,11 +634,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
+    @Nullable
     @Override
     public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
         assertPageFetched();
 
-        if (getAgeLimit() != NO_AGE_LIMIT) return null;
+        if (getAgeLimit() != NO_AGE_LIMIT) {
+            return null;
+        }
 
         try {
             final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
@@ -604,10 +673,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public String getErrorMessage() {
         try {
-            return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
+            return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse")
-                    .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
+                    .getObject("playabilityStatus").getObject("errorScreen")
-        } catch (ParsingException e) {
+                    .getObject("playerErrorMessageRenderer").getObject("reason"));
-            return null;
+        } catch (ParsingException | NullPointerException e) {
+            return null; // no error message
         }
     }
 
@@ -618,7 +688,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private static final String FORMATS = "formats";
     private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
     private static final String HTTPS = "https:";
-    private static final String DECRYPTION_FUNC_NAME = "decrypt";
+    private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";
 
     private final static String[] REGEXES = {
             "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
@@ -627,36 +697,25 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(",
             "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("
     };
-            ;
-
-    private volatile String decryptionCode = "";
 
     @Override
-    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull final Downloader downloader)
-        final String url = getUrl() + "&pbj=1";
+            throws IOException, ExtractionException {
+        initialAjaxJson = getJsonResponse(getUrl() + "&pbj=1", getExtractorLocalization());
 
-        initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
+        initialData = initialAjaxJson.getObject(3).getObject("response", null);
-
+        if (initialData == null) {
-        final String playerUrl;
+            initialData = initialAjaxJson.getObject(2).getObject("response", null);
-
+            if (initialData == null) {
-        if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos
+                throw new ParsingException("Could not get initial data");
-            initialData = initialAjaxJson.getObject(2).getObject("response");
+            }
-            ageLimit = 18;
-
-            final EmbeddedInfo info = getEmbeddedInfo();
-            final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
-            final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
-            videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
-            playerUrl = info.url;
-        } else {
-            initialData = initialAjaxJson.getObject(3).getObject("response");
-            ageLimit = NO_AGE_LIMIT;
-
-            playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player"));
-            playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player"));
         }
 
-        playerResponse = getPlayerResponse();
+        playerResponse = initialAjaxJson.getObject(2).getObject("playerResponse", null);
+        if (playerResponse == null || !playerResponse.has("streamingData")) {
+            // try to get player response by fetching video info page
+            fetchVideoInfoPage();
+        }
 
         final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
         final String status = playabilityStatus.getString("status");
@@ -665,146 +724,59 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             final String reason = playabilityStatus.getString("reason");
             throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
         }
-
-        if (decryptionCode.isEmpty()) {
-            decryptionCode = loadDecryptionCode(playerUrl);
-        }
-
-        if (subtitlesInfos.isEmpty()) {
-            subtitlesInfos.addAll(getAvailableSubtitlesInfo());
-        }
     }
 
-    private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
+    private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
-        JsonObject playerArgs;
+        final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl();
+        final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
+        final String infoPageResponse = NewPipe.getDownloader()
+                .get(videoInfoUrl, getExtractorLocalization()).responseBody();
+        videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
 
-        //attempt to load the youtube js player JSON arguments
         try {
-            playerArgs = playerConfig.getObject("args");
+            playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
-        } catch (Exception e) {
+        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse yt player config", e);
+            throw new ParsingException(
-        }
+                    "Could not parse YouTube player response from video info page", e);
-
-        return playerArgs;
-    }
-
-    private String getPlayerUrl(JsonObject playerConfig) throws ParsingException {
-        try {
-            // The Youtube service needs to be initialized by downloading the
-            // js-Youtube-player. This is done in order to get the algorithm
-            // for decrypting cryptic signatures inside certain stream urls.
-            String playerUrl;
-
-            JsonObject ytAssets = playerConfig.getObject("assets");
-            playerUrl = ytAssets.getString("js");
-
-            if (playerUrl.startsWith("//")) {
-                playerUrl = HTTPS + playerUrl;
-            }
-            return playerUrl;
-        } catch (Exception e) {
-            throw new ParsingException("Could not load decryption code for the Youtube service.", e);
-        }
-    }
-
-    private JsonObject getPlayerResponse() throws ParsingException {
-        try {
-            String playerResponseStr;
-            if (playerArgs != null) {
-                playerResponseStr = playerArgs.getString("player_response");
-            } else {
-                playerResponseStr = videoInfoPage.get("player_response");
-            }
-            return JsonParser.object().from(playerResponseStr);
-        } catch (Exception e) {
-            throw new ParsingException("Could not parse yt player response", e);
         }
     }
 
     @Nonnull
-    private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
+    private String getEmbeddedInfoStsAndStorePlayerJsUrl() {
         try {
-            final Downloader downloader = NewPipe.getDownloader();
             final String embedUrl = "https://www.youtube.com/embed/" + getId();
-            final String embedPageContent = downloader.get(embedUrl, getExtractorLocalization()).responseBody();
+            final String embedPageContent = NewPipe.getDownloader()
-
+                    .get(embedUrl, getExtractorLocalization()).responseBody();
-            // Get player url
-            final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
-            String playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
-                    .replace("\\", "").replace("\"", "");
-            if (playerUrl.startsWith("//")) {
-                playerUrl = HTTPS + playerUrl;
-            }
 
             try {
-                // Get embed sts
+                final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
-                final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
+                playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
-                final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
+                        .replace("\\", "").replace("\"", "");
-                return new EmbeddedInfo(playerUrl, sts);
+            } catch (Parser.RegexException ex) {
-            } catch (Exception i) {
+                // playerJsUrl is still available in the file, just somewhere else TODO
-                // if it fails we simply reply with no sts as then it does not seem to be necessary
+                // it is ok not to find it, see how that's handled in getDeobfuscationCode()
-                return new EmbeddedInfo(playerUrl, "");
+                final Document doc = Jsoup.parse(embedPageContent);
+                final Elements elems = doc.select("script").attr("name", "player_ias/base");
+                for (Element elem : elems) {
+                    if (elem.attr("src").contains("base.js")) {
+                        playerJsUrl = elem.attr("src");
+                        break;
+                    }
+                }
             }
 
-        } catch (IOException e) {
+            // Get embed sts
-            throw new ParsingException(
+            return Parser.matchGroup1("\"sts\"\\s*:\\s*(\\d+)", embedPageContent);
-                    "Could load decryption code form restricted video for the Youtube service.", e);
+        } catch (Exception i) {
+            // if it fails we simply reply with no sts as then it does not seem to be necessary
+            return "";
         }
     }
 
-    private String loadDecryptionCode(String playerUrl) throws DecryptException {
-        try {
-            Downloader downloader = NewPipe.getDownloader();
-            if (!playerUrl.contains("https://youtube.com")) {
-                //sometimes the https://youtube.com part does not get send with
-                //than we have to add it by hand
-                playerUrl = "https://youtube.com" + playerUrl;
-            }
 
-            final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody();
-            final String decryptionFunctionName = getDecryptionFuncName(playerCode);
 
-            final String functionPattern = "("
-                    + decryptionFunctionName.replace("$", "\\$")
-                    + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
-            final String decryptionFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
 
-            final String helperObjectName =
+    private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
-                    Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunction);
-            final String helperPattern =
-                    "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
-            final String helperObject =
-                    Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
-
-            final String callerFunction =
-                    "function " + DECRYPTION_FUNC_NAME + "(a){return " + decryptionFunctionName + "(a);}";
-
-            return helperObject + decryptionFunction + callerFunction;
-        } catch (IOException ioe) {
-            throw new DecryptException("Could not load decrypt function", ioe);
-        } catch (Exception e) {
-            throw new DecryptException("Could not parse decrypt function ", e);
-        }
-    }
-
-    private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
-        Context context = Context.enter();
-        context.setOptimizationLevel(-1);
-        Object result;
-        try {
-            ScriptableObject scope = context.initStandardObjects();
-            context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null);
-            Function decryptionFunc = (Function) scope.get("decrypt", scope);
-            result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
-        } catch (Exception e) {
-            throw new DecryptException("could not get decrypt signature", e);
-        } finally {
-            Context.exit();
-        }
-        return result == null ? "" : result.toString();
-    }
-
-    private String getDecryptionFuncName(final String playerCode) throws DecryptException {
         Parser.RegexException exception = null;
         for (final String regex : REGEXES) {
             try {
@@ -815,75 +787,81 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 }
             }
         }
-        throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception);
+        throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception);
+    }
+
+    private String loadDeobfuscationCode(@Nonnull final String playerJsUrl)
+            throws DeobfuscateException {
+        try {
+            final String playerCode = NewPipe.getDownloader()
+                    .get(playerJsUrl, getExtractorLocalization()).responseBody();
+            final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
+
+            final String functionPattern = "("
+                    + deobfuscationFunctionName.replace("$", "\\$")
+                    + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
+            final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
+
+            final String helperObjectName =
+                    Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", deobfuscateFunction);
+            final String helperPattern =
+                    "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
+            final String helperObject =
+                    Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
+
+            final String callerFunction =
+                    "function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}";
+
+            return helperObject + deobfuscateFunction + callerFunction;
+        } catch (IOException ioe) {
+            throw new DeobfuscateException("Could not load deobfuscate function", ioe);
+        } catch (Exception e) {
+            throw new DeobfuscateException("Could not parse deobfuscate function ", e);
+        }
     }
 
     @Nonnull
-    private List<SubtitlesInfo> getAvailableSubtitlesInfo() {
+    private String getDeobfuscationCode() throws ParsingException {
-        // If the video is age restricted getPlayerConfig will fail
+        if (cachedDeobfuscationCode == null) {
-        if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList();
+            if (playerJsUrl == null) {
-
+                // the currentPlayerJsUrl was not found in any page fetched so far and there is
-        final JsonObject captions;
+                // nothing cached, so try fetching embedded info
-        if (!playerResponse.has("captions")) {
+                getEmbeddedInfoStsAndStorePlayerJsUrl();
-            // Captions does not exist
+                if (playerJsUrl == null) {
-            return Collections.emptyList();
+                    throw new ParsingException(
-        }
+                            "Embedded info did not provide YouTube player js url");
-        captions = playerResponse.getObject("captions");
+                }
-
-        final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer");
-        final JsonArray captionsArray = renderer.getArray("captionTracks");
-        // todo: use this to apply auto translation to different language from a source language
-//        final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
-
-        // This check is necessary since there may be cases where subtitles metadata do not contain caption track info
-        // e.g. https://www.youtube.com/watch?v=-Vpwatutnko
-        final int captionsSize = captionsArray.size();
-        if (captionsSize == 0) return Collections.emptyList();
-
-        List<SubtitlesInfo> result = new ArrayList<>();
-        for (int i = 0; i < captionsSize; i++) {
-            final String languageCode = captionsArray.getObject(i).getString("languageCode");
-            final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
-            final String vssId = captionsArray.getObject(i).getString("vssId");
-
-            if (languageCode != null && baseUrl != null && vssId != null) {
-                final boolean isAutoGenerated = vssId.startsWith("a.");
-                result.add(new SubtitlesInfo(baseUrl, languageCode, isAutoGenerated));
             }
+
+            if (playerJsUrl.startsWith("//")) {
+                playerJsUrl = HTTPS + playerJsUrl;
+            } else if (playerJsUrl.startsWith("/")) {
+                // sometimes https://youtube.com part has to be added manually
+                playerJsUrl = HTTPS + "//youtube.com" + playerJsUrl;
+            }
+
+            cachedDeobfuscationCode = loadDeobfuscationCode(playerJsUrl);
         }
-
+        return cachedDeobfuscationCode;
-        return result;
-    }
-    /*//////////////////////////////////////////////////////////////////////////
-    // Data Class
-    //////////////////////////////////////////////////////////////////////////*/
-
-    private class EmbeddedInfo {
-        final String url;
-        final String sts;
-
-        EmbeddedInfo(final String url, final String sts) {
-            this.url = url;
-            this.sts = sts;
-        }
     }
 
-    private class SubtitlesInfo {
+    private String deobfuscateSignature(final String obfuscatedSig) throws ParsingException {
-        final String cleanUrl;
+        final String deobfuscationCode = getDeobfuscationCode();
-        final String languageCode;
-        final boolean isGenerated;
 
-        public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) {
+        final Context context = Context.enter();
-            this.cleanUrl = baseUrl
+        context.setOptimizationLevel(-1);
-                    .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
+        final Object result;
-                    .replaceAll("&tlang=[^&]*", ""); // Remove translation language
+        try {
-            this.languageCode = languageCode;
+            final ScriptableObject scope = context.initSafeStandardObjects();
-            this.isGenerated = isGenerated;
+            context.evaluateString(scope, deobfuscationCode, "deobfuscationCode", 1, null);
-        }
+            final Function deobfuscateFunc = (Function) scope.get(DEOBFUSCATION_FUNC_NAME, scope);
-
+            result = deobfuscateFunc.call(context, scope, scope, new Object[]{obfuscatedSig});
-        public SubtitlesStream getSubtitle(final MediaFormat format) {
+        } catch (Exception e) {
-            return new SubtitlesStream(format, languageCode, cleanUrl + "&fmt=" + format.getSuffix(), isGenerated);
+            throw new DeobfuscateException("Could not get deobfuscate signature", e);
+        } finally {
+            Context.exit();
         }
+        return result == null ? "" : result.toString();
     }
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -942,14 +920,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 "&sts=" + sts + "&ps=default&gl=US&hl=en";
     }
 
-    private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
+    private Map<String, ItagItem> getItags(final String streamingDataKey,
-        Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
+                                           final ItagItem.ItagType itagTypeWanted)
-        JsonObject streamingData = playerResponse.getObject("streamingData");
+            throws ParsingException {
+        final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
+        final JsonObject streamingData = playerResponse.getObject("streamingData");
         if (!streamingData.has(streamingDataKey)) {
             return urlAndItags;
         }
 
-        JsonArray formats = streamingData.getArray(streamingDataKey);
+        final JsonArray formats = streamingData.getArray(streamingDataKey);
         for (int i = 0; i != formats.size(); ++i) {
             JsonObject formatData = formats.getObject(i);
             int itag = formatData.getInt("itag");
@@ -958,22 +938,30 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 try {
                     ItagItem itagItem = ItagItem.getItag(itag);
                     if (itagItem.itagType == itagTypeWanted) {
+                        // Ignore streams that are delivered using YouTube's OTF format,
+                        // as those only work with DASH and not with progressive HTTP.
+                        if (formatData.getString("type", EMPTY_STRING)
+                                .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
+                            continue;
+                        }
+
                         String streamUrl;
                         if (formatData.has("url")) {
                             streamUrl = formatData.getString("url");
                         } else {
-                            // this url has an encrypted signature
+                            // this url has an obfuscated signature
                             final String cipherString = formatData.has("cipher")
                                     ? formatData.getString("cipher")
                                     : formatData.getString("signatureCipher");
                             final Map<String, String> cipher = Parser.compatParseMap(cipherString);
                             streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
-                                    + decryptSignature(cipher.get("s"), decryptionCode);
+                                    + deobfuscateSignature(cipher.get("s"));
                         }
 
                         urlAndItags.put(streamUrl, itagItem);
                     }
-                } catch (UnsupportedEncodingException ignored) {}
+                } catch (UnsupportedEncodingException ignored) {
+                }
             }
         }
 
@@ -984,12 +972,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public List<Frameset> getFrames() throws ExtractionException {
         try {
-            JsonObject jo = initialAjaxJson.getObject(2).getObject("player");
+            final JsonObject storyboards = playerResponse.getObject("storyboards");
-            final String resp = jo.getObject("args").getString("player_response");
+            final JsonObject storyboardsRenderer;
-            jo = JsonParser.object().from(resp);
+            if (storyboards.has("playerLiveStoryboardSpecRenderer")) {
-            final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
+                storyboardsRenderer = storyboards.getObject("playerLiveStoryboardSpecRenderer");
+            } else {
+                storyboardsRenderer = storyboards.getObject("playerStoryboardSpecRenderer");
+            }
+
+            final String[] spec = storyboardsRenderer.getString("spec").split("\\|");
             final String url = spec[0];
             final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
+
             for (int i = 1; i < spec.length; ++i) {
                 final String[] parts = spec[i].split("#");
                 if (parts.length != 8) {
@@ -1059,7 +1053,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<String> getTags() {
-        return new ArrayList<>();
+        return Collections.emptyList();
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -12,11 +12,14 @@ import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nullable;
-import java.text.SimpleDateFormat;
+import java.time.Instant;
-import java.util.Calendar;
+import java.time.OffsetDateTime;
-import java.util.Date;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -165,8 +168,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
         }
 
         if (isPremiere()) {
-            final Date date = getDateFromPremiere().getTime();
+            return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere());
-            return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
         }
 
         final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
@@ -250,15 +252,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
         return videoInfo.has("upcomingEventData");
     }
 
-    private Calendar getDateFromPremiere() throws ParsingException {
+    private OffsetDateTime getDateFromPremiere() throws ParsingException {
         final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData");
         final String startTime = upcomingEventData.getString("startTime");
 
         try {
-            final long startTimeTimestamp = Long.parseLong(startTime);
+            return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)),
-            final Calendar calendar = Calendar.getInstance();
+                    ZoneOffset.UTC);
-            calendar.setTime(new Date(startTimeTimestamp * 1000L));
-            return calendar;
         } catch (Exception e) {
             throw new ParsingException("Could not parse date from premiere:  \"" + startTime + "\"");
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java

@@ -1,126 +1,71 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import org.jsoup.Jsoup;
+import com.grack.nanojson.JsonArray;
-import org.jsoup.nodes.Document;
+import com.grack.nanojson.JsonObject;
-import org.jsoup.nodes.Element;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
-import org.schabi.newpipe.extractor.utils.Parser;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import javax.annotation.Nonnull;
+
 import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM;
 
 /**
- * Extract subscriptions from a YouTube export (OPML format supported)
+ * Extract subscriptions from a Google takout export (the user has to get the JSON out of the zip)
  */
 public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
+    private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/";
 
-    public YoutubeSubscriptionExtractor(YoutubeService service) {
+    public YoutubeSubscriptionExtractor(final YoutubeService youtubeService) {
-        super(service, Collections.singletonList(INPUT_STREAM));
+        super(youtubeService, Collections.singletonList(INPUT_STREAM));
     }
 
     @Override
     public String getRelatedUrl() {
-        return "https://www.youtube.com/subscription_manager?action_takeout=1";
+        return "https://takeout.google.com/takeout/custom/youtube";
     }
 
     @Override
-    public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws ExtractionException {
+    public List<SubscriptionItem> fromInputStream(@Nonnull final InputStream contentInputStream)
-        if (contentInputStream == null) throw new InvalidSourceException("input stream is null");
+            throws ExtractionException {
-
+        final JsonArray subscriptions;
-        return getItemsFromOPML(contentInputStream);
-    }
-
-    /*//////////////////////////////////////////////////////////////////////////
-    // OPML implementation
-    //////////////////////////////////////////////////////////////////////////*/
-
-    private static final String ID_PATTERN = "/videos.xml\\?channel_id=([A-Za-z0-9_-]*)";
-    private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/";
-
-    private List<SubscriptionItem> getItemsFromOPML(InputStream contentInputStream) throws ExtractionException {
-        final List<SubscriptionItem> result = new ArrayList<>();
-
-        final String contentString = readFromInputStream(contentInputStream);
-        Document document = Jsoup.parse(contentString, "", org.jsoup.parser.Parser.xmlParser());
-
-        if (document.select("opml").isEmpty()) {
-            throw new InvalidSourceException("document does not have OPML tag");
-        }
-
-        if (document.select("outline").isEmpty()) {
-            throw new InvalidSourceException("document does not have at least one outline tag");
-        }
-
-        for (Element outline : document.select("outline[type=rss]")) {
-            String title = outline.attr("title");
-            String xmlUrl = outline.attr("abs:xmlUrl");
-
-            try {
-                String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
-                result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
-            } catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ }
-        }
-
-        return result;
-    }
-
-    /*//////////////////////////////////////////////////////////////////////////
-    // Utils
-    //////////////////////////////////////////////////////////////////////////*/
-
-    /**
-     * Throws an exception if the string does not have the right tag/string from a valid export.
-     */
-    private void throwIfTagIsNotFound(String content) throws InvalidSourceException {
-        if (!content.trim().contains("<opml")) {
-            throw new InvalidSourceException("input stream does not have OPML tag");
-        }
-    }
-
-    private String readFromInputStream(InputStream inputStream) throws InvalidSourceException {
-        StringBuilder contentBuilder = new StringBuilder();
-        boolean hasTag = false;
         try {
-            byte[] buffer = new byte[16 * 1024];
+            subscriptions = JsonParser.array().from(contentInputStream);
-            int read;
+        } catch (JsonParserException e) {
-            while ((read = inputStream.read(buffer)) != -1) {
+            throw new InvalidSourceException("Invalid json input stream", e);
-                String currentPartOfContent = new String(buffer, 0, read, "UTF-8");
+        }
-                contentBuilder.append(currentPartOfContent);
 
-                // Fail-fast in case of reading a long unsupported input stream
+        boolean foundInvalidSubscription = false;
-                if (!hasTag && contentBuilder.length() > 128) {
+        final List<SubscriptionItem> subscriptionItems = new ArrayList<>();
-                    throwIfTagIsNotFound(contentBuilder.toString());
+        for (final Object subscriptionObject : subscriptions) {
-                    hasTag = true;
+            if (!(subscriptionObject instanceof JsonObject)) {
-                }
+                foundInvalidSubscription = true;
+                continue;
             }
-        } catch (InvalidSourceException e) {
+
-            throw e;
+            final JsonObject subscription = ((JsonObject) subscriptionObject).getObject("snippet");
-        } catch (Throwable e) {
+            final String id = subscription.getObject("resourceId").getString("channelId", "");
-            throw new InvalidSourceException(e);
+            if (id.length() != 24) { // e.g. UCsXVk37bltHxD1rDPwtNM8Q
-        } finally {
+                foundInvalidSubscription = true;
-            try {
+                continue;
-                inputStream.close();
-            } catch (IOException ignored) {
             }
+
+            subscriptionItems.add(new SubscriptionItem(service.getServiceId(),
+                    BASE_CHANNEL_URL + id, subscription.getString("title", "")));
         }
 
-        final String fileContent = contentBuilder.toString().trim();
+        if (foundInvalidSubscription && subscriptionItems.isEmpty()) {
-        if (fileContent.isEmpty()) {
+            throw new InvalidSourceException("Found only invalid channel ids");
-            throw new InvalidSourceException("Empty input stream");
         }
-
+        return subscriptionItems;
-        if (!hasTag) {
-            throwIfTagIsNotFound(fileContent);
-        }
-
-        return fileContent;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
+import java.util.regex.Pattern;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
@@ -32,6 +33,9 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final YoutubeChannelLinkHandlerFactory instance = new YoutubeChannelLinkHandlerFactory();
 
+    private static final Pattern excludedSegments =
+      Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site");
+
     public static YoutubeChannelLinkHandlerFactory getInstance() {
         return instance;
     }
@@ -48,11 +52,22 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
     public String getUrl(String id, List<String> contentFilters, String searchFilter) {
         return "https://www.youtube.com/" + id;
     }
+    
+    /**
+     * Returns true if path conform to
+     * custom short channel URLs like youtube.com/yourcustomname
+     *
+     * @param splitPath path segments array
+     * @return true - if value conform to short channel URL, false - not
+     */
+    private boolean isCustomShortChannelUrl(final String[] splitPath) {
+        return splitPath.length == 1 && !excludedSegments.matcher(splitPath[0]).matches();
+    }
 
     @Override
     public String getId(String url) throws ParsingException {
         try {
-            URL urlObj = Utils.stringToURL(url);
+            final URL urlObj = Utils.stringToURL(url);
             String path = urlObj.getPath();
 
             if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) ||
@@ -60,15 +75,21 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
                 throw new ParsingException("the URL given is not a Youtube-URL");
             }
 
-            if (!path.startsWith("/user/") && !path.startsWith("/channel/") && !path.startsWith("/c/")) {
+            // remove leading "/"
+            path = path.substring(1);
+            String[] splitPath = path.split("/");
+
+            // Handle custom short channel URLs like youtube.com/yourcustomname
+            if (isCustomShortChannelUrl(splitPath)) {
+                path = "c/" + path;
+                splitPath = path.split("/");
+            }
+
+            if (!path.startsWith("user/") && !path.startsWith("channel/") && !path.startsWith("c/")) {
                 throw new ParsingException("the URL given is neither a channel nor an user");
             }
 
-            // remove leading "/"
+            final String id = splitPath[1];
-            path = path.substring(1);
-
-            String[] splitPath = path.split("/");
-            String id = splitPath[1];
 
             if (id == null || !id.matches("[A-Za-z0-9_-]+")) {
                 throw new ParsingException("The given id is not a Youtube-Video-ID");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java

@@ -1,10 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL;
-
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 
 import java.util.List;
@@ -17,15 +14,6 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
         return instance;
     }
 
-    @Override
-    public ListLinkHandler fromUrl(String url) throws ParsingException {
-        if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){
-            return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL);
-        } else {
-            return super.fromUrl(url);
-        }
-    }
-
     @Override
     public String getUrl(String id) {
         return "https://m.youtube.com/watch?v=" + id;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java

@@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -12,8 +11,8 @@ import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
-
+import java.util.regex.Matcher;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL;
+import java.util.regex.Pattern;
 
 /*
  * Created by Christian Schabesberger on 02.02.16.
@@ -37,6 +36,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
 
 public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
 
+    private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("([a-zA-Z0-9_-]{11})");
     private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory();
 
     private YoutubeStreamLinkHandlerFactory() {
@@ -46,27 +46,24 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
         return instance;
     }
 
-    private static boolean isId(@Nullable String id) {
+    @Nullable
-        return id != null && id.matches("[a-zA-Z0-9_-]{11}");
+    private static String extractId(@Nullable final String id) {
+        if (id != null) {
+            final Matcher m = YOUTUBE_VIDEO_ID_REGEX_PATTERN.matcher(id);
+            return m.find() ? m.group(1) : null;
+        }
+        return null;
     }
 
-    private static String assertIsId(@Nullable String id) throws ParsingException {
+    private static String assertIsId(@Nullable final String id) throws ParsingException {
-        if (isId(id)) {
+        final String extractedId = extractId(id);
-            return id;
+        if (extractedId != null) {
+            return extractedId;
         } else {
             throw new ParsingException("The given string is not a Youtube-Video-ID");
         }
     }
 
-    @Override
-    public LinkHandler fromUrl(String url) throws ParsingException {
-        if (url.startsWith(BASE_YOUTUBE_INTENT_URL)) {
-            return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL);
-        } else {
-            return super.fromUrl(url);
-        }
-    }
-
     @Override
     public String getUrl(String id) {
         return "https://www.youtube.com/watch?v=" + id;
@@ -81,9 +78,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
             if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
                 String schemeSpecificPart = uri.getSchemeSpecificPart();
                 if (schemeSpecificPart.startsWith("//")) {
-                    final String possiblyId = schemeSpecificPart.substring(2);
+                    final String extractedId = extractId(schemeSpecificPart.substring(2));
-                    if (isId(possiblyId)) {
+                    if (extractedId != null) {
-                        return possiblyId;
+                        return extractedId;
                     }
 
                     urlString = "https:" + schemeSpecificPart;
@@ -153,7 +150,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
                     return assertIsId(viewQueryValue);
                 }
 
-                if (path.startsWith("embed/")) {
+                if (path.startsWith("embed/") || path.startsWith("shorts/")) {
                     String id = path.split("/")[1];
 
                     return assertIsId(id);

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -321,6 +321,7 @@ public abstract class StreamExtractor extends Extractor {
      * @throws IOException
      * @throws ExtractionException
      */
+    @Nullable
     public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException;
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java

@@ -8,7 +8,6 @@ import java.util.Locale;
 public class SubtitlesStream extends Stream implements Serializable {
     private final MediaFormat format;
     private final Locale locale;
-    private final String url;
     private final boolean autoGenerated;
     private final String code;
 
@@ -34,7 +33,6 @@ public class SubtitlesStream extends Stream implements Serializable {
         }
         this.code = languageCode;
         this.format = format;
-        this.url = url;
         this.autoGenerated = autoGenerated;
     }
 
@@ -42,10 +40,6 @@ public class SubtitlesStream extends Stream implements Serializable {
         return format.suffix;
     }
 
-    public String getURL() {
-        return url;
-    }
-
     public boolean isAutoGenerated() {
         return autoGenerated;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java

@@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.InputStream;
@@ -71,8 +72,9 @@ public abstract class SubscriptionExtractor {
      *
      * @throws InvalidSourceException when the content read from the InputStream is invalid and can not be parsed
      */
-    @SuppressWarnings("RedundantThrows")
+    public List<SubscriptionItem> fromInputStream(@Nonnull final InputStream contentInputStream)
-    public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws IOException, ExtractionException {
+            throws ExtractionException {
-        throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from an InputStream");
+        throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName()
+                + " doesn't support extracting from an InputStream");
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java

@@ -15,16 +15,14 @@ public class DonationLinkHelper {
         AMAZON,
     }
 
-    public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException {
+    public static DonationService getDonationServiceByLink(String link) throws MalformedURLException {
         URL url = new URL(fixLink(link));
         switch (url.getHost()) {
             case "www.patreon.com":
-                return DonationService.PATREON;
             case "patreon.com":
                 return DonationService.PATREON;
-            case "paypal.me":
-                return DonationService.PAYPAL;
             case "www.paypal.me":
+            case "paypal.me":
                 return DonationService.PAYPAL;
             default:
                 return DonationService.NO_DONATION;

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -181,14 +181,39 @@ public class Utils {
         return s;
     }
 
-    public static String getBaseUrl(String url) throws ParsingException {
+    public static String getBaseUrl(final String url) throws ParsingException {
-        URL uri;
         try {
-            uri = stringToURL(url);
+            final URL uri = stringToURL(url);
-        } catch (MalformedURLException e) {
+            return uri.getProtocol() + "://" + uri.getAuthority();
+        } catch (final MalformedURLException e) {
+            final String message = e.getMessage();
+            if (message.startsWith("unknown protocol: ")) {
+                // return just the protocol (e.g. vnd.youtube)
+                return message.substring("unknown protocol: ".length());
+            }
+
             throw new ParsingException("Malformed url: " + url, e);
         }
-        return uri.getProtocol() + "://" + uri.getAuthority();
+    }
+
+    /**
+     * If the provided url is a Google search redirect, then the actual url is extracted from the
+     * {@code url=} query value and returned, otherwise the original url is returned.
+     * @param url the url which can possibly be a Google search redirect
+     * @return an url with no Google search redirects
+     */
+    public static String followGoogleRedirectIfNeeded(final String url) {
+        // if the url is a redirect from a Google search, extract the actual url
+        try {
+            final URL decoded = Utils.stringToURL(url);
+            if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
+                return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8");
+            }
+        } catch (final Exception ignored) {
+        }
+
+        // url is not a google search redirect
+        return url;
     }
 
     public static boolean isNullOrEmpty(final String str) {

extractor/src/test/java/org/schabi/newpipe/FileUtils.java

@@ -0,0 +1,94 @@
+package org.schabi.newpipe;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonWriter;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * Util class to write file to disk
+ * <p>
+ * Can be used to debug and test, for example writing a service's JSON response
+ * (especially useful if the response provided by the service is not documented)
+ */
+public class FileUtils {
+
+    public static void createFile(String path, JsonObject content) throws IOException {
+        createFile(path, jsonObjToString(content));
+    }
+
+    public static void createFile(String path, JsonArray array) throws IOException {
+        createFile(path, jsonArrayToString(array));
+    }
+
+    /**
+     * Create a file given a path and its content. Create subdirectories if needed
+     *
+     * @param path    the path to write the file, including the filename (and its extension)
+     * @param content the content to write
+     * @throws IOException
+     */
+    public static void createFile(final String path, final String content) throws IOException {
+        final String[] dirs = path.split("/");
+        if (dirs.length > 1) {
+            String pathWithoutFileName = path.replace(dirs[dirs.length - 1], "");
+            if (!Files.exists(Paths.get(pathWithoutFileName))) { //create dirs if they don't exist
+                if (!new File(pathWithoutFileName).mkdirs()) {
+                    throw new IOException("An error occurred while creating directories");
+                }
+            }
+        }
+        writeFile(path, content);
+    }
+
+    /**
+     * Write a file to disk
+     *
+     * @param filename the file name (and its extension if wanted)
+     * @param content  the content to write
+     * @throws IOException
+     */
+    private static void writeFile(final String filename, final String content) throws IOException {
+        final BufferedWriter writer = new BufferedWriter(new FileWriter(filename));
+        writer.write(content);
+        writer.flush();
+        writer.close();
+    }
+
+    /**
+     * Resolves the test resource file based on its filename. Looks in
+     * {@code extractor/src/test/resources/} and {@code src/test/resources/}
+     * @param filename the resource filename
+     * @return the resource file
+     */
+    public static File resolveTestResource(final String filename) {
+        final File file = new File("extractor/src/test/resources/" + filename);
+        if (file.exists()) {
+            return file;
+        } else {
+            return new File("src/test/resources/" + filename);
+        }
+    }
+
+    /**
+     * Convert a JSON object to String
+     * toString() does not produce a valid JSON string
+     */
+    public static String jsonObjToString(JsonObject object) {
+        return JsonWriter.string(object);
+    }
+
+    /**
+     * Convert a JSON array to String
+     * toString() does not produce a valid JSON string
+     */
+    public static String jsonArrayToString(JsonArray array) {
+        return JsonWriter.string(array);
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java

@@ -1,12 +1,19 @@
 package org.schabi.newpipe.extractor;
 
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.List;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 public class ExtractorAsserts {
     public static void assertEmptyErrors(String message, List<Throwable> errors) {
@@ -56,4 +63,22 @@ public class ExtractorAsserts {
             assertTrue(message, stringToCheck.isEmpty());
         }
     }
+
+    public static void assertAtLeast(long expected, long actual) {
+        assertTrue(actual + " is not at least " + expected, actual >= expected);
+    }
+
+    // this assumes that sorting a and b in-place is not an issue, so it's only intended for tests
+    public static void assertEqualsOrderIndependent(List<String> expected, List<String> actual) {
+        if (expected == null) {
+            assertNull(actual);
+            return;
+        } else {
+            assertNotNull(actual);
+        }
+
+        Collections.sort(expected);
+        Collections.sort(actual);
+        assertEquals(expected, actual);
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java

@@ -6,6 +6,7 @@ import java.util.HashSet;
 
 import static org.junit.Assert.*;
 import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class NewPipeTest {
@@ -39,8 +40,10 @@ public class NewPipeTest {
         assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), YouTube);
         assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), YouTube);
         assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), YouTube);
+        assertEquals(getServiceByUrl("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"), YouTube);
 
-        assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), YouTube);
+        assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud);
+        assertEquals(getServiceByUrl("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd="), SoundCloud);
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java

@@ -0,0 +1,35 @@
+package org.schabi.newpipe.extractor.services;
+
+public interface BaseStreamExtractorTest extends BaseExtractorTest {
+    void testStreamType() throws Exception;
+    void testUploaderName() throws Exception;
+    void testUploaderUrl() throws Exception;
+    void testUploaderAvatarUrl() throws Exception;
+    void testSubChannelName() throws Exception;
+    void testSubChannelUrl() throws Exception;
+    void testSubChannelAvatarUrl() throws Exception;
+    void testThumbnailUrl() throws Exception;
+    void testDescription() throws Exception;
+    void testLength() throws Exception;
+    void testTimestamp() throws Exception;
+    void testViewCount() throws Exception;
+    void testUploadDate() throws Exception;
+    void testTextualUploadDate() throws Exception;
+    void testLikeCount() throws Exception;
+    void testDislikeCount() throws Exception;
+    void testRelatedStreams() throws Exception;
+    void testAgeLimit() throws Exception;
+    void testErrorMessage() throws Exception;
+    void testAudioStreams() throws Exception;
+    void testVideoStreams() throws Exception;
+    void testSubtitles() throws Exception;
+    void testGetDashMpdUrl() throws Exception;
+    void testFrames() throws Exception;
+    void testHost() throws Exception;
+    void testPrivacy() throws Exception;
+    void testCategory() throws Exception;
+    void testLicence() throws Exception;
+    void testLanguageInfo() throws Exception;
+    void testTags() throws Exception;
+    void testSupportInfo() throws Exception;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java

@@ -0,0 +1,382 @@
+package org.schabi.newpipe.extractor.services;
+
+import org.junit.Test;
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.Frameset;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.stream.SubtitlesStream;
+import org.schabi.newpipe.extractor.stream.VideoStream;
+
+import javax.annotation.Nullable;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertAtLeast;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderIndependent;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
+import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
+
+/**
+ * Test for {@link StreamExtractor}
+ */
+public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<StreamExtractor>
+        implements BaseStreamExtractorTest {
+
+    public abstract StreamType expectedStreamType();
+    public abstract String expectedUploaderName();
+    public abstract String expectedUploaderUrl();
+    public String expectedSubChannelName() { return ""; } // default: there is no subchannel
+    public String expectedSubChannelUrl() { return ""; } // default: there is no subchannel
+    public abstract List<String> expectedDescriptionContains(); // e.g. for full links
+    public abstract long expectedLength();
+    public long expectedTimestamp() { return 0; } // default: there is no timestamp
+    public abstract long expectedViewCountAtLeast();
+    @Nullable public abstract String expectedUploadDate(); // format: "yyyy-MM-dd HH:mm:ss.SSS"
+    @Nullable public abstract String expectedTextualUploadDate();
+    public abstract long expectedLikeCountAtLeast(); // return -1 if ratings are disabled
+    public abstract long expectedDislikeCountAtLeast(); // return -1 if ratings are disabled
+    public boolean expectedHasRelatedStreams() { return true; } // default: there are related videos
+    public int expectedAgeLimit() { return StreamExtractor.NO_AGE_LIMIT; } // default: no limit
+    @Nullable public String expectedErrorMessage() { return null; } // default: no error message
+    public boolean expectedHasVideoStreams() { return true; } // default: there are video streams
+    public boolean expectedHasAudioStreams() { return true; } // default: there are audio streams
+    public boolean expectedHasSubtitles() { return true; } // default: there are subtitles streams
+    @Nullable public String expectedDashMpdUrlContains() { return null; } // default: no dash mpd
+    public boolean expectedHasFrames() { return true; } // default: there are frames
+    public String expectedHost() { return ""; } // default: no host for centralized platforms
+    public String expectedPrivacy() { return ""; } // default: no privacy policy available
+    public String expectedCategory() { return ""; } // default: no category
+    public String expectedLicence() { return ""; } // default: no licence
+    public Locale expectedLanguageInfo() { return null; } // default: no language info available
+    public List<String> expectedTags() { return Collections.emptyList(); } // default: no tags
+    public String expectedSupportInfo() { return ""; } // default: no support info available
+
+    @Test
+    @Override
+    public void testStreamType() throws Exception {
+        assertEquals(expectedStreamType(), extractor().getStreamType());
+    }
+
+    @Test
+    @Override
+    public void testUploaderName() throws Exception {
+        assertEquals(expectedUploaderName(), extractor().getUploaderName());
+    }
+
+    @Test
+    @Override
+    public void testUploaderUrl() throws Exception {
+        final String uploaderUrl = extractor().getUploaderUrl();
+        assertIsSecureUrl(uploaderUrl);
+        assertEquals(expectedUploaderUrl(), uploaderUrl);
+    }
+
+    @Test
+    @Override
+    public void testUploaderAvatarUrl() throws Exception {
+        assertIsSecureUrl(extractor().getUploaderAvatarUrl());
+    }
+
+    @Test
+    @Override
+    public void testSubChannelName() throws Exception {
+        assertEquals(expectedSubChannelName(), extractor().getSubChannelName());
+    }
+
+    @Test
+    @Override
+    public void testSubChannelUrl() throws Exception {
+        final String subChannelUrl = extractor().getSubChannelUrl();
+        assertEquals(expectedSubChannelUrl(), subChannelUrl);
+
+        if (!expectedSubChannelUrl().isEmpty()) {
+            // this stream has a subchannel
+            assertIsSecureUrl(subChannelUrl);
+        }
+    }
+
+    @Test
+    @Override
+    public void testSubChannelAvatarUrl() throws Exception {
+        if (expectedSubChannelName().isEmpty() && expectedSubChannelUrl().isEmpty()) {
+            // this stream has no subchannel
+            assertEquals("", extractor().getSubChannelAvatarUrl());
+        } else {
+            // this stream has a subchannel
+            assertIsSecureUrl(extractor().getSubChannelAvatarUrl());
+        }
+    }
+
+    @Test
+    @Override
+    public void testThumbnailUrl() throws Exception {
+        assertIsSecureUrl(extractor().getThumbnailUrl());
+    }
+
+    @Test
+    @Override
+    public void testDescription() throws Exception {
+        final Description description = extractor().getDescription();
+        assertNotNull(description);
+        assertFalse("description is empty", description.getContent().isEmpty());
+
+        for (final String s : expectedDescriptionContains()) {
+            assertThat(description.getContent(), containsString(s));
+        }
+    }
+
+    @Test
+    @Override
+    public void testLength() throws Exception {
+        assertEquals(expectedLength(), extractor().getLength());
+    }
+
+    @Test
+    @Override
+    public void testTimestamp() throws Exception {
+        assertEquals(expectedTimestamp(), extractor().getTimeStamp());
+    }
+
+    @Test
+    @Override
+    public void testViewCount() throws Exception {
+        assertAtLeast(expectedViewCountAtLeast(), extractor().getViewCount());
+    }
+
+    @Test
+    @Override
+    public void testUploadDate() throws Exception {
+        final DateWrapper dateWrapper = extractor().getUploadDate();
+
+        if (expectedUploadDate() == null) {
+            assertNull(dateWrapper);
+        } else {
+            assertNotNull(dateWrapper);
+
+            final LocalDateTime expectedDateTime = LocalDateTime.parse(expectedUploadDate(),
+                    DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"));
+            final LocalDateTime actualDateTime = dateWrapper.offsetDateTime().toLocalDateTime();
+
+            assertEquals(expectedDateTime, actualDateTime);
+        }
+    }
+
+    @Test
+    @Override
+    public void testTextualUploadDate() throws Exception {
+        assertEquals(expectedTextualUploadDate(), extractor().getTextualUploadDate());
+    }
+
+    @Test
+    @Override
+    public void testLikeCount() throws Exception {
+        if (expectedLikeCountAtLeast() == -1) {
+            assertEquals(-1, extractor().getLikeCount());
+        } else {
+            assertAtLeast(expectedLikeCountAtLeast(), extractor().getLikeCount());
+        }
+    }
+
+    @Test
+    @Override
+    public void testDislikeCount() throws Exception {
+        if (expectedDislikeCountAtLeast() == -1) {
+            assertEquals(-1, extractor().getDislikeCount());
+        } else {
+            assertAtLeast(expectedDislikeCountAtLeast(), extractor().getDislikeCount());
+        }
+    }
+
+    @Test
+    @Override
+    public void testRelatedStreams() throws Exception {
+        final StreamInfoItemsCollector relatedStreams = extractor().getRelatedStreams();
+
+        if (expectedHasRelatedStreams()) {
+            assertNotNull(relatedStreams);
+            defaultTestListOfItems(extractor().getService(), relatedStreams.getItems(),
+                    relatedStreams.getErrors());
+        } else {
+            assertNull(relatedStreams);
+        }
+    }
+
+    @Test
+    @Override
+    public void testAgeLimit() throws Exception {
+        assertEquals(expectedAgeLimit(), extractor().getAgeLimit());
+    }
+
+    @Test
+    @Override
+    public void testErrorMessage() throws Exception {
+        assertEquals(expectedErrorMessage(), extractor().getErrorMessage());
+    }
+
+    @Test
+    @Override
+    public void testVideoStreams() throws Exception {
+        final List<VideoStream> videoStreams = extractor().getVideoStreams();
+        final List<VideoStream> videoOnlyStreams = extractor().getVideoOnlyStreams();
+        assertNotNull(videoStreams);
+        assertNotNull(videoOnlyStreams);
+        videoStreams.addAll(videoOnlyStreams);
+
+        if (expectedHasVideoStreams()) {
+            assertFalse(videoStreams.isEmpty());
+
+            for (final VideoStream stream : videoStreams) {
+                assertIsSecureUrl(stream.getUrl());
+                assertFalse(stream.getResolution().isEmpty());
+
+                final int formatId = stream.getFormatId();
+                // see MediaFormat: video stream formats range from 0 to 0x100
+                assertTrue("format id does not fit a video stream: " + formatId,
+                        0 <= formatId && formatId < 0x100);
+            }
+        } else {
+            assertTrue(videoStreams.isEmpty());
+        }
+    }
+
+    @Test
+    @Override
+    public void testAudioStreams() throws Exception {
+        final List<AudioStream> audioStreams = extractor().getAudioStreams();
+        assertNotNull(audioStreams);
+
+        if (expectedHasAudioStreams()) {
+            assertFalse(audioStreams.isEmpty());
+
+            for (final AudioStream stream : audioStreams) {
+                assertIsSecureUrl(stream.getUrl());
+
+                final int formatId = stream.getFormatId();
+                // see MediaFormat: video stream formats range from 0x100 to 0x1000
+                assertTrue("format id does not fit an audio stream: " + formatId,
+                        0x100 <= formatId && formatId < 0x1000);
+            }
+        } else {
+            assertTrue(audioStreams.isEmpty());
+        }
+    }
+
+    @Test
+    @Override
+    public void testSubtitles() throws Exception {
+        final List<SubtitlesStream> subtitles = extractor().getSubtitlesDefault();
+        assertNotNull(subtitles);
+
+        if (expectedHasSubtitles()) {
+            assertFalse(subtitles.isEmpty());
+
+            for (final SubtitlesStream stream : subtitles) {
+                assertIsSecureUrl(stream.getUrl());
+
+                final int formatId = stream.getFormatId();
+                // see MediaFormat: video stream formats range from 0x1000 to 0x10000
+                assertTrue("format id does not fit a subtitles stream: " + formatId,
+                        0x1000 <= formatId && formatId < 0x10000);
+            }
+        } else {
+            assertTrue(subtitles.isEmpty());
+
+            final MediaFormat[] formats = {MediaFormat.VTT, MediaFormat.TTML, MediaFormat.SRT,
+                    MediaFormat.TRANSCRIPT1, MediaFormat.TRANSCRIPT2, MediaFormat.TRANSCRIPT3};
+            for (final MediaFormat format : formats) {
+                final List<SubtitlesStream> formatSubtitles = extractor().getSubtitles(format);
+                assertNotNull(formatSubtitles);
+                assertTrue(formatSubtitles.isEmpty());
+            }
+        }
+    }
+
+    @Override
+    public void testGetDashMpdUrl() throws Exception {
+        final String dashMpdUrl = extractor().getDashMpdUrl();
+        if (expectedDashMpdUrlContains() == null) {
+            assertNotNull(dashMpdUrl);
+            assertTrue(dashMpdUrl.isEmpty());
+        } else {
+            assertIsSecureUrl(dashMpdUrl);
+            assertThat(extractor().getDashMpdUrl(), containsString(expectedDashMpdUrlContains()));
+        }
+    }
+
+    @Test
+    @Override
+    public void testFrames() throws Exception {
+        final List<Frameset> frames = extractor().getFrames();
+        assertNotNull(frames);
+
+        if (expectedHasFrames()) {
+            assertFalse(frames.isEmpty());
+            for (final Frameset f : frames) {
+                for (final String url : f.getUrls()) {
+                    assertIsValidUrl(url);
+                    assertIsSecureUrl(url);
+                }
+            }
+        } else {
+            assertTrue(frames.isEmpty());
+        }
+    }
+
+    @Test
+    @Override
+    public void testHost() throws Exception {
+        assertEquals(expectedHost(), extractor().getHost());
+    }
+
+    @Test
+    @Override
+    public void testPrivacy() throws Exception {
+        assertEquals(expectedPrivacy(), extractor().getPrivacy());
+    }
+
+    @Test
+    @Override
+    public void testCategory() throws Exception {
+        assertEquals(expectedCategory(), extractor().getCategory());
+    }
+
+    @Test
+    @Override
+    public void testLicence() throws Exception {
+        assertEquals(expectedLicence(), extractor().getLicence());
+    }
+
+    @Test
+    @Override
+    public void testLanguageInfo() throws Exception {
+        assertEquals(expectedLanguageInfo(), extractor().getLanguageInfo());
+    }
+
+    @Test
+    @Override
+    public void testTags() throws Exception {
+        assertEqualsOrderIndependent(expectedTags(), extractor().getTags());
+    }
+
+    @Test
+    @Override
+    public void testSupportInfo() throws Exception {
+        assertEquals(expectedSupportInfo(), extractor().getSupportInfo());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java

@@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
-import java.util.Calendar;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -42,7 +41,7 @@ public final class DefaultTests {
                 StreamInfoItem streamInfoItem = (StreamInfoItem) item;
                 assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
 
-//                assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
+                // assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
                 final String uploaderUrl = streamInfoItem.getUploaderUrl();
                 if (!isNullOrEmpty(uploaderUrl)) {
                     assertIsSecureUrl(uploaderUrl);
@@ -54,7 +53,6 @@ public final class DefaultTests {
                 if (!isNullOrEmpty(streamInfoItem.getTextualUploadDate())) {
                     final DateWrapper uploadDate = streamInfoItem.getUploadDate();
                     assertNotNull("No parsed upload date", uploadDate);
-                    assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance()));
                 }
 
             } else if (item instanceof ChannelInfoItem) {

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java

@@ -31,7 +31,7 @@ public class MediaCCCConferenceExtractorTest {
 
         @Test
         public void testGetUrl() throws Exception {
-            assertEquals("https://media.ccc.de/public/conferences/froscon2017", extractor.getUrl());
+            assertEquals("https://media.ccc.de/c/froscon2017", extractor.getUrl());
         }
 
         @Test
@@ -67,7 +67,7 @@ public class MediaCCCConferenceExtractorTest {
 
         @Test
         public void testGetUrl() throws Exception {
-            assertEquals("https://media.ccc.de/public/conferences/oscal19", extractor.getUrl());
+            assertEquals("https://media.ccc.de/c/oscal19", extractor.getUrl());
         }
 
         @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceLinkHandlerFactoryTest.java

@@ -0,0 +1,42 @@
+package org.schabi.newpipe.extractor.services.media_ccc;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
+
+import static org.junit.Assert.assertEquals;
+
+public class MediaCCCConferenceLinkHandlerFactoryTest {
+    private static MediaCCCConferenceLinkHandlerFactory linkHandler;
+
+    @BeforeClass
+    public static void setUp() {
+        linkHandler = new MediaCCCConferenceLinkHandlerFactory();
+        NewPipe.init(DownloaderTestImpl.getInstance());
+    }
+
+    @Test
+    public void getId() throws ParsingException {
+        assertEquals("jh20",
+                linkHandler.fromUrl("https://media.ccc.de/c/jh20#278").getId());
+        assertEquals("jh20",
+                linkHandler.fromUrl("https://media.ccc.de/b/jh20?a=b").getId());
+        assertEquals("jh20",
+                linkHandler.fromUrl("https://api.media.ccc.de/public/conferences/jh20&a=b&b=c").getId());
+    }
+
+    @Test
+    public void getUrl() throws ParsingException {
+        assertEquals("https://media.ccc.de/c/jh20",
+                linkHandler.fromUrl("https://media.ccc.de/c/jh20#278").getUrl());
+        assertEquals("https://media.ccc.de/c/jh20",
+                linkHandler.fromUrl("https://media.ccc.de/b/jh20?a=b").getUrl());
+        assertEquals("https://media.ccc.de/c/jh20",
+                linkHandler.fromUrl("https://api.media.ccc.de/public/conferences/jh20&a=b&b=c").getUrl());
+        assertEquals("https://media.ccc.de/c/jh20",
+                linkHandler.fromId("jh20").getUrl());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java

@@ -1,204 +1,152 @@
 package org.schabi.newpipe.extractor.services.media_ccc;
 
-import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
-import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.stream.StreamType;
 
-import java.text.ParseException;
+import java.util.Arrays;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
 import java.util.List;
 
-import static java.util.Objects.requireNonNull;
+import javax.annotation.Nullable;
+
 import static junit.framework.TestCase.assertEquals;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
 
 /**
  * Test {@link MediaCCCStreamExtractor}
  */
 public class MediaCCCStreamExtractorTest {
-    public static class Gpn18Tmux {
+    private static final String BASE_URL = "https://media.ccc.de/v/";
-        private static MediaCCCStreamExtractor extractor;
+
+    public static class Gpn18Tmux extends DefaultStreamExtractorTest {
+        private static final String ID = "gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht";
+        private static final String URL = BASE_URL + ID;
+        private static StreamExtractor extractor;
 
         @BeforeClass
-        public static void setUpClass() throws Exception {
+        public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
-
+            extractor = MediaCCC.getStreamExtractor(URL);
-            extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht");
             extractor.fetchPage();
         }
 
-        @Test
+        @Override public StreamExtractor extractor() { return extractor; }
-        public void testServiceId() throws Exception {
+        @Override public StreamingService expectedService() { return MediaCCC; }
-            assertEquals(2, extractor.getServiceId());
+        @Override public String expectedName() { return "tmux - Warum ein schwarzes Fenster am Bildschirm reicht"; }
-        }
+        @Override public String expectedId() { return ID; }
+        @Override public String expectedUrlContains() { return URL; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
 
-        @Test
+        @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
-        public void testName() throws Exception {
+        @Override public String expectedUploaderName() { return "gpn18"; }
-            assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName());
+        @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/gpn18"; }
-        }
+        @Override public List<String> expectedDescriptionContains() { return Arrays.asList("SSH-Sessions", "\"Terminal Multiplexer\""); }
+        @Override public long expectedLength() { return 3097; }
+        @Override public long expectedViewCountAtLeast() { return 2380; }
+        @Nullable @Override public String expectedUploadDate() { return "2018-05-11 00:00:00.000"; }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2018-05-11T02:00:00.000+02:00"; }
+        @Override public long expectedLikeCountAtLeast() { return -1; }
+        @Override public long expectedDislikeCountAtLeast() { return -1; }
+        @Override public boolean expectedHasRelatedStreams() { return false; }
+        @Override public boolean expectedHasSubtitles() { return false; }
+        @Override public boolean expectedHasFrames() { return false; }
+        @Override public List<String> expectedTags() { return Arrays.asList("gpn18", "105"); }
 
+        @Override
         @Test
-        public void testId() throws Exception {
+        public void testThumbnailUrl() throws Exception {
-            assertEquals("gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getId());
+            super.testThumbnailUrl();
-        }
-
-        @Test
-        public void testUrl() throws Exception {
-            assertIsSecureUrl(extractor.getUrl());
-            assertEquals("https://media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl());
-        }
-
-        @Test
-        public void testOriginalUrl() throws Exception {
-            assertIsSecureUrl(extractor.getOriginalUrl());
-            assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl());
-        }
-
-        @Test
-        public void testThumbnail() throws Exception {
-            assertIsSecureUrl(extractor.getThumbnailUrl());
             assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl());
         }
 
-        @Test
+        @Override
-        public void testUploaderName() throws Exception {
-            assertEquals("gpn18", extractor.getUploaderName());
-        }
-
-        @Test
-        public void testUploaderUrl() throws Exception {
-            assertIsSecureUrl(extractor.getUploaderUrl());
-            assertEquals("https://media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl());
-        }
-
         @Test
         public void testUploaderAvatarUrl() throws Exception {
-            assertIsSecureUrl(extractor.getUploaderAvatarUrl());
+            super.testUploaderAvatarUrl();
             assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl());
         }
 
+        @Override
         @Test
         public void testVideoStreams() throws Exception {
-            List<VideoStream> videoStreamList = extractor.getVideoStreams();
+            super.testVideoStreams();
-            assertEquals(4, videoStreamList.size());
+            assertEquals(4, extractor.getVideoStreams().size());
-            for (VideoStream stream : videoStreamList) {
-                assertIsSecureUrl(stream.getUrl());
-            }
         }
 
+        @Override
         @Test
         public void testAudioStreams() throws Exception {
-            List<AudioStream> audioStreamList = extractor.getAudioStreams();
+            super.testAudioStreams();
-            assertEquals(2, audioStreamList.size());
+            assertEquals(2, extractor.getAudioStreams().size());
-            for (AudioStream stream : audioStreamList) {
-                assertIsSecureUrl(stream.getUrl());
-            }
-        }
-
-        @Test
-        public void testGetTextualUploadDate() throws ParsingException {
-            Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate());
-        }
-
-        @Test
-        public void testGetUploadDate() throws ParsingException, ParseException {
-            final Calendar instance = Calendar.getInstance();
-            instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11"));
-            assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
         }
     }
 
-    public static class _36c3PrivacyMessaging {
+    public static class _36c3PrivacyMessaging extends DefaultStreamExtractorTest {
-        private static MediaCCCStreamExtractor extractor;
+        private static final String ID = "36c3-10565-what_s_left_for_private_messaging";
+        private static final String URL = BASE_URL + ID;
+        private static StreamExtractor extractor;
 
         @BeforeClass
-        public static void setUpClass() throws Exception {
+        public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging");
+            extractor = MediaCCC.getStreamExtractor(URL);
             extractor.fetchPage();
         }
 
-        @Test
+        @Override public StreamExtractor extractor() { return extractor; }
-        public void testName() throws Exception {
+        @Override public StreamingService expectedService() { return MediaCCC; }
-            assertEquals("What's left for private messaging?", extractor.getName());
+        @Override public String expectedName() { return "What's left for private messaging?"; }
-        }
+        @Override public String expectedId() { return ID; }
+        @Override public String expectedUrlContains() { return URL; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
 
-        @Test
+        @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
-        public void testId() throws Exception {
+        @Override public String expectedUploaderName() { return "36c3"; }
-            assertEquals("36c3-10565-what_s_left_for_private_messaging", extractor.getId());
+        @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/36c3"; }
-        }
+        @Override public List<String> expectedDescriptionContains() { return Arrays.asList("WhatsApp", "Signal"); }
+        @Override public long expectedLength() { return 3603; }
+        @Override public long expectedViewCountAtLeast() { return 2380; }
+        @Nullable @Override public String expectedUploadDate() { return "2020-01-11 00:00:00.000"; }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2020-01-11T01:00:00.000+01:00"; }
+        @Override public long expectedLikeCountAtLeast() { return -1; }
+        @Override public long expectedDislikeCountAtLeast() { return -1; }
+        @Override public boolean expectedHasRelatedStreams() { return false; }
+        @Override public boolean expectedHasSubtitles() { return false; }
+        @Override public boolean expectedHasFrames() { return false; }
+        @Override public List<String> expectedTags() { return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); }
 
+        @Override
         @Test
-        public void testUrl() throws Exception {
+        public void testThumbnailUrl() throws Exception {
-            assertIsSecureUrl(extractor.getUrl());
+            super.testThumbnailUrl();
-            assertEquals("https://media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl());
-        }
-
-        @Test
-        public void testOriginalUrl() throws Exception {
-            assertIsSecureUrl(extractor.getOriginalUrl());
-            assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl());
-        }
-
-        @Test
-        public void testThumbnail() throws Exception {
-            assertIsSecureUrl(extractor.getThumbnailUrl());
             assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl());
         }
 
-        @Test
+        @Override
-        public void testUploaderName() throws Exception {
-            assertEquals("36c3", extractor.getUploaderName());
-        }
-
-        @Test
-        public void testUploaderUrl() throws Exception {
-            assertIsSecureUrl(extractor.getUploaderUrl());
-            assertEquals("https://media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl());
-        }
-
         @Test
         public void testUploaderAvatarUrl() throws Exception {
-            assertIsSecureUrl(extractor.getUploaderAvatarUrl());
+            super.testUploaderAvatarUrl();
             assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl());
         }
 
+        @Override
         @Test
         public void testVideoStreams() throws Exception {
-            List<VideoStream> videoStreamList = extractor.getVideoStreams();
+            super.testVideoStreams();
-            assertEquals(8, videoStreamList.size());
+            assertEquals(8, extractor.getVideoStreams().size());
-            for (VideoStream stream : videoStreamList) {
-                assertIsSecureUrl(stream.getUrl());
-            }
         }
 
+        @Override
         @Test
         public void testAudioStreams() throws Exception {
-            List<AudioStream> audioStreamList = extractor.getAudioStreams();
+            super.testAudioStreams();
-            assertEquals(2, audioStreamList.size());
+            assertEquals(2, extractor.getAudioStreams().size());
-            for (AudioStream stream : audioStreamList) {
-                assertIsSecureUrl(stream.getUrl());
-            }
-        }
-
-        @Test
-        public void testGetTextualUploadDate() throws ParsingException {
-            Assert.assertEquals("2020-01-11T01:00:00.000+01:00", extractor.getTextualUploadDate());
-        }
-
-        @Test
-        public void testGetUploadDate() throws ParsingException, ParseException {
-            final Calendar instance = Calendar.getInstance();
-            instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-11"));
-            assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
         }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamLinkHandlerFactoryTest.java

@@ -0,0 +1,42 @@
+package org.schabi.newpipe.extractor.services.media_ccc;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
+
+import static org.junit.Assert.assertEquals;
+
+public class MediaCCCStreamLinkHandlerFactoryTest {
+    private static MediaCCCStreamLinkHandlerFactory linkHandler;
+
+    @BeforeClass
+    public static void setUp() {
+        linkHandler = new MediaCCCStreamLinkHandlerFactory();
+        NewPipe.init(DownloaderTestImpl.getInstance());
+    }
+
+    @Test
+    public void getId() throws ParsingException {
+        assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
+                linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getId());
+        assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
+                linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020?a=b").getId());
+        assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
+                linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020#3").getId());
+        assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
+                linkHandler.fromUrl("https://api.media.ccc.de/public/events/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020&a=b").getId());
+    }
+
+    @Test
+    public void getUrl() throws ParsingException {
+        assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
+                linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getUrl());
+        assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
+                linkHandler.fromUrl("https://api.media.ccc.de/public/events/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020?b=a&a=b").getUrl());
+        assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
+                linkHandler.fromId("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getUrl());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java

@@ -28,7 +28,7 @@ public class PeertubeAccountExtractorTest {
             // setting instance might break test when running in parallel
             PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
             extractor = (PeertubeAccountExtractor) PeerTube
-                    .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
+                    .getChannelExtractor("https://peertube.mastodon.host/accounts/kde");
             extractor.fetchPage();
         }
 
@@ -53,7 +53,7 @@ public class PeertubeAccountExtractorTest {
 
         @Test
         public void testUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
+            assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getUrl());
         }
 
         @Test
@@ -89,10 +89,9 @@ public class PeertubeAccountExtractorTest {
             assertIsSecureUrl(extractor.getAvatarUrl());
         }
 
-        @Ignore
         @Test
-        public void testBannerUrl() throws ParsingException {
+        public void testBannerUrl() {
-            assertIsSecureUrl(extractor.getBannerUrl());
+            assertNull(extractor.getBannerUrl());
         }
 
         @Test
@@ -115,7 +114,7 @@ public class PeertubeAccountExtractorTest {
             // setting instance might break test when running in parallel
             PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
             extractor = (PeertubeAccountExtractor) PeerTube
-                    .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
+                    .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/booteille");
             extractor.fetchPage();
         }
 
@@ -150,12 +149,12 @@ public class PeertubeAccountExtractorTest {
 
         @Test
         public void testUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
+            assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getUrl());
         }
 
         @Test
         public void testOriginalUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
+            assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getOriginalUrl());
         }
 
         /*//////////////////////////////////////////////////////////////////////////

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java

@@ -28,7 +28,7 @@ public class PeertubeChannelExtractorTest {
             // setting instance might break test when running in parallel
             PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
             extractor = (PeertubeChannelExtractor) PeerTube
-                    .getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa");
+                    .getChannelExtractor("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa");
             extractor.fetchPage();
         }
 
@@ -53,7 +53,7 @@ public class PeertubeChannelExtractorTest {
 
         @Test
         public void testUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl());
+            assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl());
         }
 
         @Test
@@ -104,10 +104,9 @@ public class PeertubeChannelExtractorTest {
             assertIsSecureUrl(extractor.getAvatarUrl());
         }
 
-        @Ignore
         @Test
         public void testBannerUrl() throws ParsingException {
-            assertIsSecureUrl(extractor.getBannerUrl());
+            assertNull(extractor.getBannerUrl());
         }
 
         @Test
@@ -130,7 +129,7 @@ public class PeertubeChannelExtractorTest {
             // setting instance might break test when running in parallel
             PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
             extractor = (PeertubeChannelExtractor) PeerTube
-                    .getChannelExtractor("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457");
+                    .getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457");
             extractor.fetchPage();
         }
 
@@ -165,12 +164,12 @@ public class PeertubeChannelExtractorTest {
 
         @Test
         public void testUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl());
+            assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl());
         }
 
         @Test
         public void testOriginalUrl() throws ParsingException {
-            assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl());
+            assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl());
         }
 
         /*//////////////////////////////////////////////////////////////////////////

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java

@@ -28,20 +28,36 @@ public class PeertubeChannelLinkHandlerFactoryTest {
     @Test
     public void acceptUrlTest() throws ParsingException {
         assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net"));
-        assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"));
+        assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos"));
+        assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net/videos"));
+        assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"));
     }
 
     @Test
-    public void getIdFromUrl() throws ParsingException {
+    public void getId() throws ParsingException {
-        assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
+        assertEquals("accounts/kranti@videos.squat.net",
-        assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
+                linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
-        assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId());
+        assertEquals("accounts/kranti@videos.squat.net",
+                linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
+        assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
+                linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId());
+        assertEquals("accounts/kranti@videos.squat.net",
+                linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net").getId());
+        assertEquals("accounts/kranti@videos.squat.net",
+                linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net/videos").getId());
+        assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
+                linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getId());
     }
 
     @Test
-    public void getUrlFromId() throws ParsingException {
+    public void getUrl() throws ParsingException {
-        assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
+        assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
-        assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl());
+                linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
-        assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("kranti@videos.squat.net").getUrl());
+        assertEquals("https://peertube.mastodon.host/accounts/kranti@videos.squat.net",
+                linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl());
+        assertEquals("https://peertube.mastodon.host/accounts/kranti@videos.squat.net",
+                linkHandler.fromId("kranti@videos.squat.net").getUrl());
+        assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
+                linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java

@@ -1,180 +0,0 @@
-package org.schabi.newpipe.extractor.services.peertube;
-
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.StreamType;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Locale;
-import java.util.TimeZone;
-
-import static java.util.Objects.requireNonNull;
-import static org.junit.Assert.*;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
-
-/**
- * Test for {@link StreamExtractor}
- */
-public class PeertubeStreamExtractorDefaultTest {
-    private static PeertubeStreamExtractor extractor;
-    private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/";
-    private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary";
-
-    @BeforeClass
-    public static void setUp() throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance());
-        // setting instance might break test when running in parallel
-        PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube"));
-        extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
-        extractor.fetchPage();
-    }
-
-    @Test
-    public void testGetUploadDate() throws ParsingException, ParseException {
-        final Calendar instance = Calendar.getInstance();
-        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
-        sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
-        instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z"));
-        assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
-
-    }
-
-    @Test
-    public void testGetInvalidTimeStamp() throws ParsingException {
-        assertTrue(extractor.getTimeStamp() + "",
-                extractor.getTimeStamp() <= 0);
-    }
-
-    @Test
-    public void testGetTitle() throws ParsingException {
-        assertEquals("What is PeerTube?", extractor.getName());
-    }
-
-    @Test
-    public void testGetLargeDescription() throws ParsingException {
-        assertEquals(expectedLargeDescription, extractor.getDescription().getContent());
-    }
-
-    @Test
-    public void testGetEmptyDescription() throws Exception {
-        PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d");
-        extractorEmpty.fetchPage();
-        assertEquals("", extractorEmpty.getDescription().getContent());
-    }
-
-    @Test
-    public void testGetSmallDescription() throws Exception {
-        PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server"));
-        PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea");
-        extractorSmall.fetchPage();
-        assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent());
-    }
-
-    @Test
-    public void testGetUploaderName() throws ParsingException {
-        assertEquals("Framasoft", extractor.getUploaderName());
-    }
-
-    @Test
-    public void testGetUploaderUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getUploaderUrl());
-        assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl());
-    }
-
-    @Test
-    public void testGetUploaderAvatarUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getUploaderAvatarUrl());
-    }
-
-    @Test
-    public void testGetSubChannelName() throws ParsingException {
-        assertEquals("Les vidéos de Framasoft", extractor.getSubChannelName());
-    }
-
-    @Test
-    public void testGetSubChannelUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getSubChannelUrl());
-        assertEquals("https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8", extractor.getSubChannelUrl());
-    }
-
-    @Test
-    public void testGetSubChannelAvatarUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getSubChannelAvatarUrl());
-    }
-
-    @Test
-    public void testGetLength() throws ParsingException {
-        assertEquals(113, extractor.getLength());
-    }
-
-    @Test
-    public void testGetViewCount() throws ParsingException {
-        assertTrue(Long.toString(extractor.getViewCount()),
-                extractor.getViewCount() > 10);
-    }
-
-    @Test
-    public void testGetThumbnailUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getThumbnailUrl());
-    }
-
-    @Test
-    public void testGetVideoStreams() throws IOException, ExtractionException {
-        assertFalse(extractor.getVideoStreams().isEmpty());
-    }
-
-    @Test
-    public void testStreamType() throws ParsingException {
-        assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM);
-    }
-
-    @Ignore
-    @Test
-    public void testGetRelatedVideos() throws ExtractionException, IOException {
-        StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
-        assertFalse(relatedVideos.getItems().isEmpty());
-        assertTrue(relatedVideos.getErrors().isEmpty());
-    }
-
-    @Test
-    public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
-        assertFalse(extractor.getSubtitlesDefault().isEmpty());
-    }
-
-    @Test
-    public void testGetSubtitlesList() throws IOException, ExtractionException {
-        assertFalse(extractor.getSubtitlesDefault().isEmpty());
-    }
-
-    @Test
-    public void testGetAgeLimit() throws ExtractionException, IOException {
-        assertEquals(0, extractor.getAgeLimit());
-        PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://nocensoring.net/videos/embed/dbd8e5e1-c527-49b6-b70c-89101dbb9c08");
-        ageLimit.fetchPage();
-        assertEquals(18, ageLimit.getAgeLimit());
-    }
-
-    @Test
-    public void testGetSupportInformation() throws ExtractionException, IOException {
-        PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37");
-        supportInfoExtractor.fetchPage();
-        assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo());
-    }
-
-    @Test
-    public void testGetLanguageInformation() throws ParsingException {
-        assertEquals(new Locale("en"), extractor.getLanguageInfo());
-    }
-}

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java

@@ -0,0 +1,172 @@
+package org.schabi.newpipe.extractor.services.peertube;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamType;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import javax.annotation.Nullable;
+
+import static org.junit.Assert.assertEquals;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+
+public class PeertubeStreamExtractorTest {
+    private static final String BASE_URL = "/videos/watch/";
+
+    public static class WhatIsPeertube extends DefaultStreamExtractorTest {
+        private static final String ID = "9c9de5e8-0a1e-484a-b099-e80766180a6d";
+        private static final String INSTANCE = "https://framatube.org";
+        private static final int TIMESTAMP_MINUTE = 1;
+        private static final int TIMESTAMP_SECOND = 21;
+        private static final String URL = INSTANCE + BASE_URL + ID + "?start=" + TIMESTAMP_MINUTE + "m" + TIMESTAMP_SECOND + "s";
+        private static StreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            // setting instance might break test when running in parallel (!)
+            PeerTube.setInstance(new PeertubeInstance(INSTANCE, "FramaTube"));
+            extractor = PeerTube.getStreamExtractor(URL);
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetLanguageInformation() throws ParsingException {
+            assertEquals(new Locale("en"), extractor.getLanguageInfo());
+        }
+
+        @Override public StreamExtractor extractor() { return extractor; }
+        @Override public StreamingService expectedService() { return PeerTube; }
+        @Override public String expectedName() { return "What is PeerTube?"; }
+        @Override public String expectedId() { return ID; }
+        @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
+
+        @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
+        @Override public String expectedUploaderName() { return "Framasoft"; }
+        @Override public String expectedUploaderUrl() { return "https://framatube.org/accounts/framasoft@framatube.org"; }
+        @Override public String expectedSubChannelName() { return "Les vidéos de Framasoft"; }
+        @Override public String expectedSubChannelUrl() { return "https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8"; }
+        @Override public List<String> expectedDescriptionContains() { // CRLF line ending
+            return Arrays.asList("**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n"
+                    + "\r\n"
+                    + "**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n"
+                    + "*A decentralized video hosting network, based on free/libre software!*\r\n"
+                    + "\r\n"
+                    + "**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n"
+                    + "*Directed by* Aryeom\r\n"
+                    + "*Assistant* Jehan\r\n"
+                    + "**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n"
+                    + "\r\n"
+                    + "**Sponsored by** [Framasoft](https://framasoft.org)\r\n"
+                    + "\r\n"
+                    + "**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n"
+                    + "\r\n"
+                    + "**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n"
+                    + "\r\n"
+                    + "**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/");
+        }
+        @Override public long expectedLength() { return 113; }
+        @Override public long expectedTimestamp() { return TIMESTAMP_MINUTE*60 + TIMESTAMP_SECOND; }
+        @Override public long expectedViewCountAtLeast() { return 38600; }
+        @Nullable @Override public String expectedUploadDate() { return "2018-10-01 10:52:46.396"; }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; }
+        @Override public long expectedLikeCountAtLeast() { return 120; }
+        @Override public long expectedDislikeCountAtLeast() { return 0; }
+        @Override public boolean expectedHasAudioStreams() { return false; }
+        @Override public boolean expectedHasFrames() { return false; }
+        @Override public String expectedHost() { return "framatube.org"; }
+        @Override public String expectedPrivacy() { return "Public"; }
+        @Override public String expectedCategory() { return "Science & Technology"; }
+        @Override public String expectedLicence() { return "Attribution - Share Alike"; }
+        @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
+        @Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
+    }
+
+    public static class AgeRestricted extends DefaultStreamExtractorTest {
+        private static final String ID = "dbd8e5e1-c527-49b6-b70c-89101dbb9c08";
+        private static final String INSTANCE = "https://nocensoring.net";
+        private static final String URL = INSTANCE + "/videos/embed/" + ID;
+        private static StreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());;
+            // setting instance might break test when running in parallel (!)
+            PeerTube.setInstance(new PeertubeInstance(INSTANCE));
+            extractor = PeerTube.getStreamExtractor(URL);
+            extractor.fetchPage();
+        }
+
+        @Override public StreamExtractor extractor() { return extractor; }
+        @Override public StreamingService expectedService() { return PeerTube; }
+        @Override public String expectedName() { return "Covid-19 ? [Court-métrage]"; }
+        @Override public String expectedId() { return ID; }
+        @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
+
+        @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
+        @Override public String expectedUploaderName() { return "Résilience humaine"; }
+        @Override public String expectedUploaderUrl() { return "https://nocensoring.net/accounts/gmt@nocensoring.net"; }
+        @Override public String expectedSubChannelName() { return "SYSTEM FAILURE Quel à-venir ?"; }
+        @Override public String expectedSubChannelUrl() { return "https://nocensoring.net/video-channels/systemfailure_quel"; }
+        @Override public List<String> expectedDescriptionContains() { // LF line ending
+            return Arrays.asList("2020, le monde est frappé par une pandémie, beaucoup d'humains sont confinés.",
+                    "System Failure Quel à-venir ? - Covid-19   / 2020");
+        }
+        @Override public long expectedLength() { return 667; }
+        @Override public long expectedViewCountAtLeast() { return 138; }
+        @Nullable @Override public String expectedUploadDate() { return "2020-05-14 17:24:35.580"; }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2020-05-14T17:24:35.580Z"; }
+        @Override public long expectedLikeCountAtLeast() { return 1; }
+        @Override public long expectedDislikeCountAtLeast() { return 0; }
+        @Override public int expectedAgeLimit() { return 18; }
+        @Override public boolean expectedHasAudioStreams() { return false; }
+        @Override public boolean expectedHasSubtitles() { return false; }
+        @Override public boolean expectedHasFrames() { return false; }
+        @Override public String expectedHost() { return "nocensoring.net"; }
+        @Override public String expectedPrivacy() { return "Public"; }
+        @Override public String expectedCategory() { return "Art"; }
+        @Override public String expectedLicence() { return "Attribution"; }
+        @Override public List<String> expectedTags() { return Arrays.asList("Covid-19", "Gérôme-Mary trebor", "Horreur et beauté", "court-métrage", "nue artistique"); }
+    }
+
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        NewPipe.init(DownloaderTestImpl.getInstance());
+        PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server"));
+    }
+
+    @Test
+    public void testGetEmptyDescription() throws Exception {
+        StreamExtractor extractorEmpty = PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d");
+        extractorEmpty.fetchPage();
+        assertEquals("", extractorEmpty.getDescription().getContent());
+    }
+
+    @Test
+    public void testGetSmallDescription() throws Exception {
+        StreamExtractor extractorSmall = PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea");
+        extractorSmall.fetchPage();
+        assertEquals("https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary", extractorSmall.getDescription().getContent());
+    }
+
+    @Test
+    public void testGetSupportInformation() throws ExtractionException, IOException {
+        StreamExtractor supportInfoExtractor = PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37");
+        supportInfoExtractor.fetchPage();
+        assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java

@@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStream
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
 
 /**
  * Test for {@link PeertubeStreamLinkHandlerFactory}
@@ -17,16 +18,34 @@ public class PeertubeStreamLinkHandlerFactoryTest {
     private static PeertubeStreamLinkHandlerFactory linkHandler;
 
     @BeforeClass
-    public static void setUp() throws Exception {
+    public static void setUp() {
+        PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
         linkHandler = PeertubeStreamLinkHandlerFactory.getInstance();
         NewPipe.init(DownloaderTestImpl.getInstance());
     }
 
     @Test
     public void getId() throws Exception {
-        assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId());
+        assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60",
-        assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId());
+                linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId());
-        assertEquals("9c9de5e8-0a1e-484a-b099-e80766180a6d", linkHandler.fromUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d").getId());
+        assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60",
+                linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId());
+        assertEquals("9c9de5e8-0a1e-484a-b099-e80766180a6d",
+                linkHandler.fromUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d").getId());
+        assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60",
+                linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId());
+        assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60",
+                linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId());
+    }
+
+    @Test
+    public void getUrl() throws Exception {
+        assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60",
+                linkHandler.fromId("986aac60-1263-4f73-9ce5-36b18225cb60").getUrl());
+        assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60",
+                linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getUrl());
+        assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60",
+                linkHandler.fromUrl("https://peertube.mastodon.host/videos/embed/986aac60-1263-4f73-9ce5-36b18225cb60").getUrl());
     }
 
 
@@ -35,5 +54,6 @@ public class PeertubeStreamLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60"));
         assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa"));
         assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d"));
+        assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa"));
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java

@@ -8,8 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
-import static org.schabi.newpipe.extractor.ServiceList.*;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
 import static org.schabi.newpipe.extractor.services.DefaultTests.*;
 
 public class PeertubeTrendingExtractorTest {

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java

@@ -41,7 +41,7 @@ public class SoundcloudChannelExtractorTest {
 
         @Test
         public void testName() {
-            assertEquals("LIL UZI VERT", extractor.getName());
+            assertEquals("Lil Uzi Vert", extractor.getName());
         }
 
         @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java

@@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChartsExtractor;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.services.DefaultTests.*;
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -98,7 +98,7 @@ public class SoundcloudPlaylistExtractorTest {
 
         @Test
         public void testUploaderName() {
-            assertTrue(extractor.getUploaderName().contains("LIL UZI VERT"));
+            assertTrue(extractor.getUploaderName().contains("Lil Uzi Vert"));
         }
 
         @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java

@@ -1,163 +0,0 @@
-package org.schabi.newpipe.extractor.services.soundcloud;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.StreamType;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.TimeZone;
-
-import static java.util.Objects.requireNonNull;
-import static org.junit.Assert.*;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
-
-/**
- * Test for {@link StreamExtractor}
- */
-public class SoundcloudStreamExtractorDefaultTest {
-
-    public static class LilUziVertDoWhatIWant {
-        private static SoundcloudStreamExtractor extractor;
-
-        @BeforeClass
-        public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (SoundcloudStreamExtractor) SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
-            extractor.fetchPage();
-        }
-
-        @Test
-        public void testGetInvalidTimeStamp() throws ParsingException {
-            assertTrue(extractor.getTimeStamp() + "",
-                    extractor.getTimeStamp() <= 0);
-        }
-
-        @Test
-        public void testGetValidTimeStamp() throws IOException, ExtractionException {
-            StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69");
-            assertEquals("69", extractor.getTimeStamp() + "");
-        }
-
-        @Test
-        public void testGetTitle() throws ParsingException {
-            assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName());
-        }
-
-        @Test
-        public void testGetDescription() throws ParsingException {
-            assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent());
-        }
-
-        @Test
-        public void testGetUploaderName() throws ParsingException {
-            assertEquals("LIL UZI VERT", extractor.getUploaderName());
-        }
-
-        @Test
-        public void testGetLength() throws ParsingException {
-            assertEquals(175, extractor.getLength());
-        }
-
-        @Test
-        public void testGetViewCount() throws ParsingException {
-            assertTrue(Long.toString(extractor.getViewCount()),
-                    extractor.getViewCount() > 44227978);
-        }
-
-        @Test
-        public void testGetTextualUploadDate() throws ParsingException {
-            Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate());
-        }
-
-        @Test
-        public void testGetUploadDate() throws ParsingException, ParseException {
-            final Calendar instance = Calendar.getInstance();
-            SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000");
-            sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
-            instance.setTime(sdf.parse("2016/07/31 18:18:07 +0000"));
-            assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
-        }
-
-        @Test
-        public void testGetUploaderUrl() throws ParsingException {
-            assertIsSecureUrl(extractor.getUploaderUrl());
-            assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl());
-        }
-
-        @Test
-        public void testGetThumbnailUrl() throws ParsingException {
-            assertIsSecureUrl(extractor.getThumbnailUrl());
-        }
-
-        @Test
-        public void testGetUploaderAvatarUrl() throws ParsingException {
-            assertIsSecureUrl(extractor.getUploaderAvatarUrl());
-        }
-
-        @Test
-        public void testGetAudioStreams() throws IOException, ExtractionException {
-            assertFalse(extractor.getAudioStreams().isEmpty());
-        }
-
-        @Test
-        public void testStreamType() throws ParsingException {
-            assertTrue(extractor.getStreamType() == StreamType.AUDIO_STREAM);
-        }
-
-        @Test
-        public void testGetRelatedVideos() throws ExtractionException, IOException {
-            StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
-            assertFalse(relatedVideos.getItems().isEmpty());
-            assertTrue(relatedVideos.getErrors().isEmpty());
-        }
-
-        @Test
-        public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
-            // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
-            assertTrue(extractor.getSubtitlesDefault().isEmpty());
-        }
-
-        @Test
-        public void testGetSubtitlesList() throws IOException, ExtractionException {
-            // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
-            assertTrue(extractor.getSubtitlesDefault().isEmpty());
-        }
-    }
-
-    public static class ContentNotSupported {
-        @BeforeClass
-        public static void setUp() {
-            NewPipe.init(DownloaderTestImpl.getInstance());
-        }
-
-        @Test(expected = ContentNotSupportedException.class)
-        public void hlsAudioStream() throws Exception {
-            final StreamExtractor extractor =
-                    SoundCloud.getStreamExtractor("https://soundcloud.com/dualipa/cool");
-            extractor.fetchPage();
-            extractor.getAudioStreams();
-        }
-
-        @Test(expected = ContentNotSupportedException.class)
-        public void bothHlsAndOpusAudioStreams() throws Exception {
-            final StreamExtractor extractor =
-                    SoundCloud.getStreamExtractor("https://soundcloud.com/lil-baby-4pf/no-sucker");
-            extractor.fetchPage();
-            extractor.getAudioStreams();
-        }
-    }
-}
-

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java

@@ -0,0 +1,60 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
+import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamType;
+
+import java.util.Arrays;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+
+public class SoundcloudStreamExtractorTest {
+
+    public static class CreativeCommonsPlaysWellWithOthers extends DefaultStreamExtractorTest {
+        private static final String ID = "plays-well-with-others-ep-2-what-do-an-army-of-ants-and-an-online-encyclopedia-have-in-common";
+        private static final String UPLOADER = "https://soundcloud.com/wearecc";
+        private static final int TIMESTAMP = 69;
+        private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
+        private static StreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = SoundCloud.getStreamExtractor(URL);
+            extractor.fetchPage();
+        }
+
+        @Override public StreamExtractor extractor() { return extractor; }
+        @Override public StreamingService expectedService() { return SoundCloud; }
+        @Override public String expectedName() { return "Plays Well with Others, Ep 2: What Do an Army of Ants and an Online Encyclopedia Have in Common?"; }
+        @Override public String expectedId() { return "597253485"; }
+        @Override public String expectedUrlContains() { return UPLOADER + "/" + ID; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
+
+        @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; }
+        @Override public String expectedUploaderName() { return "Creative Commons"; }
+        @Override public String expectedUploaderUrl() { return UPLOADER; }
+        @Override public List<String> expectedDescriptionContains() { return Arrays.asList("Stigmergy is a mechanism of indirect coordination",
+                "All original content in Plays Well with Others is available under a Creative Commons BY license."); }
+        @Override public long expectedLength() { return 1400; }
+        @Override public long expectedTimestamp() { return TIMESTAMP; }
+        @Override public long expectedViewCountAtLeast() { return 27000; }
+        @Nullable @Override public String expectedUploadDate() { return "2019-03-28 13:36:18.000"; }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2019-03-28 13:36:18"; }
+        @Override public long expectedLikeCountAtLeast() { return -1; }
+        @Override public long expectedDislikeCountAtLeast() { return -1; }
+        @Override public boolean expectedHasVideoStreams() { return false; }
+        @Override public boolean expectedHasSubtitles() { return false; }
+        @Override public boolean expectedHasFrames() { return false; }
+    }
+
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java

@@ -53,6 +53,7 @@ public class SoundcloudStreamLinkHandlerFactoryTest {
         assertEquals("309689103", linkHandler.fromUrl("https://soundcloud.com/liluzivert/15-ysl").getId());
         assertEquals("309689082", linkHandler.fromUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko").getId());
         assertEquals("309689035", linkHandler.fromUrl("http://soundcloud.com/liluzivert/15-boring-shit").getId());
+        assertEquals("259273264", linkHandler.fromUrl("https://soundcloud.com/liluzivert/ps-qs-produced-by-don-cannon/").getId());
         assertEquals("294488599", linkHandler.fromUrl("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats").getId());
         assertEquals("294488438", linkHandler.fromUrl("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz").getId());
         assertEquals("294488147", linkHandler.fromUrl("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69").getId());
@@ -60,6 +61,7 @@ public class SoundcloudStreamLinkHandlerFactoryTest {
         assertEquals("294487684", linkHandler.fromUrl("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9").getId());
         assertEquals("294487428", linkHandler.fromUrl("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s").getId());
         assertEquals("294487157", linkHandler.fromUrl("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s").getId());
+        assertEquals("44556776", linkHandler.fromUrl("https://soundcloud.com/kechuspider-sets-1/last-days").getId());
     }
 
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java

@@ -0,0 +1,64 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class YouTubeCommentsLinkHandlerFactoryTest {
+
+    private static YoutubeCommentsLinkHandlerFactory linkHandler;
+
+    @BeforeClass
+    public static void setUp() {
+        NewPipe.init(DownloaderTestImpl.getInstance());
+        linkHandler = YoutubeCommentsLinkHandlerFactory.getInstance();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void getIdWithNullAsUrl() throws ParsingException {
+        linkHandler.fromId(null);
+    }
+
+    @Test
+    public void getIdFromYt() throws ParsingException {
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.youtube.com/watch?v=VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://m.youtube.com/watch?v=VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtube.com/watch?v=VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://WWW.youtube.com/watch?v=VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtu.be/VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtu.be/VM_6n762j6M&t=20").getId());
+    }
+
+    @Test
+    public void testAcceptUrl() throws ParsingException {
+        assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=VM_6n762j6M&t=20"));
+        assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/watch?v=VM_6n762j6M&t=20"));
+        assertTrue(linkHandler.acceptUrl("https://youtube.com/watch?v=VM_6n762j6M&t=20"));
+        assertTrue(linkHandler.acceptUrl("https://youtu.be/VM_6n762j6M&t=20"));
+    }
+
+    @Test
+    public void testDeniesUrl() throws ParsingException {
+        assertFalse(linkHandler.acceptUrl("https://www.you com/watch?v=VM_6n762j6M"));
+        assertFalse(linkHandler.acceptUrl("https://com/watch?v=VM_6n762j6M"));
+        assertFalse(linkHandler.acceptUrl("htt ://com/watch?v=VM_6n762j6M"));
+        assertFalse(linkHandler.acceptUrl("ftp://www.youtube.com/watch?v=VM_6n762j6M"));
+    }
+
+    @Test
+    public void getIdFromInvidious() throws ParsingException {
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.invidio.us/watch?v=VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/watch?v=VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://INVIDIO.US/watch?v=VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/VM_6n762j6M").getId());
+        assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/VM_6n762j6M&t=20").getId());
+    }
+
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -432,190 +432,6 @@ public class YoutubeChannelExtractorTest {
         }
     }
 
-    // this channel has no "Subscribe" button
-    public static class EminemVEVO implements BaseChannelExtractorTest {
-        private static YoutubeChannelExtractor extractor;
-
-        @BeforeClass
-        public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeChannelExtractor) YouTube
-                    .getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
-            extractor.fetchPage();
-        }
-
-        /*//////////////////////////////////////////////////////////////////////////
-        // Extractor
-        //////////////////////////////////////////////////////////////////////////*/
-
-        @Test
-        public void testServiceId() {
-            assertEquals(YouTube.getServiceId(), extractor.getServiceId());
-        }
-
-        @Test
-        public void testName() throws Exception {
-            assertEquals("EminemVEVO", extractor.getName());
-        }
-
-        @Test
-        public void testId() throws Exception {
-            assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId());
-        }
-
-        @Test
-        public void testUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl());
-        }
-
-        @Test
-        public void testOriginalUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl());
-        }
-
-        /*//////////////////////////////////////////////////////////////////////////
-        // ListExtractor
-        //////////////////////////////////////////////////////////////////////////*/
-
-        @Test
-        public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor);
-        }
-
-        @Test
-        public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor);
-        }
-
-         /*//////////////////////////////////////////////////////////////////////////
-         // ChannelExtractor
-         //////////////////////////////////////////////////////////////////////////*/
-
-        @Test
-        public void testDescription() throws Exception {
-            final String description = extractor.getDescription();
-            assertTrue(description, description.contains("Eminem on Vevo"));
-        }
-
-        @Test
-        public void testAvatarUrl() throws Exception {
-            String avatarUrl = extractor.getAvatarUrl();
-            assertIsSecureUrl(avatarUrl);
-            assertTrue(avatarUrl, avatarUrl.contains("yt3"));
-        }
-
-        @Test
-        public void testBannerUrl() throws Exception {
-            String bannerUrl = extractor.getBannerUrl();
-            assertIsSecureUrl(bannerUrl);
-            assertTrue(bannerUrl, bannerUrl.contains("yt3"));
-        }
-
-        @Test
-        public void testFeedUrl() throws Exception {
-            assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl());
-        }
-
-        @Test
-        public void testSubscriberCount() throws Exception {
-            // there is no "Subscribe" button
-            long subscribers = extractor.getSubscriberCount();
-            assertEquals("Wrong subscriber count", -1, subscribers);
-        }
-    }
-
-    /**
-     * Some VEVO channels will redirect to a new page with a new channel id.
-     * <p>
-     * Though, it isn't a simple redirect, but a redirect instruction embed in the response itself, this
-     * test assure that we account for that.
-     */
-    public static class RedirectedChannel implements BaseChannelExtractorTest {
-        private static YoutubeChannelExtractor extractor;
-
-        @BeforeClass
-        public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeChannelExtractor) YouTube
-                    .getChannelExtractor("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ");
-            extractor.fetchPage();
-        }
-
-        /*//////////////////////////////////////////////////////////////////////////
-        // Extractor
-        //////////////////////////////////////////////////////////////////////////*/
-
-        @Test
-        public void testServiceId() {
-            assertEquals(YouTube.getServiceId(), extractor.getServiceId());
-        }
-
-        @Test
-        public void testName() throws Exception {
-            assertEquals("LordiVEVO", extractor.getName());
-        }
-
-        @Test
-        public void testId() throws Exception {
-            assertEquals("UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getId());
-        }
-
-        @Test
-        public void testUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getUrl());
-        }
-
-        @Test
-        public void testOriginalUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ", extractor.getOriginalUrl());
-        }
-
-        /*//////////////////////////////////////////////////////////////////////////
-        // ListExtractor
-        //////////////////////////////////////////////////////////////////////////*/
-
-        @Test
-        public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor);
-        }
-
-        @Test
-        public void testMoreRelatedItems() throws Exception {
-            assertNoMoreItems(extractor);
-        }
-
-         /*//////////////////////////////////////////////////////////////////////////
-         // ChannelExtractor
-         //////////////////////////////////////////////////////////////////////////*/
-
-        @Test
-        public void testDescription() throws Exception {
-            assertEmpty(extractor.getDescription());
-        }
-
-        @Test
-        public void testAvatarUrl() throws Exception {
-            String avatarUrl = extractor.getAvatarUrl();
-            assertIsSecureUrl(avatarUrl);
-            assertTrue(avatarUrl, avatarUrl.contains("yt3"));
-        }
-
-        @Test
-        public void testBannerUrl() throws Exception {
-            assertEmpty(extractor.getBannerUrl());
-        }
-
-        @Test
-        public void testFeedUrl() throws Exception {
-            assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getFeedUrl());
-        }
-
-        @Test
-        public void testSubscriberCount() throws Exception {
-            assertEquals(-1, extractor.getSubscriberCount());
-        }
-    }
-
     public static class RandomChannel implements BaseChannelExtractorTest {
         private static YoutubeChannelExtractor extractor;
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java

@@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -30,6 +31,8 @@ public class YoutubeChannelLinkHandlerFactoryTest {
 
         assertTrue(linkHandler.acceptUrl("https://www.youtube.com/c/creatoracademy"));
 
+        assertTrue(linkHandler.acceptUrl("https://youtube.com/DIMENSI0N"));
+
         assertTrue(linkHandler.acceptUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA"));
         assertTrue(linkHandler.acceptUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
 
@@ -44,6 +47,18 @@ public class YoutubeChannelLinkHandlerFactoryTest {
 
         assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA"));
         assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
+        assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watchismo"));
+
+
+        // do not accept URLs which are not channels
+        assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100"));
+        assertFalse(linkHandler.acceptUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY"));
+        assertFalse(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
+        assertFalse(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1d"));
+        assertFalse(linkHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI"));
+        assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
+        assertFalse(linkHandler.acceptUrl("https://www.youtube.com/?app=desktop&persist_app=1"));
+        assertFalse(linkHandler.acceptUrl("https://m.youtube.com/select_site"));
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java

@@ -6,12 +6,15 @@ import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
-import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
-import java.text.SimpleDateFormat;
+import java.time.format.DateTimeFormatter;
-import java.util.*;
+import java.time.temporal.ChronoUnit;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import static org.junit.Assert.fail;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@@ -23,7 +26,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
 @Ignore("Should be ran manually from time to time, as it's too time consuming.")
 public class YoutubeChannelLocalizationTest {
     private static final boolean DEBUG = true;
-    private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
+    private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
 
     @Test
     public void testAllSupportedLocalizations() throws Exception {
@@ -64,7 +67,7 @@ public class YoutubeChannelLocalizationTest {
                         + "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount();
                 final DateWrapper uploadDate = item.getUploadDate();
                 if (uploadDate != null) {
-                    String dateAsText = dateFormat.format(uploadDate.date().getTime());
+                    String dateAsText = dateTimeFormatter.format(uploadDate.offsetDateTime());
                     debugMessage += "\n:::: " + item.getTextualUploadDate() +
                             "\n:::: " + dateAsText;
                 }
@@ -107,13 +110,13 @@ public class YoutubeChannelLocalizationTest {
                 final DateWrapper currentUploadDate = currentItem.getUploadDate();
 
                 final String referenceDateString = referenceUploadDate == null ? "null" :
-                        dateFormat.format(referenceUploadDate.date().getTime());
+                        dateTimeFormatter.format(referenceUploadDate.offsetDateTime());
                 final String currentDateString = currentUploadDate == null ? "null" :
-                        dateFormat.format(currentUploadDate.date().getTime());
+                        dateTimeFormatter.format(currentUploadDate.offsetDateTime());
 
                 long difference = -1;
                 if (referenceUploadDate != null && currentUploadDate != null) {
-                    difference = Math.abs(referenceUploadDate.date().getTimeInMillis() - currentUploadDate.date().getTimeInMillis());
+                    difference = ChronoUnit.MILLIS.between(referenceUploadDate.offsetDateTime(), currentUploadDate.offsetDateTime());
                 }
 
                 final boolean areTimeEquals = difference < 5 * 60 * 1000L;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -23,91 +23,133 @@ import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class YoutubeCommentsExtractorTest {
-    private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o";
+    /**
-    private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o";
+     * Test a "normal" YouTube
-    private static YoutubeCommentsExtractor extractorYT;
+     */
-    private static YoutubeCommentsExtractor extractorInvidious;
+    public static class Thomas {
+        private static final String url = "https://www.youtube.com/watch?v=D00Au7k3i6o";
+        private static YoutubeCommentsExtractor extractor;
 
-    @BeforeClass
+        private static final String commentContent = "sub 4 sub";
-    public static void setUp() throws Exception {
-        NewPipe.init(DownloaderTestImpl.getInstance());
-        extractorYT = (YoutubeCommentsExtractor) YouTube
-                .getCommentsExtractor(urlYT);
-        extractorYT.fetchPage();
-        extractorInvidious = (YoutubeCommentsExtractor) YouTube
-                .getCommentsExtractor(urlInvidious);
-    }
 
-    @Test
+        @BeforeClass
-    public void testGetComments() throws IOException, ExtractionException {
+        public static void setUp() throws Exception {
-        assertTrue(getCommentsHelper(extractorYT));
+            NewPipe.init(DownloaderTestImpl.getInstance());
-        assertTrue(getCommentsHelper(extractorInvidious));
+            extractor = (YoutubeCommentsExtractor) YouTube
-    }
+                    .getCommentsExtractor(url);
-
+            extractor.fetchPage();
-    private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
-        InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
-        boolean result = findInComments(comments, "s1ck m3m3");
-
-        while (comments.hasNextPage() && !result) {
-            comments = extractor.getPage(comments.getNextPage());
-            result = findInComments(comments, "s1ck m3m3");
         }
 
-        return result;
+        @Test
-    }
+        public void testGetComments() throws IOException, ExtractionException {
-
+            assertTrue(getCommentsHelper(extractor));
-    @Test
-    public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
-        assertTrue(getCommentsFromCommentsInfoHelper(urlYT));
-        assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious));
-    }
-
-    private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
-        CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
-
-        assertEquals("Comments", commentsInfo.getName());
-        boolean result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
-
-        Page nextPage = commentsInfo.getNextPage();
-        InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
-        while (moreItems.hasNextPage() && !result) {
-            moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
-            result = findInComments(moreItems.getItems(), "s1ck m3m3");
-            nextPage = moreItems.getNextPage();
         }
-        return result;
-    }
 
-    @Test
+        private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
-    public void testGetCommentsAllData() throws IOException, ExtractionException {
+            InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
-        InfoItemsPage<CommentsInfoItem> comments = extractorYT.getInitialPage();
+            boolean result = findInComments(comments, commentContent);
 
-        DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
+            while (comments.hasNextPage() && !result) {
-        for (CommentsInfoItem c : comments.getItems()) {
+                comments = extractor.getPage(comments.getNextPage());
-            assertFalse(Utils.isBlank(c.getUploaderUrl()));
+                result = findInComments(comments, commentContent);
-            assertFalse(Utils.isBlank(c.getUploaderName()));
+            }
-            assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
+
-            assertFalse(Utils.isBlank(c.getCommentId()));
+            return result;
-            assertFalse(Utils.isBlank(c.getCommentText()));
-            assertFalse(Utils.isBlank(c.getName()));
-            assertFalse(Utils.isBlank(c.getTextualUploadDate()));
-            assertNotNull(c.getUploadDate());
-            assertFalse(Utils.isBlank(c.getThumbnailUrl()));
-            assertFalse(Utils.isBlank(c.getUrl()));
-            assertFalse(c.getLikeCount() < 0);
         }
-    }
 
-    private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
+        @Test
-        return findInComments(comments.getItems(), comment);
+        public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
-    }
+            assertTrue(getCommentsFromCommentsInfoHelper(url));
+        }
 
-    private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
+        private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
-        for (CommentsInfoItem c : comments) {
+            final CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
-            if (c.getCommentText().contains(comment)) {
+
-                return true;
+            assertEquals("Comments", commentsInfo.getName());
+            boolean result = findInComments(commentsInfo.getRelatedItems(), commentContent);
+
+            Page nextPage = commentsInfo.getNextPage();
+            InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
+            while (moreItems.hasNextPage() && !result) {
+                moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
+                result = findInComments(moreItems.getItems(), commentContent);
+                nextPage = moreItems.getNextPage();
+            }
+            return result;
+        }
+
+        @Test
+        public void testGetCommentsAllData() throws IOException, ExtractionException {
+            InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
+
+            DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
+            for (CommentsInfoItem c : comments.getItems()) {
+                assertFalse(Utils.isBlank(c.getUploaderUrl()));
+                assertFalse(Utils.isBlank(c.getUploaderName()));
+                assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
+                assertFalse(Utils.isBlank(c.getCommentId()));
+                assertFalse(Utils.isBlank(c.getCommentText()));
+                assertFalse(Utils.isBlank(c.getName()));
+                assertFalse(Utils.isBlank(c.getTextualUploadDate()));
+                assertNotNull(c.getUploadDate());
+                assertFalse(Utils.isBlank(c.getThumbnailUrl()));
+                assertFalse(Utils.isBlank(c.getUrl()));
+                assertFalse(c.getLikeCount() < 0);
             }
         }
-        return false;
+
+        private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
+            return findInComments(comments.getItems(), comment);
+        }
+
+        private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
+            for (CommentsInfoItem c : comments) {
+                if (c.getCommentText().contains(comment)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+    }
+
+    /**
+     * Test a video with an empty comment
+     */
+    public static class EmptyComment {
+        private static YoutubeCommentsExtractor extractor;
+        private final static String url = "https://www.youtube.com/watch?v=VM_6n762j6M";
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (YoutubeCommentsExtractor) YouTube
+                    .getCommentsExtractor(url);
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetCommentsAllData() throws IOException, ExtractionException {
+            final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
+
+            DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
+            for (CommentsInfoItem c : comments.getItems()) {
+                assertFalse(Utils.isBlank(c.getUploaderUrl()));
+                assertFalse(Utils.isBlank(c.getUploaderName()));
+                assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
+                assertFalse(Utils.isBlank(c.getCommentId()));
+                assertFalse(Utils.isBlank(c.getName()));
+                assertFalse(Utils.isBlank(c.getTextualUploadDate()));
+                assertNotNull(c.getUploadDate());
+                assertFalse(Utils.isBlank(c.getThumbnailUrl()));
+                assertFalse(Utils.isBlank(c.getUrl()));
+                assertFalse(c.getLikeCount() < 0);
+                if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text
+                    assertTrue(Utils.isBlank(c.getCommentText()));
+                } else {
+                    assertFalse(Utils.isBlank(c.getCommentText()));
+                }
+            }
+        }
+
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java

@@ -81,6 +81,15 @@ public class YoutubeStreamLinkHandlerFactoryTest {
         assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
         assertEquals("n8X9_MgEdCg", linkHandler.fromUrl("vnd.youtube://n8X9_MgEdCg").getId());
         assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
+        assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://m.youtube.com/watch?v=-cdveCh1kQk)").getId());
+        assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://www.youtube.com/watch?v=-cdveCh1kQk-").getId());
+        assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://WWW.YouTube.com/watch?v=-cdveCh1kQkwhatever").getId());
+        assertEquals("O0EDx9WAelc", linkHandler.fromUrl("HTTPS://www.youtube.com/watch?v=O0EDx9WAelc]").getId());
+        assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://youtu.be/-cdveCh1kQk)hello").getId());
+        assertEquals("OGS7c0-CmRs", linkHandler.fromUrl("https://YouTu.be/OGS7c0-CmRswhatever)").getId());
+        assertEquals("-cdveCh1kQk", linkHandler.fromUrl("HTTPS://youtu.be/-cdveCh1kQk)").getId());
+        assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("https://www.youtube.com/shorts/IOS2fqxwYbAhi").getId());
+        assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/shorts/IOS2fqxwYbA").getId());
     }
 
     @Test
@@ -101,6 +110,7 @@ public class YoutubeStreamLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
         assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI"));
         assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
+        assertTrue(linkHandler.acceptUrl("https://www.youtube.com/shorts/IOS2fqxwYbA"));
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java

@@ -11,12 +11,16 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
 
 import java.io.ByteArrayInputStream;
-import java.io.File;
 import java.io.FileInputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.schabi.newpipe.FileUtils.resolveTestResource;
 
 /**
  * Test for {@link YoutubeSubscriptionExtractor}
@@ -34,54 +38,48 @@ public class YoutubeSubscriptionExtractorTest {
 
     @Test
     public void testFromInputStream() throws Exception {
-        File testFile = new File("extractor/src/test/resources/youtube_export_test.xml");
+        final List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(
-        if (!testFile.exists()) testFile = new File("src/test/resources/youtube_export_test.xml");
+                new FileInputStream(resolveTestResource("youtube_takeout_import_test.json")));
+        assertEquals(7, subscriptionItems.size());
 
-        List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(new FileInputStream(testFile));
+        for (final SubscriptionItem item : subscriptionItems) {
-        assertTrue("List doesn't have exactly 8 items (had " + subscriptionItems.size() + ")", subscriptionItems.size() == 8);
-
-        for (SubscriptionItem item : subscriptionItems) {
             assertNotNull(item.getName());
             assertNotNull(item.getUrl());
             assertTrue(urlHandler.acceptUrl(item.getUrl()));
-            assertFalse(item.getServiceId() == -1);
+            assertEquals(ServiceList.YouTube.getServiceId(), item.getServiceId());
         }
     }
 
     @Test
     public void testEmptySourceException() throws Exception {
-        String emptySource = "<opml version=\"1.1\"><body>" +
+        final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
-                "<outline text=\"Testing\" title=\"123\" />" +
+                new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
-                "</body></opml>";
-
-        List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8")));
         assertTrue(items.isEmpty());
     }
 
     @Test
     public void testSubscriptionWithEmptyTitleInSource() throws Exception {
-        String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA";
+        final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
-        String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
+        final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
-                "<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_id=" + channelId + "\" />" +
+                new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
-                "</outline></body></opml>";
 
-        List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
+        assertEquals(1, items.size());
-        assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1);
+        assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
-        assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty());
+        assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", items.get(0).getUrl());
-        assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId));
+        assertEquals("", items.get(0).getName());
     }
 
     @Test
     public void testSubscriptionWithInvalidUrlInSource() throws Exception {
-        String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
+        final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
-                "<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/>" +
+                "{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
-                "<outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/>" +
+        final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
-                "<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"\"/>" +
+                new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
-                "<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"\"/>" +
-                "</outline></body></opml>";
 
-        List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
+        assertEquals(1, items.size());
-        assertTrue(items.isEmpty());
+        assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
+        assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", items.get(0).getUrl());
+        assertEquals("name2", items.get(0).getName());
     }
 
     @Test
@@ -89,26 +87,26 @@ public class YoutubeSubscriptionExtractorTest {
         List<String> invalidList = Arrays.asList(
                 "<xml><notvalid></notvalid></xml>",
                 "<opml><notvalid></notvalid></opml>",
-                "<opml><body></body></opml>",
+                "{\"a\":\"b\"}",
+                "[{}]",
+                "[\"\", 5]",
+                "[{\"snippet\":{\"title\":\"name\"}}]",
+                "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"}}}]",
                 "",
-                null,
                 "\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
                 "gibberish");
 
         for (String invalidContent : invalidList) {
             try {
-                if (invalidContent != null) {
+                byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
-                    byte[] bytes = invalidContent.getBytes("UTF-8");
+                subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
-                    subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
+                fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
-                    fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
+            } catch (final Exception e) {
-                } else {
+                boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException;
-                    subscriptionExtractor.fromInputStream(null);
+                if (!correctType) {
-                    fail("Extracting from null String didn't throw an exception");
+                    e.printStackTrace();
                 }
-            } catch (Exception e) {
+                assertTrue(e.getClass().getSimpleName() + " is not InvalidSourceException", correctType);
-                // System.out.println(" -> " + e);
-                boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;
-                assertTrue("\"" + e.getClass().getSimpleName() + "\" is not the expected exception", isExpectedException);
             }
         }
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java

@@ -153,8 +153,8 @@ public class YoutubeMusicSearchExtractorTest {
 
     public static class CorrectedSearch extends DefaultSearchExtractorTest {
         private static SearchExtractor extractor;
-        private static final String QUERY = "duo lipa";
+        private static final String QUERY = "nocopyrigh sounds";
-        private static final String EXPECTED_SUGGESTION = "dua lipa";
+        private static final String EXPECTED_SUGGESTION = "nocopyrightsounds";
 
         @BeforeClass
         public static void setUp() throws Exception {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -1,144 +1,53 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
-import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.stream.StreamType;
 
-import java.io.IOException;
+import java.util.Arrays;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
 import java.util.List;
 
-import static java.util.Objects.requireNonNull;
+import javax.annotation.Nullable;
-import static org.junit.Assert.*;
+
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
-/**
+public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtractorTest {
- * Test for {@link YoutubeStreamLinkHandlerFactory}
+    private static final String ID = "MmBeUZqv1QA";
- */
+    private static final int TIMESTAMP = 196;
-public class YoutubeStreamExtractorAgeRestrictedTest {
+    private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
-    public static final String HTTPS = "https://";
+    private static StreamExtractor extractor;
-    private static YoutubeStreamExtractor extractor;
 
     @BeforeClass
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
-        extractor = (YoutubeStreamExtractor) YouTube
+        extractor = YouTube.getStreamExtractor(URL);
-                .getStreamExtractor("https://www.youtube.com/watch?v=MmBeUZqv1QA");
         extractor.fetchPage();
     }
 
-    @Test
+    @Override public StreamExtractor extractor() { return extractor; }
-    public void testGetInvalidTimeStamp() throws ParsingException {
+    @Override public StreamingService expectedService() { return YouTube; }
-        assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0);
+    @Override public String expectedName() { return "FINGERING PORNSTARS @ AVN Expo 2017 In Las Vegas!"; }
-    }
+    @Override public String expectedId() { return ID; }
+    @Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; }
+    @Override public String expectedOriginalUrlContains() { return URL; }
 
-    @Test
+    @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
-    public void testGetValidTimeStamp() throws IOException, ExtractionException {
+    @Override public String expectedUploaderName() { return "EpicFiveTV"; }
-        StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
+    @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCuPUHlLP5POZphOIrjrNxiw"; }
-        assertEquals(extractor.getTimeStamp() + "", "174");
+    @Override public List<String> expectedDescriptionContains() { return Arrays.asList("http://instagram.com/Ruben_Sole", "AVN"); }
-        extractor = YouTube.getStreamExtractor("https://youtube.com/embed/FmG385_uUys?start=174");
+    @Override public long expectedLength() { return 1790; }
-        assertEquals(extractor.getTimeStamp() + "", "174");
+    @Override public long expectedTimestamp() { return TIMESTAMP; }
-    }
+    @Override public long expectedViewCountAtLeast() { return 28500000; }
-
+    @Nullable @Override public String expectedUploadDate() { return "2017-01-25 00:00:00.000"; }
-    @Test
+    @Nullable @Override public String expectedTextualUploadDate() { return "2017-01-25"; }
-    public void testGetAgeLimit() throws ParsingException {
+    @Override public long expectedLikeCountAtLeast() { return 149000; }
-        assertEquals(18, extractor.getAgeLimit());
+    @Override public long expectedDislikeCountAtLeast() { return 38000; }
-    }
+    @Override public boolean expectedHasRelatedStreams() { return false; } // no related videos (!)
-
+    @Override public int expectedAgeLimit() { return 18; }
-    @Test
+    @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
-    public void testGetName() throws ParsingException {
+    @Override public boolean expectedHasSubtitles() { return false; }
-        assertNotNull("name is null", extractor.getName());
-        assertFalse("name is empty", extractor.getName().isEmpty());
-    }
-
-    @Test
-    public void testGetDescription() throws ParsingException {
-        assertNotNull(extractor.getDescription());
-        assertFalse(extractor.getDescription().getContent().isEmpty());
-    }
-
-    @Test
-    public void testGetUploaderName() throws ParsingException {
-        assertNotNull(extractor.getUploaderName());
-        assertFalse(extractor.getUploaderName().isEmpty());
-    }
-
-    @Test
-    public void testGetLength() throws ParsingException {
-        assertEquals(1790, extractor.getLength());
-    }
-
-    @Test
-    public void testGetViews() throws ParsingException {
-        assertTrue(extractor.getViewCount() > 0);
-    }
-
-    @Test
-    public void testGetTextualUploadDate() throws ParsingException {
-        assertEquals("2017-01-25", extractor.getTextualUploadDate());
-    }
-
-    @Test
-    public void testGetUploadDate() throws ParsingException, ParseException {
-        final Calendar instance = Calendar.getInstance();
-        instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2017-01-25"));
-        assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
-    }
-
-    @Test
-    public void testGetThumbnailUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getThumbnailUrl());
-    }
-
-    @Test
-    public void testGetUploaderAvatarUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getUploaderAvatarUrl());
-    }
-
-    @Test
-    public void testGetAudioStreams() throws IOException, ExtractionException {
-        // audio streams are not always necessary
-        assertFalse(extractor.getAudioStreams().isEmpty());
-    }
-
-    @Test
-    public void testGetVideoStreams() throws IOException, ExtractionException {
-        List<VideoStream> streams = new ArrayList<>();
-        streams.addAll(extractor.getVideoStreams());
-        streams.addAll(extractor.getVideoOnlyStreams());
-
-        assertTrue(Integer.toString(streams.size()), streams.size() > 0);
-        for (VideoStream s : streams) {
-            assertTrue(s.getUrl(),
-                    s.getUrl().contains(HTTPS));
-            assertTrue(s.resolution.length() > 0);
-            assertTrue(Integer.toString(s.getFormatId()),
-                    0 <= s.getFormatId() && s.getFormatId() <= 0x100);
-        }
-    }
-
-
-    @Test
-    public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
-        // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
-        assertTrue(extractor.getSubtitlesDefault().isEmpty());
-    }
-
-    @Test
-    public void testGetSubtitlesList() throws IOException, ExtractionException {
-        // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
-        assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
-    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -1,134 +1,54 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.stream.StreamType;
 
-import java.io.IOException;
+import java.util.Arrays;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
 import java.util.List;
 
-import static java.util.Objects.requireNonNull;
+import javax.annotation.Nullable;
-import static org.junit.Assert.*;
+
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 /**
  * Test for {@link YoutubeStreamLinkHandlerFactory}
  */
-public class YoutubeStreamExtractorControversialTest {
+public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
-    private static YoutubeStreamExtractor extractor;
+    private static final String ID = "T4XJQO3qol8";
+    private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
+    private static StreamExtractor extractor;
 
     @BeforeClass
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
-        extractor = (YoutubeStreamExtractor) YouTube
+        extractor = YouTube.getStreamExtractor(URL);
-                .getStreamExtractor("https://www.youtube.com/watch?v=T4XJQO3qol8");
         extractor.fetchPage();
     }
 
-    @Test
+    @Override public StreamExtractor extractor() { return extractor; }
-    public void testGetInvalidTimeStamp() throws ParsingException {
+    @Override public StreamingService expectedService() { return YouTube; }
-        assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0);
+    @Override public String expectedName() { return "Burning Everyone's Koran"; }
-    }
+    @Override public String expectedId() { return ID; }
+    @Override public String expectedUrlContains() { return URL; }
+    @Override public String expectedOriginalUrlContains() { return URL; }
 
-    @Test
+    @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
-    public void testGetValidTimeStamp() throws IOException, ExtractionException {
+    @Override public String expectedUploaderName() { return "Amazing Atheist"; }
-        StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
+    @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCjNxszyFPasDdRoD9J6X-sw"; }
-        assertEquals(extractor.getTimeStamp() + "", "174");
+    @Override public List<String> expectedDescriptionContains() {
-    }
+        return Arrays.asList("http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html",
-
+                "freedom");
-    @Test
-    @Ignore
-    public void testGetAgeLimit() throws ParsingException {
-        assertEquals(18, extractor.getAgeLimit());
-    }
-
-    @Test
-    public void testGetName() throws ParsingException {
-        assertNotNull("name is null", extractor.getName());
-        assertFalse("name is empty", extractor.getName().isEmpty());
-    }
-
-    @Test
-    public void testGetDescription() throws ParsingException {
-        assertNotNull(extractor.getDescription());
-        assertFalse(extractor.getDescription().getContent().isEmpty());
-    }
-
-    @Test
-    public void testGetUploaderName() throws ParsingException {
-        assertNotNull(extractor.getUploaderName());
-        assertFalse(extractor.getUploaderName().isEmpty());
-    }
-
-    @Test
-    public void testGetLength() throws ParsingException {
-        assertEquals(219, extractor.getLength());
-    }
-
-    @Test
-    public void testGetViews() throws ParsingException {
-        assertTrue(extractor.getViewCount() > 0);
-    }
-
-    @Test
-    public void testGetTextualUploadDate() throws ParsingException {
-        assertEquals("2010-09-09", extractor.getTextualUploadDate());
-    }
-
-    @Test
-    public void testGetUploadDate() throws ParsingException, ParseException {
-        final Calendar instance = Calendar.getInstance();
-        instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2010-09-09"));
-        assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
-    }
-
-    @Test
-    public void testGetThumbnailUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getThumbnailUrl());
-    }
-
-    @Test
-    public void testGetUploaderAvatarUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getUploaderAvatarUrl());
-    }
-
-    @Test
-    public void testGetAudioStreams() throws IOException, ExtractionException {
-        // audio streams are not always necessary
-        assertFalse(extractor.getAudioStreams().isEmpty());
-    }
-
-    @Test
-    public void testGetVideoStreams() throws IOException, ExtractionException {
-        List<VideoStream> streams = new ArrayList<>();
-        streams.addAll(extractor.getVideoStreams());
-        streams.addAll(extractor.getVideoOnlyStreams());
-        assertTrue(streams.size() > 0);
-    }
-
-    @Test
-    public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
-        // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions
-        assertFalse(extractor.getSubtitlesDefault().isEmpty());
-    }
-
-    @Test
-    public void testGetSubtitlesList() throws IOException, ExtractionException {
-        // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions
-        assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
     }
+    @Override public long expectedLength() { return 219; }
+    @Override public long expectedViewCountAtLeast() { return 285000; }
+    @Nullable @Override public String expectedUploadDate() { return "2010-09-09 00:00:00.000"; }
+    @Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; }
+    @Override public long expectedLikeCountAtLeast() { return 13300; }
+    @Override public long expectedDislikeCountAtLeast() { return 2600; }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -1,35 +1,22 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
-import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.ExtractorAsserts;
-import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.stream.Frameset;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.VideoStream;
-import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.text.ParseException;
+import java.util.Arrays;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
 import java.util.List;
 
-import static java.util.Objects.requireNonNull;
+import javax.annotation.Nullable;
-import static org.junit.Assert.assertEquals;
+
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 /*
@@ -51,11 +38,8 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
  * You should have received a copy of the GNU General Public License
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
-
-/**
- * Test for {@link StreamExtractor}
- */
 public class YoutubeStreamExtractorDefaultTest {
+    static final String BASE_URL = "https://www.youtube.com/watch?v=";
 
     public static class NotAvailable {
         @BeforeClass
@@ -66,266 +50,120 @@ public class YoutubeStreamExtractorDefaultTest {
         @Test(expected = ContentNotAvailableException.class)
         public void nonExistentFetch() throws Exception {
             final StreamExtractor extractor =
-                    YouTube.getStreamExtractor("https://www.youtube.com/watch?v=don-t-exist");
+                    YouTube.getStreamExtractor(BASE_URL + "don-t-exist");
             extractor.fetchPage();
         }
 
         @Test(expected = ParsingException.class)
         public void invalidId() throws Exception {
             final StreamExtractor extractor =
-                    YouTube.getStreamExtractor("https://www.youtube.com/watch?v=INVALID_ID_INVALID_ID");
+                    YouTube.getStreamExtractor(BASE_URL + "INVALID_ID_INVALID_ID");
             extractor.fetchPage();
         }
     }
 
-    /**
+    public static class DescriptionTestPewdiepie extends DefaultStreamExtractorTest {
-     * Test for {@link StreamExtractor}
+        private static final String ID = "7PIMiDcwNvc";
-     */
+        private static final int TIMESTAMP = 17;
-    public static class AdeleHello {
+        private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP;
-        private static YoutubeStreamExtractor extractor;
+        private static StreamExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeStreamExtractor) YouTube
+            extractor = YouTube.getStreamExtractor(URL);
-                    .getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
             extractor.fetchPage();
         }
 
-        @Test
+        @Override public StreamExtractor extractor() { return extractor; }
-        public void testGetInvalidTimeStamp() throws ParsingException {
+        @Override public StreamingService expectedService() { return YouTube; }
-            assertTrue(extractor.getTimeStamp() + "",
+        @Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; }
-                    extractor.getTimeStamp() <= 0);
+        @Override public String expectedId() { return ID; }
-        }
+        @Override public String expectedUrlContains() { return BASE_URL + ID; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
 
-        @Test
+        @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
-        public void testGetValidTimeStamp() throws ExtractionException {
+        @Override public String expectedUploaderName() { return "PewDiePie"; }
-            StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
+        @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; }
-            assertEquals(extractor.getTimeStamp() + "", "174");
+        @Override public List<String> expectedDescriptionContains() {
-        }
+            return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA",
-
+                    "https://www.handcraftpictures.com/");
-        @Test
-        public void testGetTitle() throws ParsingException {
-            assertFalse(extractor.getName().isEmpty());
-        }
-
-        @Test
-        public void testGetDescription() throws ParsingException {
-            assertNotNull(extractor.getDescription());
-            assertFalse(extractor.getDescription().getContent().isEmpty());
-        }
-
-        @Test
-        public void testGetFullLinksInDescription() throws ParsingException {
-            assertTrue(extractor.getDescription().getContent().contains("http://adele.com"));
-        }
-
-        @Test
-        public void testGetUploaderName() throws ParsingException {
-            assertNotNull(extractor.getUploaderName());
-            assertFalse(extractor.getUploaderName().isEmpty());
-        }
-
-
-        @Test
-        public void testGetLength() throws ParsingException {
-            assertEquals(367, extractor.getLength());
-        }
-
-        @Test
-        public void testGetViewCount() throws ParsingException {
-            Long count = extractor.getViewCount();
-            assertTrue(Long.toString(count), count >= /* specific to that video */ 1220025784);
-        }
-
-        @Test
-        public void testGetTextualUploadDate() throws ParsingException {
-            Assert.assertEquals("2015-10-22", extractor.getTextualUploadDate());
-        }
-
-        @Test
-        public void testGetUploadDate() throws ParsingException, ParseException {
-            final Calendar instance = Calendar.getInstance();
-            instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2015-10-22"));
-            assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
-        }
-
-        @Test
-        public void testGetUploaderUrl() throws ParsingException {
-            String url = extractor.getUploaderUrl();
-            if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") &&
-                    !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) {
-                fail("Uploader url is neither the music channel one nor the Vevo one");
-            }
-        }
-
-        @Test
-        public void testGetThumbnailUrl() throws ParsingException {
-            assertIsSecureUrl(extractor.getThumbnailUrl());
-        }
-
-        @Test
-        public void testGetUploaderAvatarUrl() throws ParsingException {
-            assertIsSecureUrl(extractor.getUploaderAvatarUrl());
-        }
-
-        @Test
-        public void testGetAudioStreams() throws ExtractionException {
-            assertFalse(extractor.getAudioStreams().isEmpty());
-        }
-
-        @Test
-        public void testGetVideoStreams() throws ExtractionException {
-            for (VideoStream s : extractor.getVideoStreams()) {
-                assertIsSecureUrl(s.url);
-                assertTrue(s.resolution.length() > 0);
-                assertTrue(Integer.toString(s.getFormatId()),
-                        0 <= s.getFormatId() && s.getFormatId() <= 0x100);
-            }
-        }
-
-        @Test
-        public void testStreamType() throws ParsingException {
-            assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM);
-        }
-
-        @Test
-        public void testGetDashMpd() throws ParsingException {
-            // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class.
-            assertTrue(extractor.getDashMpdUrl(),
-                    extractor.getDashMpdUrl() != null && extractor.getDashMpdUrl().isEmpty());
-        }
-
-        @Test
-        public void testGetRelatedVideos() throws ExtractionException {
-            StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
-            Utils.printErrors(relatedVideos.getErrors());
-            assertFalse(relatedVideos.getItems().isEmpty());
-            assertTrue(relatedVideos.getErrors().isEmpty());
-        }
-
-        @Test
-        public void testGetSubtitlesListDefault() {
-            // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
-            assertTrue(extractor.getSubtitlesDefault().isEmpty());
-        }
-
-        @Test
-        public void testGetSubtitlesList() {
-            // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
-            assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
-        }
-
-        @Test
-        public void testGetLikeCount() throws ParsingException {
-            long likeCount = extractor.getLikeCount();
-            assertTrue("" + likeCount, likeCount >= 15000000);
-        }
-
-        @Test
-        public void testGetDislikeCount() throws ParsingException {
-            long dislikeCount = extractor.getDislikeCount();
-            assertTrue("" + dislikeCount, dislikeCount >= 818000);
         }
+        @Override public long expectedLength() { return 381; }
+        @Override public long expectedTimestamp() { return TIMESTAMP; }
+        @Override public long expectedViewCountAtLeast() { return 26682500; }
+        @Nullable @Override public String expectedUploadDate() { return "2019-08-24 00:00:00.000"; }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2019-08-24"; }
+        @Override public long expectedLikeCountAtLeast() { return 5212900; }
+        @Override public long expectedDislikeCountAtLeast() { return 30600; }
     }
 
-    public static class DescriptionTestPewdiepie {
+    public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
-        private static YoutubeStreamExtractor extractor;
+        private static final String ID = "cV5TjZCJkuA";
+        private static final String URL = BASE_URL + ID;
+        private static StreamExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeStreamExtractor) YouTube
+            extractor = YouTube.getStreamExtractor(URL);
-                    .getStreamExtractor("https://www.youtube.com/watch?v=fBc4Q_htqPg");
             extractor.fetchPage();
         }
 
-        @Test
+        @Override public StreamExtractor extractor() { return extractor; }
-        public void testGetDescription() throws ParsingException {
+        @Override public StreamingService expectedService() { return YouTube; }
-            assertNotNull(extractor.getDescription());
+        @Override public String expectedName() { return "This Smartphone Changes Everything..."; }
-            assertFalse(extractor.getDescription().getContent().isEmpty());
+        @Override public String expectedId() { return ID; }
-        }
+        @Override public String expectedUrlContains() { return URL; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
 
-        @Test
+        @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
-        public void testGetFullLinksInDescription() throws ParsingException {
+        @Override public String expectedUploaderName() { return "Unbox Therapy"; }
-            assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
+        @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; }
-            assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
+        @Override public List<String> expectedDescriptionContains() {
-            assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
+            return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
+                    "https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
+                    "https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
+                    "https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34");
         }
+        @Override public long expectedLength() { return 434; }
+        @Override public long expectedViewCountAtLeast() { return 21229200; }
+        @Nullable @Override public String expectedUploadDate() { return "2018-06-19 00:00:00.000"; }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; }
+        @Override public long expectedLikeCountAtLeast() { return 340100; }
+        @Override public long expectedDislikeCountAtLeast() { return 18700; }
     }
 
-    public static class DescriptionTestUnboxing {
+    public static class RatingsDisabledTest extends DefaultStreamExtractorTest {
-        private static YoutubeStreamExtractor extractor;
+        private static final String ID = "HRKu0cvrr_o";
+        private static final int TIMESTAMP = 17;
+        private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP;
+        private static StreamExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeStreamExtractor) YouTube
+            extractor = YouTube.getStreamExtractor(URL);
-                    .getStreamExtractor("https://www.youtube.com/watch?v=cV5TjZCJkuA");
             extractor.fetchPage();
         }
 
-        @Test
+        @Override public StreamExtractor extractor() { return extractor; }
-        public void testGetDescription() throws ParsingException {
+        @Override public StreamingService expectedService() { return YouTube; }
-            assertNotNull(extractor.getDescription());
+        @Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; }
-            assertFalse(extractor.getDescription().getContent().isEmpty());
+        @Override public String expectedId() { return ID; }
-        }
+        @Override public String expectedUrlContains() { return BASE_URL + ID; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
 
-        @Test
+        @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
-        public void testGetFullLinksInDescription() throws ParsingException {
+        @Override public String expectedUploaderName() { return "YouTuber PrinceOfFALLEN"; }
-            final String description = extractor.getDescription().getContent();
+        @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCQT2yul0lr6Ie9qNQNmw-sg"; }
-            assertTrue(description.contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
+        @Override public List<String> expectedDescriptionContains() { return Arrays.asList("dislikes", "Alpha", "wrong"); }
-            assertTrue(description.contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
+        @Override public long expectedLength() { return 84; }
-            assertTrue(description.contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
+        @Override public long expectedTimestamp() { return TIMESTAMP; }
-            assertTrue(description.contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
+        @Override public long expectedViewCountAtLeast() { return 190; }
-        }
+        @Nullable @Override public String expectedUploadDate() { return "2019-01-02 00:00:00.000"; }
-    }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; }
-
+        @Override public long expectedLikeCountAtLeast() { return -1; }
-    public static class RatingsDisabledTest {
+        @Override public long expectedDislikeCountAtLeast() { return -1; }
-        private static YoutubeStreamExtractor extractor;
-
-        @BeforeClass
-        public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeStreamExtractor) YouTube
-                    .getStreamExtractor("https://www.youtube.com/watch?v=HRKu0cvrr_o");
-            extractor.fetchPage();
-        }
-
-        @Test
-        public void testGetLikeCount() throws ParsingException {
-            assertEquals(-1, extractor.getLikeCount());
-        }
-
-        @Test
-        public void testGetDislikeCount() throws ParsingException {
-            assertEquals(-1, extractor.getDislikeCount());
-        }
-
-    }
-
-    public static class FramesTest {
-        private static YoutubeStreamExtractor extractor;
-
-        @BeforeClass
-        public static void setUp() throws Exception {
-            NewPipe.init(DownloaderTestImpl.getInstance());
-            extractor = (YoutubeStreamExtractor) YouTube
-                    .getStreamExtractor("https://www.youtube.com/watch?v=HoK9shIJ2xQ");
-            extractor.fetchPage();
-        }
-
-        @Test
-        public void testGetFrames() throws ExtractionException {
-            final List<Frameset> frames = extractor.getFrames();
-            assertNotNull(frames);
-            assertFalse(frames.isEmpty());
-            for (final Frameset f : frames) {
-                for (final String url : f.getUrls()) {
-                    ExtractorAsserts.assertIsValidUrl(url);
-                    ExtractorAsserts.assertIsSecureUrl(url);
-                }
-            }
-        }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -1,139 +1,55 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
-import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.VideoStream;
-import org.schabi.newpipe.extractor.utils.Utils;
 
-import static org.junit.Assert.assertEquals;
+import java.util.Arrays;
-import static org.junit.Assert.assertFalse;
+import java.util.List;
-import static org.junit.Assert.assertNotNull;
+
-import static org.junit.Assert.assertNull;
+import javax.annotation.Nullable;
-import static org.junit.Assert.assertSame;
+
-import static org.junit.Assert.assertTrue;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
-public class YoutubeStreamExtractorLivestreamTest {
+public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractorTest {
-    private static YoutubeStreamExtractor extractor;
+    private static final String ID = "5qap5aO4i9A";
+    private static final int TIMESTAMP = 1737;
+    private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
+    private static StreamExtractor extractor;
 
     @BeforeClass
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
-        extractor = (YoutubeStreamExtractor) YouTube
+        extractor = YouTube.getStreamExtractor(URL);
-                .getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A");
         extractor.fetchPage();
     }
 
-    @Test
+    @Override public StreamExtractor extractor() { return extractor; }
-    public void testGetInvalidTimeStamp() throws ParsingException {
+    @Override public StreamingService expectedService() { return YouTube; }
-        assertTrue(extractor.getTimeStamp() + "",
+    @Override public String expectedName() { return "lofi hip hop radio - beats to relax/study to"; }
-                extractor.getTimeStamp() <= 0);
+    @Override public String expectedId() { return ID; }
-    }
+    @Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; }
+    @Override public String expectedOriginalUrlContains() { return URL; }
 
-    @Test
+    @Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; }
-    public void testGetTitle() throws ParsingException {
+    @Override public String expectedUploaderName() { return "ChilledCow"; }
-        assertFalse(extractor.getName().isEmpty());
+    @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; }
-    }
+    @Override public List<String> expectedDescriptionContains() {
-
+        return Arrays.asList("https://bit.ly/chilledcow-playlists",
-    @Test
+                "https://bit.ly/chilledcow-submissions");
-    public void testGetDescription() throws ParsingException {
-        assertNotNull(extractor.getDescription());
-        assertFalse(extractor.getDescription().getContent().isEmpty());
-    }
-
-    @Test
-    public void testGetFullLinksInDescription() throws ParsingException {
-        assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists"));
-    }
-
-    @Test
-    public void testGetUploaderName() throws ParsingException {
-        assertNotNull(extractor.getUploaderName());
-        assertFalse(extractor.getUploaderName().isEmpty());
-    }
-
-
-    @Test
-    public void testGetLength() throws ParsingException {
-        assertEquals(0, extractor.getLength());
-    }
-
-    @Test
-    public void testGetViewCount() throws ParsingException {
-        long count = extractor.getViewCount();
-        assertTrue(Long.toString(count), count > -1);
-    }
-
-    @Test
-    public void testGetUploadDate() throws ParsingException {
-        assertNull(extractor.getUploadDate());
-        assertNull(extractor.getTextualUploadDate());
-    }
-
-    @Test
-    public void testGetUploaderUrl() throws ParsingException {
-        assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl());
-    }
-
-    @Test
-    public void testGetThumbnailUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getThumbnailUrl());
-    }
-
-    @Test
-    public void testGetUploaderAvatarUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getUploaderAvatarUrl());
-    }
-
-    @Test
-    public void testGetAudioStreams() throws ExtractionException {
-        assertFalse(extractor.getAudioStreams().isEmpty());
-    }
-
-    @Test
-    public void testGetVideoStreams() throws ExtractionException {
-        for (VideoStream s : extractor.getVideoStreams()) {
-            assertIsSecureUrl(s.url);
-            assertTrue(s.resolution.length() > 0);
-            assertTrue(Integer.toString(s.getFormatId()),
-                    0 <= s.getFormatId() && s.getFormatId() <= 0x100);
-        }
-    }
-
-    @Test
-    public void testStreamType() throws ParsingException {
-        assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM);
-    }
-
-    @Test
-    public void testGetDashMpd() throws ParsingException {
-        assertTrue(extractor.getDashMpdUrl().startsWith("https://manifest.googlevideo.com/api/manifest/dash/"));
-    }
-
-    @Test
-    public void testGetRelatedVideos() throws ExtractionException {
-        StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
-        Utils.printErrors(relatedVideos.getErrors());
-        assertFalse(relatedVideos.getItems().isEmpty());
-        assertTrue(relatedVideos.getErrors().isEmpty());
-    }
-
-    @Test
-    public void testGetSubtitlesListDefault() {
-        assertTrue(extractor.getSubtitlesDefault().isEmpty());
-    }
-
-    @Test
-    public void testGetSubtitlesList() {
-        assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
     }
+    @Override public long expectedLength() { return 0; }
+    @Override public long expectedTimestamp() { return TIMESTAMP; }
+    @Override public long expectedViewCountAtLeast() { return 0; }
+    @Nullable @Override public String expectedUploadDate() { return "2020-02-22 00:00:00.000"; }
+    @Nullable @Override public String expectedTextualUploadDate() { return "2020-02-22"; }
+    @Override public long expectedLikeCountAtLeast() { return 825000; }
+    @Override public long expectedDislikeCountAtLeast() { return 15600; }
+    @Override public boolean expectedHasSubtitles() { return false; }
+    @Nullable @Override public String expectedDashMpdUrlContains() { return "https://manifest.googlevideo.com/api/manifest/dash/"; }
+    @Override public boolean expectedHasFrames() { return false; }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java

@@ -1,161 +1,50 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
-import org.junit.Assert;
 import org.junit.BeforeClass;
-import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.AudioStream;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.VideoStream;
-import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.text.ParseException;
+import java.util.Arrays;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
 import java.util.List;
 
-import static org.junit.Assert.*;
+import javax.annotation.Nullable;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
-public class YoutubeStreamExtractorUnlistedTest {
+public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
-    private static YoutubeStreamExtractor extractor;
+    static final String ID = "udsB8KnIJTg";
+    static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
+    private static StreamExtractor extractor;
 
     @BeforeClass
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
-        extractor = (YoutubeStreamExtractor) YouTube
+        extractor = YouTube.getStreamExtractor(URL);
-                .getStreamExtractor("https://www.youtube.com/watch?v=udsB8KnIJTg");
         extractor.fetchPage();
     }
 
-    @Test
+    @Override public StreamExtractor extractor() { return extractor; }
-    public void testGetInvalidTimeStamp() throws ParsingException {
+    @Override public StreamingService expectedService() { return YouTube; }
-        assertTrue(extractor.getTimeStamp() + "",
+    @Override public String expectedName() { return "Praise the Casual: Ein Neuling trifft Dark Souls - Folge 5"; }
-                extractor.getTimeStamp() <= 0);
+    @Override public String expectedId() { return ID; }
-    }
+    @Override public String expectedUrlContains() { return URL; }
+    @Override public String expectedOriginalUrlContains() { return URL; }
 
-    @Test
+    @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
-    public void testGetTitle() throws ParsingException {
+    @Override public String expectedUploaderName() { return "Hooked"; }
-        assertFalse(extractor.getName().isEmpty());
+    @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw"; }
-    }
+    @Override public List<String> expectedDescriptionContains() {
-
+        return Arrays.asList("https://www.youtube.com/user/Roccowschiptune",
-    @Test
+                "https://www.facebook.com/HookedMagazinDE");
-    public void testGetDescription() throws ParsingException {
-        assertNotNull(extractor.getDescription());
-        assertFalse(extractor.getDescription().getContent().isEmpty());
-    }
-
-    @Test
-    public void testGetFullLinksInDescription() throws ParsingException {
-        assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/user/Roccowschiptune"));
-    }
-
-    @Test
-    public void testGetUploaderName() throws ParsingException {
-        assertNotNull(extractor.getUploaderName());
-        assertFalse(extractor.getUploaderName().isEmpty());
-    }
-
-
-    @Test
-    public void testGetLength() throws ParsingException {
-        assertEquals(2488, extractor.getLength());
-    }
-
-    @Test
-    public void testGetViewCount() throws ParsingException {
-        long count = extractor.getViewCount();
-        assertTrue(Long.toString(count), count >= /* specific to that video */ 1225);
-    }
-
-    @Test
-    public void testGetTextualUploadDate() throws ParsingException {
-        Assert.assertEquals("2017-09-22", extractor.getTextualUploadDate());
-    }
-
-    @Test
-    public void testGetUploadDate() throws ParsingException, ParseException {
-        final Calendar instance = Calendar.getInstance();
-        instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2017-09-22"));
-        assertNotNull(extractor.getUploadDate());
-        assertEquals(instance, extractor.getUploadDate().date());
-    }
-
-    @Test
-    public void testGetUploaderUrl() throws ParsingException {
-        assertEquals("https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw", extractor.getUploaderUrl());
-    }
-
-    @Test
-    public void testGetThumbnailUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getThumbnailUrl());
-    }
-
-    @Test
-    public void testGetUploaderAvatarUrl() throws ParsingException {
-        assertIsSecureUrl(extractor.getUploaderAvatarUrl());
-    }
-
-    @Test
-    public void testGetAudioStreams() throws ExtractionException {
-        List<AudioStream> audioStreams = extractor.getAudioStreams();
-        assertFalse(audioStreams.isEmpty());
-        for (AudioStream s : audioStreams) {
-            assertIsSecureUrl(s.url);
-            assertTrue(Integer.toString(s.getFormatId()),
-                    0x100 <= s.getFormatId() && s.getFormatId() < 0x1000);
-        }
-    }
-
-    @Test
-    public void testGetVideoStreams() throws ExtractionException {
-        for (VideoStream s : extractor.getVideoStreams()) {
-            assertIsSecureUrl(s.url);
-            assertTrue(s.resolution.length() > 0);
-            assertTrue(Integer.toString(s.getFormatId()),
-                    0 <= s.getFormatId() && s.getFormatId() < 0x100);
-        }
-    }
-
-    @Test
-    public void testStreamType() throws ParsingException {
-        assertSame(StreamType.VIDEO_STREAM, extractor.getStreamType());
-    }
-
-    @Test
-    public void testGetRelatedVideos() throws ExtractionException {
-        StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
-        Utils.printErrors(relatedVideos.getErrors());
-        assertFalse(relatedVideos.getItems().isEmpty());
-        assertTrue(relatedVideos.getErrors().isEmpty());
-    }
-
-    @Test
-    public void testGetSubtitlesListDefault() {
-        assertFalse(extractor.getSubtitlesDefault().isEmpty());
-    }
-
-    @Test
-    public void testGetSubtitlesList() {
-        assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
-    }
-
-    @Test
-    public void testGetLikeCount() throws ParsingException {
-        long likeCount = extractor.getLikeCount();
-        assertTrue("" + likeCount, likeCount >= 96);
-    }
-
-    @Test
-    public void testGetDislikeCount() throws ParsingException {
-        long dislikeCount = extractor.getDislikeCount();
-        assertTrue("" + dislikeCount, dislikeCount >= 0);
     }
+    @Override public long expectedLength() { return 2488; }
+    @Override public long expectedViewCountAtLeast() { return 1500; }
+    @Nullable @Override public String expectedUploadDate() { return "2017-09-22 00:00:00.000"; }
+    @Nullable @Override public String expectedTextualUploadDate() { return "2017-09-22"; }
+    @Override public long expectedLikeCountAtLeast() { return 110; }
+    @Override public long expectedDislikeCountAtLeast() { return 0; }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java

@@ -21,4 +21,28 @@ public class UtilsTest {
     public void testJoin() {
         assertEquals("some,random,stuff", Utils.join(",", Arrays.asList("some", "random", "stuff")));
     }
+
+    @Test
+    public void testGetBaseUrl() throws ParsingException {
+        assertEquals("https://www.youtube.com", Utils.getBaseUrl("https://www.youtube.com/watch?v=Hu80uDzh8RY"));
+        assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube:jZViOEv90dI"));
+        assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://n8X9_MgEdCg"));
+        assertEquals("https://music.youtube.com", Utils.getBaseUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
+    }
+
+    @Test
+    public void testFollowGoogleRedirect() {
+        assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY",
+                Utils.followGoogleRedirectIfNeeded("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"));
+        assertEquals("https://www.youtube.com/watch?v=0b6cFWG45kA",
+                Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=video&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3D0b6cFWG45kA"));
+        assertEquals("https://soundcloud.com/ciaoproduction",
+                Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd="));
+
+        assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY&param=xyz",
+                Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY&param=xyz"));
+        assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello",
+                Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello"));
+    }
 }

extractor/src/test/resources/youtube_export_test.xml

@@ -1,23 +0,0 @@
-<opml version="1.1">
-    <body>
-        <outline text="YouTube Subscriptions" title="YouTube Subscriptions">
-            <outline text="Kurzgesagt – In a Nutshell" title="Kurzgesagt – In a Nutshell" type="rss"
-                     xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
-            <outline text="CaptainDisillusion" title="CaptainDisillusion" type="rss"
-                     xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCEOXxzW2vU0P-0THehuIIeg"/>
-            <outline text="TED" title="TED" type="rss"
-                     xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCAuUUnT6oDeKwE6v1NGQxug"/>
-            <outline text="Gorillaz" title="Gorillaz" type="rss"
-                     xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCfIXdjDQH9Fau7y99_Orpjw"/>
-            <outline text="ElectroBOOM" title="ElectroBOOM" type="rss"
-                     xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCJ0-OtVpF0wOKEqT2Z1HEtA"/>
-
-            <outline text="ⓤⓝⓘⓒⓞⓓⓔ" title="ⓤⓝⓘⓒⓞⓓⓔ" type="rss"
-                     xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
-            <outline text="中文" title="中文" type="rss"
-                     xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
-            <outline text="हिंदी" title="हिंदी" type="rss"
-                     xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
-        </outline>
-    </body>
-</opml>

extractor/src/test/resources/youtube_takeout_import_test.json

@@ -0,0 +1,211 @@
+[ {
+  "contentDetails" : {
+    "activityType" : "all",
+    "newItemCount" : 0,
+    "totalItemCount" : 229
+  },
+  "etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
+  "id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
+  "kind" : "youtube#subscription",
+  "snippet" : {
+    "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
+    "description" : "The official YouTube home of Gorillaz.",
+    "publishedAt" : "2020-11-01T17:24:34.498Z",
+    "resourceId" : {
+      "channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
+      "kind" : "youtube#channel"
+    },
+    "thumbnails" : {
+      "default" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "high" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "medium" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      }
+    },
+    "title" : "Gorillaz"
+  }
+}, {
+  "contentDetails" : {
+    "activityType" : "all",
+    "newItemCount" : 0,
+    "totalItemCount" : 3502
+  },
+  "etag" : "wUgip-X0qBlnjj0frSTwP6B8XoY",
+  "id" : "qUAzBV8xkoLYOP-1gwzy63zpjj8SMTtDReGwIa2sHp8",
+  "kind" : "youtube#subscription",
+  "snippet" : {
+    "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
+    "description" : "The TED Talks channel features the best talks and performances from the TED Conference, where the world's leading thinkers and doers give the talk of their lives in 18 minutes (or less). Look for talks on Technology, Entertainment and Design -- plus science, business, global issues, the arts and more. You're welcome to link to or embed these videos, forward them to others and share these ideas with people you know.\n\nTED's videos may be used for non-commercial purposes under a Creative Commons License, Attribution–Non Commercial–No Derivatives (or the CC BY – NC – ND 4.0 International) and in accordance with our TED Talks Usage Policy (https://www.ted.com/about/our-organization/our-policies-terms/ted-talks-usage-policy). For more information on using TED for commercial purposes (e.g. employee learning, in a film or online course), please submit a Media Request at https://media-requests.ted.com",
+    "publishedAt" : "2020-11-01T17:24:11.769Z",
+    "resourceId" : {
+      "channelId" : "UCAuUUnT6oDeKwE6v1NGQxug",
+      "kind" : "youtube#channel"
+    },
+    "thumbnails" : {
+      "default" : {
+        "url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "high" : {
+        "url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "medium" : {
+        "url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      }
+    },
+    "title" : "TED"
+  }
+}, {
+  "contentDetails" : {
+    "activityType" : "all",
+    "newItemCount" : 0,
+    "totalItemCount" : 98
+  },
+  "etag" : "M3Hl6FQUAD3e-fH9pcvcE9aPSWQ",
+  "id" : "qUAzBV8xkoLYOP-1gwzy64Vo-PpWMPDyIYBM1JUfepk",
+  "kind" : "youtube#subscription",
+  "snippet" : {
+    "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
+    "description" : "In a world where the content of digital images and videos can no longer be taken at face value, an unlikely hero fights for the acceptance of truth.\r\n\r\nCaptain Disillusion guides children of all ages through the maze of visual fakery to the open spaces of reality and peace of mind.\r\n\r\nSubscribe to get fun and detailed explanations of current \"unbelievable\" viral videos that fool the masses!",
+    "publishedAt" : "2020-11-01T17:23:52.909Z",
+    "resourceId" : {
+      "channelId" : "UCEOXxzW2vU0P-0THehuIIeg",
+      "kind" : "youtube#channel"
+    },
+    "thumbnails" : {
+      "default" : {
+        "url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "high" : {
+        "url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "medium" : {
+        "url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      }
+    },
+    "title" : "Captain Disillusion"
+  }
+}, {
+  "contentDetails" : {
+    "activityType" : "all",
+    "newItemCount" : 0,
+    "totalItemCount" : 130
+  },
+  "etag" : "crkTVZbDHS3arRZErMaLMnNqtac",
+  "id" : "qUAzBV8xkoLYOP-1gwzy66EVopYHE34m06PVw8Pvheg",
+  "kind" : "youtube#subscription",
+  "snippet" : {
+    "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
+    "description" : "Videos explaining things with optimistic nihilism. \n\nWe are a small team who want to make science look beautiful. Because it is beautiful. \n\nCurrently we make one animation video per month. Follow us on Twitter, Facebook to get notified when a new one comes out.\n\nFAQ:\n \n- We do the videos with After Effects and Illustrator.",
+    "publishedAt" : "2020-11-01T17:23:39.659Z",
+    "resourceId" : {
+      "channelId" : "UCsXVk37bltHxD1rDPwtNM8Q",
+      "kind" : "youtube#channel"
+    },
+    "thumbnails" : {
+      "default" : {
+        "url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "high" : {
+        "url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "medium" : {
+        "url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      }
+    },
+    "title" : "Kurzgesagt – In a Nutshell"
+  }
+}, {
+  "contentDetails" : {
+    "activityType" : "all",
+    "newItemCount" : 0,
+    "totalItemCount" : 229
+  },
+  "etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
+  "id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
+  "kind" : "youtube#subscription",
+  "snippet" : {
+    "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
+    "description" : "ⓤⓝⓘⓒⓞⓓⓔ",
+    "publishedAt" : "2020-11-01T17:24:34.498Z",
+    "resourceId" : {
+      "channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
+      "kind" : "youtube#channel"
+    },
+    "thumbnails" : {
+      "default" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "high" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "medium" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      }
+    },
+    "title" : "ⓤⓝⓘⓒⓞⓓⓔ"
+  }
+}, {
+  "contentDetails" : {
+    "activityType" : "all",
+    "newItemCount" : 0,
+    "totalItemCount" : 229
+  },
+  "etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
+  "id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
+  "kind" : "youtube#subscription",
+  "snippet" : {
+    "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
+    "description" : "中文",
+    "publishedAt" : "2020-11-01T17:24:34.498Z",
+    "resourceId" : {
+      "channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
+      "kind" : "youtube#channel"
+    },
+    "thumbnails" : {
+      "default" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "high" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "medium" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      }
+    },
+    "title" : "中文"
+  }
+}, {
+  "contentDetails" : {
+    "activityType" : "all",
+    "newItemCount" : 0,
+    "totalItemCount" : 229
+  },
+  "etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
+  "id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
+  "kind" : "youtube#subscription",
+  "snippet" : {
+    "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
+    "description" : "हिंदी",
+    "publishedAt" : "2020-11-01T17:24:34.498Z",
+    "resourceId" : {
+      "channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
+      "kind" : "youtube#channel"
+    },
+    "thumbnails" : {
+      "default" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "high" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      },
+      "medium" : {
+        "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
+      }
+    },
+    "title" : "हिंदी"
+  }
+} ]

timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/PatternsHolder.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.timeago;
 
+import java.time.temporal.ChronoUnit;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -16,7 +17,7 @@ public abstract class PatternsHolder {
     private final Collection<String> months;
     private final Collection<String> years;
 
-    private final Map<TimeAgoUnit, Map<String, Integer>> specialCases = new LinkedHashMap<>();
+    private final Map<ChronoUnit, Map<String, Integer>> specialCases = new LinkedHashMap<>();
 
     protected PatternsHolder(String wordSeparator, Collection<String> seconds, Collection<String> minutes,
                              Collection<String> hours, Collection<String> days,
@@ -69,30 +70,25 @@ public abstract class PatternsHolder {
         return years;
     }
 
-    public Map<TimeAgoUnit, Map<String, Integer>> specialCases() {
+    public Map<ChronoUnit, Map<String, Integer>> specialCases() {
         return specialCases;
     }
 
-    protected void putSpecialCase(TimeAgoUnit unit, String caseText, int caseAmount) {
+    protected void putSpecialCase(ChronoUnit unit, String caseText, int caseAmount) {
-        Map<String, Integer> item = specialCases.get(unit);
+        Map<String, Integer> item = specialCases.computeIfAbsent(unit, k -> new LinkedHashMap<>());
-
-        if (item == null) {
-            item = new LinkedHashMap<>();
-            specialCases.put(unit, item);
-        }
 
         item.put(caseText, caseAmount);
     }
 
-    public Map<TimeAgoUnit, Collection<String>> asMap() {
+    public Map<ChronoUnit, Collection<String>> asMap() {
-        final Map<TimeAgoUnit, Collection<String>> returnMap = new LinkedHashMap<>();
+        final Map<ChronoUnit, Collection<String>> returnMap = new LinkedHashMap<>();
-        returnMap.put(TimeAgoUnit.SECONDS, seconds());
+        returnMap.put(ChronoUnit.SECONDS, seconds());
-        returnMap.put(TimeAgoUnit.MINUTES, minutes());
+        returnMap.put(ChronoUnit.MINUTES, minutes());
-        returnMap.put(TimeAgoUnit.HOURS, hours());
+        returnMap.put(ChronoUnit.HOURS, hours());
-        returnMap.put(TimeAgoUnit.DAYS, days());
+        returnMap.put(ChronoUnit.DAYS, days());
-        returnMap.put(TimeAgoUnit.WEEKS, weeks());
+        returnMap.put(ChronoUnit.WEEKS, weeks());
-        returnMap.put(TimeAgoUnit.MONTHS, months());
+        returnMap.put(ChronoUnit.MONTHS, months());
-        returnMap.put(TimeAgoUnit.YEARS, years());
+        returnMap.put(ChronoUnit.YEARS, years());
 
         return returnMap;
     }

timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/TimeAgoUnit.java

@@ -1,11 +0,0 @@
-package org.schabi.newpipe.extractor.timeago;
-
-public enum TimeAgoUnit {
-    SECONDS,
-    MINUTES,
-    HOURS,
-    DAYS,
-    WEEKS,
-    MONTHS,
-    YEARS
-}

timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/iw.java

@@ -5,7 +5,8 @@
 package org.schabi.newpipe.extractor.timeago.patterns;
 
 import org.schabi.newpipe.extractor.timeago.PatternsHolder;
-import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
+
+import java.time.temporal.ChronoUnit;
 
 public class iw extends PatternsHolder {
     private static final String WORD_SEPARATOR = " ";
@@ -26,10 +27,10 @@ public class iw extends PatternsHolder {
 
     private iw() {
         super(WORD_SEPARATOR, SECONDS, MINUTES, HOURS, DAYS, WEEKS, MONTHS, YEARS);
-        putSpecialCase(TimeAgoUnit.HOURS, "שעתיים", 2);
+        putSpecialCase(ChronoUnit.HOURS, "שעתיים", 2);
-        putSpecialCase(TimeAgoUnit.DAYS, "יומיים", 2);
+        putSpecialCase(ChronoUnit.DAYS, "יומיים", 2);
-        putSpecialCase(TimeAgoUnit.WEEKS, "שבועיים", 2);
+        putSpecialCase(ChronoUnit.WEEKS, "שבועיים", 2);
-        putSpecialCase(TimeAgoUnit.MONTHS, "חודשיים", 2);
+        putSpecialCase(ChronoUnit.MONTHS, "חודשיים", 2);
-        putSpecialCase(TimeAgoUnit.YEARS, "שנתיים", 2);
+        putSpecialCase(ChronoUnit.YEARS, "שנתיים", 2);
     }
 }