Merge branch 'dev' into dev

build.gradle

@@ -5,7 +5,7 @@ allprojects {
     sourceCompatibility = 1.7
     targetCompatibility = 1.7
 
-    version 'v0.18.0'
+    version 'v0.18.6'
     group 'com.github.TeamNewPipe'
 
     repositories {

extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -1,10 +1,5 @@
 package org.schabi.newpipe.extractor;
 
-import java.io.IOException;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -13,6 +8,10 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+
 public abstract class Extractor {
     /**
      * {@link StreamingService} currently related to this extractor.<br>
@@ -21,8 +20,10 @@ public abstract class Extractor{
     private final StreamingService service;
     private final LinkHandler linkHandler;
 
-    @Nullable private Localization forcedLocalization = null;
+    @Nullable
-    @Nullable private ContentCountry forcedContentCountry = null;
+    private Localization forcedLocalization = null;
+    @Nullable
+    private ContentCountry forcedContentCountry = null;
 
     private boolean pageFetched = false;
     private final Downloader downloader;
@@ -46,6 +47,7 @@ public abstract class Extractor{
 
     /**
      * Fetch the current page.
+     *
      * @throws IOException         if the page can not be loaded
      * @throws ExtractionException if the pages content is not understood
      */
@@ -65,6 +67,7 @@ public abstract class Extractor{
 
     /**
      * Fetch the current page.
+     *
      * @param downloader the download to use
      * @throws IOException         if the page can not be loaded
      * @throws ExtractionException if the pages content is not understood
@@ -78,6 +81,7 @@ public abstract class Extractor{
 
     /**
      * Get the name
+     *
      * @return the name
      * @throws ParsingException if the name cannot be extracted
      */

extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java

@@ -115,7 +115,8 @@ public enum MediaFormat {
     }
 
     /**
-     * Get the media format by it's id.
+     * Get the media format by its id.
+     *
      * @param id the id
      * @return the id of the media format or null.
      */
@@ -135,6 +136,7 @@ public enum MediaFormat {
 
     /**
      * Get the name of the format
+     *
      * @return the name of the format
      */
     public String getName() {
@@ -143,6 +145,7 @@ public enum MediaFormat {
 
     /**
      * Get the filename extension
+     *
      * @return the filename extension
      */
     public String getSuffix() {
@@ -151,6 +154,7 @@ public enum MediaFormat {
 
     /**
      * Get the mime type
+     *
      * @return the mime type
      */
     public String getMimeType() {

extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java

@@ -1,15 +1,15 @@
 package org.schabi.newpipe.extractor;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
 import org.schabi.newpipe.extractor.services.bandcamp.BandcampService;
 import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeService;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
 /*
  * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
  * ServiceList.java is part of NewPipe.

extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -1,20 +1,12 @@
 package org.schabi.newpipe.extractor;
 
-import java.util.Collections;
-import java.util.List;
-
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.feed.FeedExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
+import org.schabi.newpipe.extractor.linkhandler.*;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@@ -26,6 +18,8 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
 import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
 
 /*
  * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
@@ -269,7 +263,7 @@ public abstract class StreamingService {
 
     public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException {
         ListLinkHandlerFactory llhf = getCommentsLHFactory();
-        if(null == llhf) {
+        if (llhf == null) {
             return null;
         }
         return getCommentsExtractor(llhf.fromUrl(url));

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java

@@ -1,7 +1,5 @@
 package org.schabi.newpipe.extractor.comments;
 
-import java.io.IOException;
-
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -10,11 +8,12 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 
+import java.io.IOException;
+
 public class CommentsInfo extends ListInfo<CommentsInfoItem> {
 
     private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) {
         super(serviceId, listUrlIdHandler, name);
-		// TODO Auto-generated constructor stub
     }
 
     public static CommentsInfo getInfo(String url) throws IOException, ExtractionException {

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java

@@ -15,12 +15,16 @@ public class Response {
     private final Map<String, List<String>> responseHeaders;
     private final String responseBody;
 
-    public Response(int responseCode, String responseMessage, Map<String, List<String>> responseHeaders, @Nullable String responseBody) {
+    private final String latestUrl;
+
+    public Response(int responseCode, String responseMessage, Map<String, List<String>> responseHeaders,
+                    @Nullable String responseBody, @Nullable String latestUrl) {
         this.responseCode = responseCode;
         this.responseMessage = responseMessage;
         this.responseHeaders = responseHeaders != null ? responseHeaders : Collections.<String, List<String>>emptyMap();
 
         this.responseBody = responseBody == null ? "" : responseBody;
+        this.latestUrl = latestUrl;
     }
 
     public int responseCode() {
@@ -40,6 +44,16 @@ public class Response {
         return responseBody;
     }
 
+    /**
+     * Used for detecting a possible redirection, limited to the latest one.
+     *
+     * @return latest url known right before this response object was created
+     */
+    @Nonnull
+    public String latestUrl() {
+        return latestUrl;
+    }
+
     /*//////////////////////////////////////////////////////////////////////////
     // Utils
     //////////////////////////////////////////////////////////////////////////*/
@@ -54,7 +68,8 @@ public class Response {
     @Nullable
     public String getHeader(String name) {
         for (Map.Entry<String, List<String>> headerEntry : responseHeaders.entrySet()) {
-            if (headerEntry.getKey().equalsIgnoreCase(name)) {
+            final String key = headerEntry.getKey();
+            if (key != null && key.equalsIgnoreCase(name)) {
                 if (headerEntry.getValue().size() > 0) {
                     return headerEntry.getValue().get(0);
                 }

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java

@@ -25,7 +25,6 @@ import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import javax.annotation.Nonnull;
 

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java

@@ -26,14 +26,17 @@ public  class KioskList {
     private final HashMap<String, KioskEntry> kioskList = new HashMap<>();
     private String defaultKiosk = null;
 
-    @Nullable private Localization forcedLocalization;
+    @Nullable
-    @Nullable private ContentCountry forcedContentCountry;
+    private Localization forcedLocalization;
+    @Nullable
+    private ContentCountry forcedContentCountry;
 
     private class KioskEntry {
         public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) {
             extractorFactory = ef;
             handlerFactory = h;
         }
+
         final KioskExtractorFactory extractorFactory;
         final ListLinkHandlerFactory handlerFactory;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java

@@ -1,10 +1,10 @@
 package org.schabi.newpipe.extractor.linkhandler;
 
-import java.io.Serializable;
-
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import java.io.Serializable;
+
 public class LinkHandler implements Serializable {
     protected final String originalUrl;
     protected final String url;

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java

@@ -1,19 +1,25 @@
 package org.schabi.newpipe.extractor.linkhandler;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
 
     ///////////////////////////////////
     // To Override
     ///////////////////////////////////
 
-    public List<String> getContentFilter(String url) throws ParsingException { return new ArrayList<>(0);}
+    public List<String> getContentFilter(String url) throws ParsingException {
-    public String getSortFilter(String url) throws ParsingException {return ""; }
+        return new ArrayList<>(0);
+    }
+
+    public String getSortFilter(String url) throws ParsingException {
+        return "";
+    }
+
     public abstract String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException;
 
     public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl) throws ParsingException {
@@ -63,8 +69,9 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
 
 
     /**
-     * For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this,
+     * For making ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this,
      * however it should not be overridden by the actual implementation.
+     *
      * @param id
      * @return the url coresponding to id without any filters applied
      */

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java

@@ -24,6 +24,7 @@ public class SearchQueryHandler extends ListLinkHandler {
     /**
      * Returns the search string. Since ListQIHandler is based on ListLinkHandler
      * getSearchString() is equivalent to calling getId().
+     *
      * @return the search string
      */
     public String getSearchString() {

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java

@@ -13,14 +13,19 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
 
     @Override
     public abstract String getUrl(String querry, List<String> contentFilter, String sortFilter) throws ParsingException;
-    public  String getSearchString(String url) { return "";}
+
+    public String getSearchString(String url) {
+        return "";
+    }
 
     ///////////////////////////////////
     // Logic
     ///////////////////////////////////
 
     @Override
-    public String getId(String url) { return getSearchString(url); }
+    public String getId(String url) {
+        return getSearchString(url);
+    }
 
     @Override
     public SearchQueryHandler fromQuery(String querry,
@@ -34,10 +39,13 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
     }
 
     /**
-     * It's not mandatorry for NewPipe to handle the Url
+     * It's not mandatory for NewPipe to handle the Url
+     *
      * @param url
      * @return
      */
     @Override
-    public boolean onAcceptUrl(String url) { return false; }
+    public boolean onAcceptUrl(String url) {
+        return false;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java

@@ -3,7 +3,10 @@ package org.schabi.newpipe.extractor.localization;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
 
 public class Localization implements Serializable {
     public static final Localization DEFAULT = new Localization("en", "GB");

extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java

@@ -23,6 +23,7 @@ public class TimeAgoParser {
      * <p>
      * Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items.
      * </p>
+     *
      * @param patternsHolder An object that holds the "time ago" patterns, special cases, and the language word separator.
      */
     public TimeAgoParser(PatternsHolder patternsHolder) {
@@ -164,6 +165,7 @@ public class TimeAgoParser {
 
     /**
      * Marks the time as approximated by setting minutes, seconds and milliseconds to 0.
+     *
      * @param calendarTime Time to be marked as approximated
      */
     private void markApproximatedTime(Calendar calendarTime) {

extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java

@@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItemExtractor;
 import org.schabi.newpipe.extractor.InfoItemsCollector;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
@@ -34,7 +33,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
 /**
  * Collector for search results
- *
+ * <p>
  * This collector can handle the following extractor types:
  * <ul>
  *     <li>{@link StreamInfoItemExtractor}</li>

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -16,7 +16,9 @@ import org.schabi.newpipe.extractor.stream.*;
 import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 
 public class MediaCCCStreamExtractor extends StreamExtractor {
 
@@ -47,8 +49,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
 
     @Nonnull
     @Override
-    public String getDescription() throws ParsingException {
+    public Description getDescription() throws ParsingException {
-        return data.getString("description");
+        return new Description(data.getString("description"), Description.PLAIN_TEXT);
     }
 
     @Override
@@ -103,13 +105,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getDashMpdUrl() throws ParsingException {
-        return null;
+        return "";
     }
 
     @Nonnull
     @Override
     public String getHlsUrl() throws ParsingException {
-        return null;
+        return "";
     }
 
     @Override
@@ -172,13 +174,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
-        return null;
+        return Collections.emptyList();
     }
 
     @Nonnull
     @Override
-    public List<SubtitlesStream> getSubtitles(MediaFormat format) throws IOException, ExtractionException {
+    public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
-        return null;
+        return Collections.emptyList();
     }
 
     @Override
@@ -211,7 +213,6 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
         } catch (JsonParserException jpe) {
             throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe);
         }
-
     }
 
     @Nonnull
@@ -225,4 +226,41 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
     public String getOriginalUrl() throws ParsingException {
         return data.getString("frontend_link");
     }
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public Locale getLanguageInfo() throws ParsingException {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java

@@ -19,14 +19,19 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory
             return url.replace("https://api.media.ccc.de/public/conferences/", "");
         } else if (url.startsWith("https://media.ccc.de/c/")) {
             return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url);
-        } else {
+        } else if (url.startsWith("https://media.ccc.de/b/")) {
-            throw new ParsingException("Could not get id from url: " + url);
+            return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url);
         }
+        throw new ParsingException("Could not get id from url: " + url);
     }
 
     @Override
     public boolean onAcceptUrl(String url) throws ParsingException {
-        return url.startsWith("https://api.media.ccc.de/public/conferences/")
+        try {
-                || url.startsWith("https://media.ccc.de/c/");
+            getId(url);
+            return true;
+        } catch (ParsingException e) {
+            return false;
+        }
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java

@@ -1,7 +1,6 @@
 package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 
 import java.util.List;

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java

@@ -1,16 +1,39 @@
 package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
 
+import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import java.net.MalformedURLException;
+import java.net.URL;
 
 public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
 
     @Override
-    public String getId(String url) throws ParsingException {
+    public String getId(String urlString) throws ParsingException {
-        if(url.startsWith("https://api.media.ccc.de/public/events/") &&
+        if (urlString.startsWith("https://api.media.ccc.de/public/events/") &&
-            !url.contains("?q=")) {
+                !urlString.contains("?q=")) {
-            return url.replace("https://api.media.ccc.de/public/events/", "");
+            return urlString.substring(39); //remove api…/public/events part
         }
+
+        URL url;
+        try {
+            url = Utils.stringToURL(urlString);
+        } catch (MalformedURLException e) {
+            throw new IllegalArgumentException("The given URL is not valid");
+        }
+
+        String path = url.getPath();
+        // remove leading "/" of URL-path if URL-path is given
+        if (!path.isEmpty()) {
+            path = path.substring(1);
+        }
+
+        if (path.startsWith("v/")) {
+            return path.substring(2);
+        }
+
         throw new ParsingException("Could not get id from url: " + url);
     }
 
@@ -21,7 +44,11 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
 
     @Override
     public boolean onAcceptUrl(String url) throws ParsingException {
-        return url.startsWith("https://api.media.ccc.de/public/events/") &&
+        try {
-                !url.contains("?q=");
+            getId(url);
+            return true;
+        } catch (ParsingException e) {
+            return false;
+        }
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java

@@ -1,7 +1,8 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import java.io.IOException;
+import com.grack.nanojson.JsonObject;
-
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -10,9 +11,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
-import com.grack.nanojson.JsonObject;
+import java.io.IOException;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
 
 public class PeertubeInstance {
 
@@ -44,7 +43,7 @@ public class PeertubeInstance {
             throw new Exception("unable to configure instance " + url, e);
         }
 
-        if(null == response || StringUtil.isBlank(response.responseBody())) {
+        if (response == null || StringUtil.isBlank(response.responseBody())) {
             throw new Exception("unable to configure instance " + url);
         }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java

@@ -1,15 +1,15 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
+import com.grack.nanojson.JsonObject;
+import org.jsoup.helper.StringUtil;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
-
+import java.util.TimeZone;
-import org.jsoup.helper.StringUtil;
-import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-
-import com.grack.nanojson.JsonObject;
 
 public class PeertubeParsingHelper {
 
@@ -26,7 +26,9 @@ public class PeertubeParsingHelper {
     public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
         Date date;
         try {
-            date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate);
+            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
+            sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+            date = sdf.parse(textualUploadDate);
         } catch (ParseException e) {
             throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java

@@ -1,38 +1,24 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static java.util.Arrays.asList;
-import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
-import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
-
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
+import org.schabi.newpipe.extractor.linkhandler.*;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
-import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
+import org.schabi.newpipe.extractor.services.peertube.extractors.*;
-import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
+import org.schabi.newpipe.extractor.services.peertube.linkHandler.*;
-import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
-import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor;
-import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor;
-import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
-import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
-import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;
-import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
-import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
-import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
+import static java.util.Arrays.asList;
+import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
+import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
+
 public class PeertubeService extends StreamingService {
 
     private PeertubeInstance instance;

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java

@@ -1,7 +1,9 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
-import java.io.IOException;
+import com.grack.nanojson.JsonArray;
-
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@@ -17,10 +19,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 
-import com.grack.nanojson.JsonArray;
+import java.io.IOException;
-import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
 
 public class PeertubeChannelExtractor extends ChannelExtractor {
 
@@ -125,7 +124,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
             if (number != null) this.total = number.longValue();
             collectStreamsFrom(collector, json, pageUrl);
         } else {
-            throw new ExtractionException("Unable to get peertube kiosk info");
+            throw new ExtractionException("Unable to get PeerTube kiosk info");
         }
         return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
     }
@@ -159,7 +158,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
         if (null != response && null != response.responseBody()) {
             setInitialData(response.responseBody());
         } else {
-            throw new ExtractionException("Unable to extract peertube channel data");
+            throw new ExtractionException("Unable to extract PeerTube channel data");
         }
 
         String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
@@ -172,7 +171,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
         } catch (JsonParserException e) {
             throw new ExtractionException("Unable to extract peertube channel data", e);
         }
-        if(null == json) throw new ExtractionException("Unable to extract peertube channel data");
+        if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java

@@ -1,7 +1,8 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
-import java.io.IOException;
+import com.grack.nanojson.JsonArray;
-
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@@ -16,9 +17,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 
-import com.grack.nanojson.JsonArray;
+import java.io.IOException;
-import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
 
 public class PeertubeCommentsExtractor extends CommentsExtractor {
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
+import com.grack.nanojson.JsonObject;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.ServiceList;
@@ -9,8 +10,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
-import com.grack.nanojson.JsonObject;
-
 
 public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
 
@@ -91,7 +90,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
 
     @Override
     public String getAuthorName() throws ParsingException {
-        return JsonUtils.getString(item, "account.displayName");
+        return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java

@@ -1,7 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
-import java.io.IOException;
-
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -10,6 +8,8 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
+import java.io.IOException;
+
 public class PeertubePlaylistExtractor extends PlaylistExtractor {
 
     public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
@@ -73,7 +73,6 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor{
 
     @Override
     public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
-        // TODO Auto-generated method stub
 
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java

@@ -1,7 +1,8 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
-import java.io.IOException;
+import com.grack.nanojson.JsonArray;
-
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.InfoItemExtractor;
@@ -18,9 +19,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 
-import com.grack.nanojson.JsonArray;
+import java.io.IOException;
-import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
 
 public class PeertubeSearchExtractor extends SearchExtractor {
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -1,15 +1,12 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
-import java.io.IOException;
+import com.grack.nanojson.JsonArray;
-import java.io.UnsupportedEncodingException;
+import com.grack.nanojson.JsonObject;
-import java.net.URLEncoder;
+import com.grack.nanojson.JsonParser;
-import java.util.ArrayList;
+import com.grack.nanojson.JsonParserException;
-import java.util.Collections;
-import java.util.List;
-
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.MediaFormat;
-import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -20,28 +17,24 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
-import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.*;
-import org.schabi.newpipe.extractor.stream.Stream;
-import org.schabi.newpipe.extractor.stream.StreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamInfoItem;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.SubtitlesStream;
-import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
-import org.schabi.newpipe.extractor.utils.Utils;
 
-import com.grack.nanojson.JsonArray;
+import javax.annotation.Nonnull;
-import com.grack.nanojson.JsonObject;
+import java.io.IOException;
-import com.grack.nanojson.JsonParser;
+import java.io.UnsupportedEncodingException;
-import com.grack.nanojson.JsonParserException;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
 
 public class PeertubeStreamExtractor extends StreamExtractor {
 
 
+    private final String baseUrl;
     private JsonObject json;
     private List<SubtitlesStream> subtitles = new ArrayList<>();
-    private final String baseUrl;
 
     public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
         super(service, linkHandler);
@@ -66,22 +59,40 @@ public class PeertubeStreamExtractor extends StreamExtractor {
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
-        return baseUrl + JsonUtils.getString(json, "thumbnailPath");
+        return baseUrl + JsonUtils.getString(json, "previewPath");
     }
 
     @Override
-    public String getDescription() throws ParsingException {
+    public Description getDescription() throws ParsingException {
+        String text;
         try {
-            return JsonUtils.getString(json, "description");
+            text = JsonUtils.getString(json, "description");
         } catch (ParsingException e) {
-            return "No description";
+            return Description.emptyDescription;
         }
+        if (text.length() == 250 && text.substring(247).equals("...")) {
+            //if description is shortened, get full description
+            Downloader dl = NewPipe.getDownloader();
+            try {
+                Response response = dl.get(getUrl() + "/description");
+                JsonObject jsonObject = JsonParser.object().from(response.responseBody());
+                text = JsonUtils.getString(jsonObject, "description");
+            } catch (ReCaptchaException | IOException | JsonParserException e) {
+                e.printStackTrace();
+            }
+        }
+        return new Description(text, Description.MARKDOWN);
     }
 
     @Override
     public int getAgeLimit() throws ParsingException {
+        boolean isNSFW = JsonUtils.getBoolean(json, "nsfw");
+        if (isNSFW) {
+            return 18;
+        } else {
             return NO_AGE_LIMIT;
         }
+    }
 
     @Override
     public long getLength() throws ParsingException {
@@ -225,7 +236,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return collector;
     }
 
-    private List<String> getTags(){
+    @Override
+    public List<String> getTags() {
         try {
             return (List) JsonUtils.getArray(json, "tags");
         } catch (Exception e) {
@@ -233,6 +245,16 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         }
     }
 
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        try {
+            return JsonUtils.getString(json, "support");
+        } catch (ParsingException e) {
+            return "";
+        }
+    }
+
     private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
         String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
         StringBuilder params = new StringBuilder();
@@ -320,7 +342,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
                         String languageCode = JsonUtils.getString(caption, "language.id");
                         String ext = url.substring(url.lastIndexOf(".") + 1);
                         MediaFormat fmt = MediaFormat.getFromSuffix(ext);
-                        if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
+                        if (fmt != null && languageCode != null)
+                            subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
                     }
                 }
             } catch (Exception e) {
@@ -339,4 +362,32 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         return baseUrl + "/videos/watch/" + getId();
     }
 
+    @Override
+    public String getHost() throws ParsingException {
+        return JsonUtils.getString(json, "account.host");
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return JsonUtils.getString(json, "privacy.label");
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return JsonUtils.getString(json, "category.label");
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return JsonUtils.getString(json, "licence.label");
+    }
+
+    @Override
+    public Locale getLanguageInfo() throws ParsingException {
+        try {
+            return new Locale(JsonUtils.getString(json, "language.id"));
+        } catch (ParsingException e) {
+            return null;
+        }
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
+import com.grack.nanojson.JsonObject;
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
@@ -8,8 +9,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
-import com.grack.nanojson.JsonObject;
-
 public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
 
     protected final JsonObject item;

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java

@@ -1,10 +1,10 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
-import java.util.List;
-
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 
+import java.util.List;
+
 public class PeertubeSubscriptionExtractor extends SubscriptionExtractor {
 
     public PeertubeSubscriptionExtractor(StreamingService service, List<ContentSource> supportedSources) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java

@@ -1,13 +1,13 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
 public class PeertubeSuggestionExtractor extends SuggestionExtractor {
 
     public PeertubeSuggestionExtractor(StreamingService service) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java

@@ -1,7 +1,8 @@
 package org.schabi.newpipe.extractor.services.peertube.extractors;
 
-import java.io.IOException;
+import com.grack.nanojson.JsonArray;
-
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -16,9 +17,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 
-import com.grack.nanojson.JsonArray;
+import java.io.IOException;
-import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
 
 public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
 
@@ -50,7 +49,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
         try {
             contents = (JsonArray) JsonUtils.getValue(json, "data");
         } catch (Exception e) {
-            throw new ParsingException("unable to extract kiosk info", e);
+            throw new ParsingException("Unable to extract kiosk info", e);
         }
 
         String baseUrl = getBaseUrl();

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java

@@ -1,12 +1,12 @@
 package org.schabi.newpipe.extractor.services.peertube.linkHandler;
 
-import java.util.List;
-
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 
+import java.util.List;
+
 public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java

@@ -1,13 +1,13 @@
 package org.schabi.newpipe.extractor.services.peertube.linkHandler;
 
-import java.util.List;
-
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 
+import java.util.List;
+
 public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory();

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java

@@ -1,13 +1,13 @@
 package org.schabi.newpipe.extractor.services.peertube.linkHandler;
 
 
-import java.util.List;
-
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 
+import java.util.List;
+
 public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory();

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java

@@ -1,13 +1,13 @@
 package org.schabi.newpipe.extractor.services.peertube.linkHandler;
 
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.util.List;
-
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.util.List;
+
 public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
 
     public static final String CHARSET_UTF_8 = "UTF-8";

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java

@@ -1,14 +1,14 @@
 package org.schabi.newpipe.extractor.services.peertube.linkHandler;
 
+import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
+
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.schabi.newpipe.extractor.ServiceList;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
-
 public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
 
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java

@@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java

@@ -1,7 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java

@@ -1,7 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public class SoundcloudParsingHelper {
-    private static final String HARDCODED_CLIENT_ID = "r5ELVSy3RkcjX7ilaL7n2v1Z8irA9SL8"; // Updated on 31/12/19
+    private static final String HARDCODED_CLIENT_ID = "t0h1jzYMsaZXy6ggnZO71gHK3Ms6CFwE"; // Updated on 14/03/20
     private static String clientId;
 
     private SoundcloudParsingHelper() {
@@ -103,7 +103,7 @@ public class SoundcloudParsingHelper {
 
     /**
      * Call the endpoint "/resolve" of the api.<p>
-     * 
+     * <p>
      * See https://developers.soundcloud.com/docs/api/reference#resolve
      */
     public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -3,8 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java

@@ -1,7 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java

@@ -4,7 +4,10 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.schabi.newpipe.extractor.*;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.InfoItemExtractor;
+import org.schabi.newpipe.extractor.InfoItemsCollector;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -1,25 +1,20 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import static java.util.Collections.singletonList;
-import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
-
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
+import org.schabi.newpipe.extractor.linkhandler.*;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 
+import static java.util.Collections.singletonList;
+import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
+
 public class SoundcloudService extends StreamingService {
 
     public SoundcloudService(int id) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -4,7 +4,9 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.schabi.newpipe.extractor.*;
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -20,6 +22,7 @@ import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 
 public class SoundcloudStreamExtractor extends StreamExtractor {
     private JsonObject track;
@@ -73,10 +76,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         return artworkUrlBetterResolution;
     }
 
-    @Nonnull
     @Override
-    public String getDescription() {
+    public Description getDescription() {
-        return track.getString("description");
+        return new Description(track.getString("description"), Description.PLAIN_TEXT);
     }
 
     @Override
@@ -254,4 +256,41 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     public String getErrorMessage() {
         return null;
     }
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public Locale getLanguageInfo() throws ParsingException {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java

@@ -1,7 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java

@@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -1,13 +1,5 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-import static java.util.Arrays.asList;
-import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
-import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
-import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
-import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
-
-import java.util.List;
-
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@@ -15,28 +7,22 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.feed.FeedExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
+import org.schabi.newpipe.extractor.linkhandler.*;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
-import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
-import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.youtube.extractors.*;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.*;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
 import javax.annotation.Nonnull;
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*;
 
 /*
  * Created by Christian Schabesberger on 23.08.15.

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -1,28 +1,29 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
+
-import com.grack.nanojson.JsonParserException;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 
+import javax.annotation.Nonnull;
+
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
+
 /*
  * Created by Christian Schabesberger on 25.07.16.
  *
@@ -45,10 +46,8 @@ import java.io.IOException;
 
 @SuppressWarnings("WeakerAccess")
 public class YoutubeChannelExtractor extends ChannelExtractor {
-    /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
+    private JsonObject initialData;
-    private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
+    private JsonObject videoTab;
-
-    private Document doc;
 
     public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
         super(service, linkHandler);
@@ -56,21 +55,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
+        final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
-        final Response response = downloader.get(channelUrl, getExtractorLocalization());
+
-        doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response);
+        final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
+        initialData = ajaxJson.getObject(1).getObject("response");
+        YoutubeParsingHelper.defaultAlertsCheck(initialData);
     }
 
+
     @Override
     public String getNextPageUrl() throws ExtractionException {
-        return getNextPageUrlFrom(doc);
+        if (getVideoTab() == null) return "";
+        return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer")
+                .getArray("contents").getObject(0).getObject("itemSectionRenderer")
+                .getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations"));
     }
 
     @Nonnull
     @Override
     public String getUrl() throws ParsingException {
         try {
-            return CHANNEL_URL_BASE + getId();
+            return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId());
         } catch (ParsingException e) {
             return super.getUrl();
         }
@@ -80,15 +85,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getId() throws ParsingException {
         try {
-            return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
+            return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("channelId");
-        } catch (Exception ignored) {}
-
-        // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
-        try {
-            Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
-            if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
-
-            return element.attr("data-channel-external-id");
         } catch (Exception e) {
             throw new ParsingException("Could not get channel id", e);
         }
@@ -98,7 +95,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getName() throws ParsingException {
         try {
-            return doc.select("meta[property=\"og:title\"]").first().attr("content");
+            return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title");
         } catch (Exception e) {
             throw new ParsingException("Could not get channel name", e);
         }
@@ -107,7 +104,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getAvatarUrl() throws ParsingException {
         try {
-            return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
+            String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
+                    .getArray("thumbnails").getObject(0).getString("url");
+
+            return fixThumbnailUrl(url);
         } catch (Exception e) {
             throw new ParsingException("Could not get avatar", e);
         }
@@ -116,13 +116,18 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getBannerUrl() throws ParsingException {
         try {
-            Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
+            String url = null;
-            String cssContent = el.html();
+            try {
-            String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
+                url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
+                        .getArray("thumbnails").getObject(0).getString("url");
+            } catch (Exception ignored) {}
+            if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
+                return null;
+            }
 
-            return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
+            return fixThumbnailUrl(url);
         } catch (Exception e) {
-            throw new ParsingException("Could not get Banner", e);
+            throw new ParsingException("Could not get banner", e);
         }
     }
 
@@ -137,25 +142,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
     @Override
     public long getSubscriberCount() throws ParsingException {
-
+        final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText");
-        final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
+        if (subscriberInfo != null) {
-        if (el != null) {
-            String elTitle = el.attr("title");
             try {
-                return Utils.mixedNumberWordToLong(elTitle);
+                return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo));
             } catch (NumberFormatException e) {
                 throw new ParsingException("Could not get subscriber count", e);
             }
         } else {
-            // If the element is null, the channel have the subscriber count disabled
+            // If there's no subscribe button, the channel has the subscriber count disabled
+            if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) {
                 return -1;
+            } else {
+                return 0;
+            }
         }
     }
 
     @Override
     public String getDescription() throws ParsingException {
         try {
-            return doc.select("meta[name=\"description\"]").first().attr("content");
+            return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description");
         } catch (Exception e) {
             throw new ParsingException("Could not get channel description", e);
         }
@@ -165,8 +172,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
+
-        collectStreamsFrom(collector, ul);
+        if (getVideoTab() != null) {
+            JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents")
+                    .getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0)
+                    .getObject("gridRenderer").getArray("items");
+            collectStreamsFrom(collector, videos);
+        }
+
         return new InfoItemsPage<>(collector, getNextPageUrl());
     }
 
@@ -181,106 +194,81 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         fetchPage();
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        JsonObject ajaxJson;
+        final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
-        try {
+
-            final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
+        JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
-            ajaxJson = JsonParser.object().from(response);
+                .getObject("continuationContents").getObject("gridContinuation");
-        } catch (JsonParserException pe) {
+
-            throw new ParsingException("Could not parse json data for next streams", pe);
+        collectStreamsFrom(collector, sectionListContinuation.getArray("items"));
+
+        return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
     }
 
-        final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl);
-        collectStreamsFrom(collector, ajaxHtml.select("body").first());
 
-        return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
+    private String getNextPageUrlFrom(JsonArray continuations) {
+        if (continuations == null) return "";
+
+        JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
+        String continuation = nextContinuationData.getString("continuation");
+        String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
+        return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation
+                + "&itct=" + clickTrackingParams;
     }
 
-    private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
+    private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException {
-        throws ParsingException {
-        String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");
-        if (!loadMoreHtmlDataRaw.isEmpty()) {
-            return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl));
-        } else {
-            return "";
-        }
-    }
-
-    private String getNextPageUrlFrom(Document d) throws ParsingException {
-        try {
-            Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
-            if (button != null) {
-                return button.attr("abs:data-uix-load-more-href");
-            } else {
-                // Sometimes channels are simply so small, they don't have a more streams/videos
-                return "";
-            }
-        } catch (Exception e) {
-            throw new ParsingException("Could not get next page url", e);
-        }
-    }
-
-    private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) throws ParsingException {
         collector.reset();
 
         final String uploaderName = getName();
         final String uploaderUrl = getUrl();
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
-        for (final Element li : element.children()) {
+        for (Object video : videos) {
-            if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
+            if (((JsonObject) video).getObject("gridVideoRenderer") != null) {
-                collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
+                collector.commit(new YoutubeStreamInfoItemExtractor(
+                        ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
                     @Override
-                    public String getUrl() throws ParsingException {
+                    public String getUploaderName() {
-                        try {
-                            Element el = li.select("div[class=\"feed-item-dismissable\"]").first();
-                            Element dl = el.select("h3").first().select("a").first();
-                            return dl.attr("abs:href");
-                        } catch (Exception e) {
-                            throw new ParsingException("Could not get web page url for the video", e);
-                        }
-                    }
-
-                    @Override
-                    public String getName() throws ParsingException {
-                        try {
-                            Element el = li.select("div[class=\"feed-item-dismissable\"]").first();
-                            Element dl = el.select("h3").first().select("a").first();
-                            return dl.text();
-                        } catch (Exception e) {
-                            throw new ParsingException("Could not get title", e);
-                        }
-                    }
-
-                    @Override
-                    public String getUploaderName() throws ParsingException {
                         return uploaderName;
                     }
 
                     @Override
-                    public String getUploaderUrl() throws ParsingException {
+                    public String getUploaderUrl() {
                         return uploaderUrl;
                     }
-
-                    @Override
-                    public String getThumbnailUrl() throws ParsingException {
-                        try {
-                            String url;
-                            Element te = li.select("span[class=\"yt-thumb-clip\"]").first()
-                                    .select("img").first();
-                            url = te.attr("abs:src");
-                            // Sometimes youtube sends links to gif files which somehow seem to not exist
-                            // anymore. Items with such gif also offer a secondary image source. So we are going
-                            // to use that if we've caught such an item.
-                            if (url.contains(".gif")) {
-                                url = te.attr("abs:data-thumb");
-                            }
-                            return url;
-                        } catch (Exception e) {
-                            throw new ParsingException("Could not get thumbnail url", e);
-                        }
-                    }
                 });
             }
         }
     }
+
+    private JsonObject getVideoTab() throws ParsingException {
+        if (this.videoTab != null) return this.videoTab;
+
+        JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
+                .getArray("tabs");
+        JsonObject videoTab = null;
+
+        for (Object tab : tabs) {
+            if (((JsonObject) tab).getObject("tabRenderer") != null) {
+                if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) {
+                    videoTab = ((JsonObject) tab).getObject("tabRenderer");
+                    break;
+                }
+            }
+        }
+
+        if (videoTab == null) {
+            throw new ParsingException("Could not find Videos tab");
+        }
+
+        try {
+            if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
+                    .getArray("contents").getObject(0).getObject("itemSectionRenderer")
+                    .getArray("contents").getObject(0).getObject("messageRenderer")
+                    .getObject("text")).equals("This channel has no videos."))
+                return null;
+        } catch (Exception ignored) {}
+
+        this.videoTab = videoTab;
+        return videoTab;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java

@@ -1,12 +1,14 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import org.jsoup.nodes.Element;
+import com.grack.nanojson.JsonObject;
+
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.util.regex.Matcher;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
-import java.util.regex.Pattern;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
 
 /*
  * Created by Christian Schabesberger on 12.02.17.
@@ -29,87 +31,67 @@ import java.util.regex.Pattern;
  */
 
 public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor {
-    private final Element el;
+    private JsonObject channelInfoItem;
 
-    public YoutubeChannelInfoItemExtractor(Element el) {
+    public YoutubeChannelInfoItemExtractor(JsonObject channelInfoItem) {
-        this.el = el;
+        this.channelInfoItem = channelInfoItem;
     }
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
-        Element img = el.select("span[class*=\"yt-thumb-simple\"]").first()
+        try {
-                .select("img").first();
+            String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
 
-        String url = img.attr("abs:src");
+            return fixThumbnailUrl(url);
-
+        } catch (Exception e) {
-        if (url.contains("gif")) {
+            throw new ParsingException("Could not get thumbnail url", e);
-            url = img.attr("abs:data-thumb");
         }
-        return url;
     }
 
     @Override
     public String getName() throws ParsingException {
-        return el.select("a[class*=\"yt-uix-tile-link\"]").first()
+        try {
-                .text();
+            return getTextFromObject(channelInfoItem.getObject("title"));
+        } catch (Exception e) {
+            throw new ParsingException("Could not get name", e);
+        }
     }
 
     @Override
     public String getUrl() throws ParsingException {
         try {
-            String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
+            String id = "channel/" + channelInfoItem.getString("channelId");
-                    .attr("abs:data-href");
+            return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
-
-            Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
-            Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
-
-            if (match.matches()) {
-                return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
-            }
-        } catch(Exception ignored) {}
-
-        // fallback method for channels without "Subscribe" button (or just in case yt changes things)
-        // provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail
-        try {
-            return el.select("a[class*=\"yt-uix-tile-link\"]").first()
-                    .attr("abs:href");
         } catch (Exception e) {
-            throw new ParsingException("Could not get channel url", e);
+            throw new ParsingException("Could not get url", e);
         }
     }
 
     @Override
     public long getSubscriberCount() throws ParsingException {
-        final Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first();
-        if (subsEl != null) {
         try {
-                return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text()));
+            String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText"));
-            } catch (NumberFormatException e) {
+            return Utils.mixedNumberWordToLong(subscribers);
+        } catch (Exception e) {
             throw new ParsingException("Could not get subscriber count", e);
         }
-        } else {
-            // If the element is null, the channel have the subscriber count disabled
-            return -1;
-        }
     }
 
     @Override
     public long getStreamCount() throws ParsingException {
-        Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first();
+        try {
-        if (metaEl == null) {
+            return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText"))));
-            return 0;
+        } catch (Exception e) {
-        } else {
+            throw new ParsingException("Could not get stream count", e);
-            return Long.parseLong(Utils.removeNonDigitCharacters(metaEl.text()));
         }
     }
 
     @Override
     public String getDescription() throws ParsingException {
-        Element desEl = el.select("div[class*=\"yt-lockup-description\"]").first();
+        try {
-        if (desEl == null) {
+            return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
-            return "";
+        } catch (Exception e) {
-        } else {
+            throw new ParsingException("Could not get description", e);
-            return desEl.text();
         }
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
-import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@@ -22,7 +21,9 @@ import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.singletonList;
@@ -132,7 +133,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
     }
 
     private void fetchTitle(JsonArray contents) {
-        if(null == title) {
+        if (title == null) {
             try {
                 title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle"));
             } catch (Exception e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -4,8 +4,8 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -1,34 +1,33 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
+
-import com.grack.nanojson.JsonParserException;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.io.IOException;
 
+import javax.annotation.Nonnull;
+
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+
 @SuppressWarnings("WeakerAccess")
 public class YoutubePlaylistExtractor extends PlaylistExtractor {
-
+    private JsonObject initialData;
-    private Document doc;
+    private JsonObject playlistInfo;
 
     public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
         super(service, linkHandler);
@@ -36,21 +35,66 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        final String url = getUrl();
+        final String url = getUrl() + "&pbj=1";
-        final Response response = downloader.get(url, getExtractorLocalization());
+
-        doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
+        final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
+
+        initialData = ajaxJson.getObject(1).getObject("response");
+        YoutubeParsingHelper.defaultAlertsCheck(initialData);
+
+        playlistInfo = getPlaylistInfo();
+    }
+
+    private JsonObject getUploaderInfo() throws ParsingException {
+        JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items");
+        try {
+            JsonObject uploaderInfo = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer")
+                    .getObject("videoOwner").getObject("videoOwnerRenderer");
+            if (uploaderInfo != null) {
+                return uploaderInfo;
+            }
+        } catch (Exception ignored) {}
+
+        // we might want to create a loop here instead of using duplicated code
+        try {
+            JsonObject uploaderInfo = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer")
+                    .getObject("videoOwner").getObject("videoOwnerRenderer");
+            if (uploaderInfo != null) {
+                return uploaderInfo;
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Could not get uploader info", e);
+        }
+        throw new ParsingException("Could not get uploader info");
+    }
+
+    private JsonObject getPlaylistInfo() throws ParsingException {
+        try {
+            return initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items")
+                    .getObject(0).getObject("playlistSidebarPrimaryInfoRenderer");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get PlaylistInfo", e);
+        }
     }
 
     @Override
-    public String getNextPageUrl() throws ExtractionException {
+    public String getNextPageUrl() {
-        return getNextPageUrlFrom(doc);
+        return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
+                .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
+                .getObject("sectionListRenderer").getArray("contents").getObject(0)
+                .getObject("itemSectionRenderer").getArray("contents").getObject(0)
+                .getObject("playlistVideoListRenderer").getArray("continuations"));
     }
 
     @Nonnull
     @Override
     public String getName() throws ParsingException {
         try {
-            return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
+            String name = getTextFromObject(playlistInfo.getObject("title"));
+            if (name != null) return name;
+        } catch (Exception ignored) {}
+        try {
+            return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist name", e);
         }
@@ -58,25 +102,35 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
+        String url = null;
+
         try {
-            return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
+            url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
-        } catch (Exception e) {
+                    .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
-            throw new ParsingException("Could not get playlist thumbnail", e);
+        } catch (Exception ignored) {}
+
+        if (url == null) {
+            try {
+                url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
+                        .getArray("thumbnails").getObject(0).getString("url");
+            } catch (Exception ignored) {}
+
+            if (url == null) throw new ParsingException("Could not get playlist thumbnail");
         }
+
+        return fixThumbnailUrl(url);
     }
 
     @Override
     public String getBannerUrl() {
         return "";      // Banner can't be handled by frontend right now.
-                        // Whoever is willing to implement this should also implement this in the fornt end
+        // Whoever is willing to implement this should also implement it in the frontend.
     }
 
     @Override
     public String getUploaderUrl() throws ParsingException {
         try {
-            return YoutubeChannelExtractor.CHANNEL_URL_BASE +
+            return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint"));
-                    doc.select("button[class*=\"yt-uix-subscription-button\"]")
-                            .first().attr("data-channel-external-id");
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist uploader url", e);
         }
@@ -85,7 +139,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getUploaderName() throws ParsingException {
         try {
-            return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
+            return getTextFromObject(getUploaderInfo().getObject("title"));
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist uploader name", e);
         }
@@ -94,7 +148,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getUploaderAvatarUrl() throws ParsingException {
         try {
-            return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
+            String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
+
+            return fixThumbnailUrl(url);
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist uploader avatar", e);
         }
@@ -102,33 +158,26 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
     @Override
     public long getStreamCount() throws ParsingException {
-        String input;
-
         try {
-            input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text();
+            String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
-        } catch (IndexOutOfBoundsException e) {
+            return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
+        } catch (Exception e) {
             throw new ParsingException("Could not get video count from playlist", e);
         }
-
-        try {
-            return Long.parseLong(Utils.removeNonDigitCharacters(input));
-        } catch (NumberFormatException e) {
-            // When there's no videos in a playlist, there's no number in the "innerHtml",
-            // all characters that is not a number is removed, so we try to parse a empty string
-            if (!input.isEmpty()) {
-                return 0;
-            } else {
-                throw new ParsingException("Could not handle input: " + input, e);
-            }
-        }
     }
 
     @Nonnull
     @Override
-    public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
+    public InfoItemsPage<StreamInfoItem> getInitialPage() {
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
+
-        collectStreamsFrom(collector, tbody);
+        JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
+                .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
+                .getObject("sectionListRenderer").getArray("contents").getObject(0)
+                .getObject("itemSectionRenderer").getArray("contents").getObject(0)
+                .getObject("playlistVideoListRenderer").getArray("contents");
+
+        collectStreamsFrom(collector, videos);
         return new InfoItemsPage<>(collector, getNextPageUrl());
     }
 
@@ -139,155 +188,42 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         }
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        JsonObject pageJson;
+        final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
-        try {
+
-            final String responseBody = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
+        JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
-            pageJson = JsonParser.object().from(responseBody);
+                .getObject("continuationContents").getObject("playlistVideoListContinuation");
-        } catch (JsonParserException pe) {
+
-            throw new ParsingException("Could not parse ajax json", pe);
+        collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
+
+        return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
     }
 
-        final Document pageHtml = Jsoup.parse("<table><tbody id=\"pl-load-more-destination\">"
+    private String getNextPageUrlFrom(JsonArray continuations) {
-                + pageJson.getString("content_html")
+        if (continuations == null) {
-                + "</tbody></table>", pageUrl);
-
-        collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first());
-
-        return new InfoItemsPage<>(collector, getNextPageUrlFromAjax(pageJson, pageUrl));
-    }
-
-    private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl)
-            throws ParsingException{
-        String nextPageHtml = pageJson.getString("load_more_widget_html");
-        if (!nextPageHtml.isEmpty()) {
-            return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl));
-        } else {
             return "";
         }
+
+        JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
+        String continuation = nextContinuationData.getString("continuation");
+        String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
+        return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation
+                + "&itct=" + clickTrackingParams;
     }
 
-    private String getNextPageUrlFrom(Document d) throws ParsingException {
+    private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) {
-        try {
-            Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
-            if (button != null) {
-                return button.attr("abs:data-uix-load-more-href");
-            } else {
-                // Sometimes playlists are simply so small, they don't have a more streams/videos
-                return "";
-            }
-        } catch (Exception e) {
-            throw new ParsingException("could not get next streams' url", e);
-        }
-    }
-
-    private void collectStreamsFrom(@Nonnull StreamInfoItemsCollector collector, @Nullable Element element) {
         collector.reset();
 
-        if (element == null) {
-            return;
-        }
-
-        final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory();
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
-        for (final Element li : element.children()) {
+        for (Object video : videos) {
-            if(isDeletedItem(li)) {
+            if (((JsonObject) video).getObject("playlistVideoRenderer") != null) {
-                continue;
+                collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) {
-            }
-
-            collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
-                public Element uploaderLink;
-
                     @Override
-                public boolean isAd() {
+                    public long getViewCount() {
-                    return false;
-                }
-
-                @Override
-                public String getUrl() throws ParsingException {
-                    try {
-                        return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl();
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get web page url for the video", e);
-                    }
-                }
-
-                @Override
-                public String getName() throws ParsingException {
-                    try {
-                        return li.attr("data-title");
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get title", e);
-                    }
-                }
-
-                @Override
-                public long getDuration() throws ParsingException {
-                    try {
-                        if (getStreamType() == StreamType.LIVE_STREAM) return -1;
-
-                        Element first = li.select("div[class=\"timestamp\"] span").first();
-                        if (first == null) {
-                            // Video unavailable (private, deleted, etc.), this is a thing that happens specifically with playlists,
-                            // because in other cases, those videos don't even show up
                         return -1;
                     }
-
-                        return YoutubeParsingHelper.parseDurationString(first.text());
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get duration" + getUrl(), e);
-                    }
-                }
-
-
-                private Element getUploaderLink() {
-                    // should always be present since we filter deleted items
-                    if(uploaderLink == null) {
-                        uploaderLink = li.select("div[class=pl-video-owner] a").first();
-                    }
-                    return uploaderLink;
-                }
-
-                @Override
-                public String getUploaderName() throws ParsingException {
-                    return getUploaderLink().text();
-                }
-
-                @Override
-                public String getUploaderUrl() throws ParsingException {
-                    // this url is not always in the form "/channel/..."
-                    // sometimes Youtube provides urls in the from "/user/..."
-                    return getUploaderLink().attr("abs:href");
-                }
-
-                @Override
-                public String getTextualUploadDate() throws ParsingException {
-                    return "";
-                }
-
-                @Override
-                public long getViewCount() throws ParsingException {
-                    return -1;
-                }
-
-                @Override
-                public String getThumbnailUrl() throws ParsingException {
-                    try {
-                        return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId() + "/hqdefault.jpg";
-                    } catch (Exception e) {
-                        throw new ParsingException("Could not get thumbnail url", e);
-                    }
-                }
                 });
             }
         }
-
-    /**
-     * Check if the playlist item is deleted
-     * @param li the list item
-     * @return true if the item is deleted
-     */
-    private boolean isDeletedItem(Element li) {
-        return li.select("div[class=pl-video-owner] a").isEmpty();
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java

@@ -1,97 +1,68 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import org.jsoup.nodes.Element;
+import com.grack.nanojson.JsonObject;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
-    private final Element el;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
 
-    public YoutubePlaylistInfoItemExtractor(Element el) {
+public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
-        this.el = el;
+    private JsonObject playlistInfoItem;
+
+    public YoutubePlaylistInfoItemExtractor(JsonObject playlistInfoItem) {
+        this.playlistInfoItem = playlistInfoItem;
     }
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
-        String url;
-
         try {
-            Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first()
+            String url = playlistInfoItem.getArray("thumbnails").getObject(0)
-                    .select("img").first();
+                    .getArray("thumbnails").getObject(0).getString("url");
-            url = te.attr("abs:src");
 
-            if (url.contains(".gif")) {
+            return fixThumbnailUrl(url);
-                url = te.attr("abs:data-thumb");
-            }
         } catch (Exception e) {
-            throw new ParsingException("Failed to extract playlist thumbnail url", e);
+            throw new ParsingException("Could not get thumbnail url", e);
         }
-
-        return url;
     }
 
     @Override
     public String getName() throws ParsingException {
-        String name;
         try {
-            final Element title = el.select("[class=\"yt-lockup-title\"]").first()
+            return getTextFromObject(playlistInfoItem.getObject("title"));
-                    .select("a").first();
-
-            name = title == null ? "" : title.text();
         } catch (Exception e) {
-            throw new ParsingException("Failed to extract playlist name", e);
+            throw new ParsingException("Could not get name", e);
         }
-
-        return name;
     }
 
     @Override
     public String getUrl() throws ParsingException {
         try {
-            final Element a = el.select("div[class=\"yt-lockup-meta\"]")
+            String id = playlistInfoItem.getString("playlistId");
-                    .select("ul[class=\"yt-lockup-meta-info\"]")
+            return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(id);
-                    .select("li").select("a").first();
-
-            if(a != null) {
-                return a.attr("abs:href");
-            }
-
-            // this is for yt premium playlists
-            return el.select("h3[class=\"yt-lockup-title\"").first()
-                    .select("a").first()
-                    .attr("abs:href");
-
         } catch (Exception e) {
-            throw new ParsingException("Failed to extract playlist url", e);
+            throw new ParsingException("Could not get url", e);
         }
     }
 
     @Override
     public String getUploaderName() throws ParsingException {
-        String name;
-
         try {
-            final Element div = el.select("div[class=\"yt-lockup-byline\"]").first()
+            return getTextFromObject(playlistInfoItem.getObject("longBylineText"));
-                    .select("a").first();
-
-            name = div.text();
         } catch (Exception e) {
-            throw new ParsingException("Failed to extract playlist uploader", e);
+            throw new ParsingException("Could not get uploader name", e);
         }
-
-        return name;
     }
 
     @Override
     public long getStreamCount() throws ParsingException {
         try {
-            final Element count = el.select("span[class=\"formatted-video-count-label\"]").first()
+            return Long.parseLong(Utils.removeNonDigitCharacters(playlistInfoItem.getString("videoCount")));
-                    .select("b").first();
-
-            return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text()));
         } catch (Exception e) {
-            throw new ParsingException("Failed to extract playlist stream count", e);
+            throw new ParsingException("Could not get stream count", e);
         }
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -1,26 +1,24 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import org.jsoup.Jsoup;
+import com.grack.nanojson.JsonArray;
-import org.jsoup.nodes.Document;
+import com.grack.nanojson.JsonObject;
-import org.jsoup.nodes.Element;
+
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
+
-import org.schabi.newpipe.extractor.utils.Parser;
+import java.io.IOException;
 
 import javax.annotation.Nonnull;
-import java.io.IOException;
+
-import java.io.UnsupportedEncodingException;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
-import java.net.MalformedURLException;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
-import java.net.URL;
 
 /*
  * Created by Christian Schabesberger on 22.07.2018
@@ -43,8 +41,7 @@ import java.net.URL;
  */
 
 public class YoutubeSearchExtractor extends SearchExtractor {
-
+    private JsonObject initialData;
-    private Document doc;
 
     public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
         super(service, linkHandler);
@@ -52,9 +49,11 @@ public class YoutubeSearchExtractor extends SearchExtractor {
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        final String url = getUrl();
+        final String url = getUrl() + "&pbj=1";
-        final Response response = downloader.get(url, getExtractorLocalization());
+
-        doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
+        final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
+
+        initialData = ajaxJson.getObject(1).getObject("response");
     }
 
     @Nonnull
@@ -64,81 +63,85 @@ public class YoutubeSearchExtractor extends SearchExtractor {
     }
 
     @Override
-    public String getSearchSuggestion() {
+    public String getSearchSuggestion() throws ParsingException {
-        final Element el = doc.select("div[class*=\"spell-correction\"]").first();
+        JsonObject showingResultsForRenderer = initialData.getObject("contents")
-        if (el != null) {
+                .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
-            return el.select("a").first().text();
+                .getObject("sectionListRenderer").getArray("contents").getObject(0)
-        } else {
+                .getObject("itemSectionRenderer").getArray("contents").getObject(0)
+                .getObject("showingResultsForRenderer");
+        if (showingResultsForRenderer == null) {
             return "";
+        } else {
+            return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
         }
     }
 
     @Nonnull
     @Override
     public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
-        return new InfoItemsPage<>(collectItems(doc), getNextPageUrl());
+        InfoItemsSearchCollector collector = getInfoItemSearchCollector();
+        JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
+                .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
+
+        for (Object section : sections) {
+            collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents"));
+        }
+
+        return new InfoItemsPage<>(collector, getNextPageUrl());
     }
 
     @Override
     public String getNextPageUrl() throws ExtractionException {
-        return getUrl() + "&page=" + 2;
+        return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
+                .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents")
+                .getObject(0).getObject("itemSectionRenderer").getArray("continuations"));
     }
 
     @Override
     public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
-        final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
+        if (pageUrl == null || pageUrl.isEmpty()) {
-        doc = Jsoup.parse(response, pageUrl);
+            throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
-
-        return new InfoItemsPage<>(collectItems(doc), getNextPageUrlFromCurrentUrl(pageUrl));
         }
 
-    private String getNextPageUrlFromCurrentUrl(String currentUrl)
-            throws MalformedURLException, UnsupportedEncodingException {
-        final int pageNr = Integer.parseInt(
-                Parser.compatParseMap(
-                        new URL(currentUrl)
-                                .getQuery())
-                        .get("page"));
-
-        return currentUrl.replace("&page=" + pageNr,
-                "&page=" + Integer.toString(pageNr + 1));
-    }
-
-    private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException  {
         InfoItemsSearchCollector collector = getInfoItemSearchCollector();
+        final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
+
+        JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
+                .getObject("continuationContents").getObject("itemSectionContinuation");
+
+        collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
+
+        return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations")));
+    }
+
+    private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException {
         collector.reset();
 
-        Element list = doc.select("ol[class=\"item-section\"]").first();
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
-        for (Element item : list.children()) {
+        for (Object item : videos) {
-            /* First we need to determine which kind of item we are working with.
+            if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) {
-               Youtube depicts five different kinds of items on its search result page. These are
+                throw new NothingFoundException(getTextFromObject(((JsonObject) item)
-               regular videos, playlists, channels, two types of video suggestions, and a "no video
+                        .getObject("backgroundPromoRenderer").getObject("bodyText")));
-               found" item. Since we only want videos, we need to filter out all the others.
+            } else if (((JsonObject) item).getObject("videoRenderer") != null) {
-               An example for this can be seen here:
+                collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser));
-               https://www.youtube.com/results?search_query=asdf&page=1
+            } else if (((JsonObject) item).getObject("channelRenderer") != null) {
-
+                collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer")));
-               We already applied a filter to the url, so we don't need to care about channels and
+            } else if (((JsonObject) item).getObject("playlistRenderer") != null) {
-               playlists now.
+                collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer")));
-            */
+            }
-
-            Element el;
-
-            if ((el = item.select("div[class*=\"search-message\"]").first()) != null) {
-                throw new NothingFoundException(el.text());
-
-                // video item type
-            } else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) {
-                collector.commit(new YoutubeStreamInfoItemExtractor(el, timeAgoParser));
-            } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
-                collector.commit(new YoutubeChannelInfoItemExtractor(el));
-            } else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null &&
-                    item.select(".yt-pl-icon-mix").isEmpty()) {
-                collector.commit(new YoutubePlaylistInfoItemExtractor(el));
         }
     }
 
-        return collector;
+    private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException {
+        if (continuations == null) {
+            return "";
+        }
+
+        JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
+        String continuation = nextContinuationData.getString("continuation");
+        String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
+        return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
+                + "&itct=" + clickTrackingParams;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -3,11 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
+
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptableObject;
@@ -15,31 +11,52 @@ import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.downloader.Request;
-import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
-import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.Frameset;
+import org.schabi.newpipe.extractor.stream.Stream;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.stream.SubtitlesStream;
+import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.net.MalformedURLException;
+import java.text.SimpleDateFormat;
-import java.net.URL;
+import java.util.ArrayList;
-import java.util.*;
+import java.util.Calendar;
-import java.util.regex.Matcher;
+import java.util.Collections;
-import java.util.regex.Pattern;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 
 /*
  * Created by Christian Schabesberger on 06.08.15.
@@ -62,8 +79,6 @@ import java.util.regex.Pattern;
  */
 
 public class YoutubeStreamExtractor extends StreamExtractor {
-    private static final String TAG = YoutubeStreamExtractor.class.getSimpleName();
-
     /*//////////////////////////////////////////////////////////////////////////
     // Exceptions
     //////////////////////////////////////////////////////////////////////////*/
@@ -74,26 +89,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
-    public class SubtitlesException extends ContentNotAvailableException {
-        SubtitlesException(String message, Throwable cause) {
-            super(message, cause);
-        }
-    }
-
     /*//////////////////////////////////////////////////////////////////////////*/
 
-    private Document doc;
+    private JsonArray initialAjaxJson;
     @Nullable
     private JsonObject playerArgs;
     @Nonnull
     private final Map<String, String> videoInfoPage = new HashMap<>();
     private JsonObject playerResponse;
+    private JsonObject initialData;
+    private JsonObject videoPrimaryInfoRenderer;
+    private JsonObject videoSecondaryInfoRenderer;
+    private int ageLimit;
 
     @Nonnull
     private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
 
-    private boolean isAgeRestricted;
-
     public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
         super(service, linkHandler);
     }
@@ -106,21 +117,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public String getName() throws ParsingException {
         assertPageFetched();
-        try {
+        String title = null;
-            return playerResponse.getObject("videoDetails").getString("title");
 
-        } catch (Exception e) {
-            // fallback HTML method
-            String name = null;
         try {
-                name = doc.select("meta[name=title]").attr(CONTENT);
+            title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
         } catch (Exception ignored) {}
 
-            if (name == null) {
+        if (title == null) {
-                throw new ParsingException("Could not get name", e);
+            try {
-            }
+                title = playerResponse.getObject("videoDetails").getString("title");
-            return name;
+            } catch (Exception ignored) {}
+
+            if (title == null) throw new ParsingException("Could not get name");
         }
+
+        return title;
     }
 
     @Override
@@ -130,18 +141,39 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
 
         try {
-            return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate");
+            JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
-        } catch (Exception e) {
+            if (micro.getString("uploadDate") != null && !micro.getString("uploadDate").isEmpty()) {
-            String uploadDate = null;
+                return micro.getString("uploadDate");
-            try {
+            }
-                uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT);
+            if (micro.getString("publishDate") != null && !micro.getString("publishDate").isEmpty()) {
+                return micro.getString("publishDate");
+            }
         } catch (Exception ignored) {}
 
-            if (uploadDate == null) {
+        try {
-                throw new ParsingException("Could not get upload date", e);
+            if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
-            }
+                String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
-            return uploadDate;
+
+                try { // Premiered 20 hours ago
+                    TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
+                    Calendar parsedTime = timeAgoParser.parse(time).date();
+                    return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
+                } catch (Exception ignored) {}
+
+                try { // Premiered Feb 21, 2020
+                    Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
+                    return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
+                } catch (Exception ignored) {}
             }
+        } catch (Exception ignored) {}
+
+        try {
+            // TODO this parses English formatted dates only, we need a better approach to parse the textual date
+            Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
+                    getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
+            return new SimpleDateFormat("yyyy-MM-dd").format(d);
+        } catch (Exception ignored) {}
+        throw new ParsingException("Could not get upload date");
     }
 
     @Override
@@ -162,122 +194,38 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         try {
             JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
             // the last thumbnail is the one with the highest resolution
-            return thumbnails.getObject(thumbnails.size() - 1).getString("url");
+            String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
 
+            return fixThumbnailUrl(url);
         } catch (Exception e) {
-            String url = null;
+            throw new ParsingException("Could not get thumbnail url");
-            try {
-                url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
-            } catch (Exception ignored) {}
-
-            if (url == null) {
-                throw new ParsingException("Could not get thumbnail url", e);
-            }
-            return url;
         }
 
     }
 
     @Nonnull
     @Override
-    public String getDescription() throws ParsingException {
+    public Description getDescription() throws ParsingException {
         assertPageFetched();
+        // description with more info on links
         try {
-            // first try to get html-formatted description
+            String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
-            return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html());
+            return new Description(description, Description.HTML);
-        } catch (Exception e) {
+        } catch (Exception ignored) { }
+
+        // raw non-html description
         try {
-                // fallback to raw non-html description
+            return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
-                return playerResponse.getObject("videoDetails").getString("shortDescription");
         } catch (Exception ignored) {
-                throw new ParsingException("Could not get the description", e);
+            throw new ParsingException("Could not get description");
         }
     }
-    }
-
-    // onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;"
-    // :00 is NOT recognized as a timestamp in description or comments.
-    // 0:00 is recognized in both description and comments.
-    // https://www.youtube.com/watch?v=4cccfDXu1vA
-    private final static Pattern DESCRIPTION_TIMESTAMP_ONCLICK_REGEX = Pattern.compile(
-        "seekTo\\("
-            + "(?:(\\d+)\\*3600\\+)?"  // hours?
-            + "(\\d+)\\*60\\+"  // minutes
-            + "(\\d+)"  // seconds
-            + "\\)");
-
-    @SafeVarargs
-    private static <T> T coalesce(T... args) {
-        for (T arg : args) {
-            if (arg != null) return arg;
-        }
-        throw new IllegalArgumentException("all arguments to coalesce() were null");
-    }
-
-    private String parseHtmlAndGetFullLinks(String descriptionHtml)
-            throws MalformedURLException, UnsupportedEncodingException, ParsingException {
-        final Document description = Jsoup.parse(descriptionHtml, getUrl());
-        for(Element a : description.select("a")) {
-            final String rawUrl = a.attr("abs:href");
-            final URL redirectLink = new URL(rawUrl);
-
-            final Matcher onClickTimestamp;
-            final String queryString;
-            if ((onClickTimestamp = DESCRIPTION_TIMESTAMP_ONCLICK_REGEX.matcher(a.attr("onclick")))
-                    .find()) {
-                a.removeAttr("onclick");
-
-                String hours = coalesce(onClickTimestamp.group(1), "0");
-                String minutes = onClickTimestamp.group(2);
-                String seconds = onClickTimestamp.group(3);
-
-                int timestamp = 0;
-                timestamp += Integer.parseInt(hours) * 3600;
-                timestamp += Integer.parseInt(minutes) * 60;
-                timestamp += Integer.parseInt(seconds);
-
-                String setTimestamp = "&t=" + timestamp;
-
-                // Even after clicking https://youtu.be/...?t=6,
-                // getUrl() is https://www.youtube.com/watch?v=..., never youtu.be, never &t=.
-                a.attr("href", getUrl() + setTimestamp);
-
-            } else if((queryString = redirectLink.getQuery()) != null) {
-                // if the query string is null we are not dealing with a redirect link,
-                // so we don't need to override it.
-                final String link =
-                        Parser.compatParseMap(queryString).get("q");
-
-                if(link != null) {
-                    // if link is null the a tag is a hashtag.
-                    // They refer to the youtube search. We do not handle them.
-                    a.text(link);
-                    a.attr("href", link);
-                } else if(redirectLink.toString().contains("https://www.youtube.com/")) {
-                    a.text(redirectLink.toString());
-                    a.attr("href", redirectLink.toString());
-                }
-            } else if(redirectLink.toString().contains("https://www.youtube.com/")) {
-                descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString());
-                a.text(redirectLink.toString());
-                a.attr("href", redirectLink.toString());
-            }
-        }
-        return description.select("body").first().html();
-    }
 
     @Override
-    public int getAgeLimit() throws ParsingException {
+    public int getAgeLimit() {
-        assertPageFetched();
+        if (initialData == null || initialData.isEmpty()) throw new IllegalStateException("initialData is not parsed yet");
-        if (!isAgeRestricted) {
+
-            return NO_AGE_LIMIT;
+        return ageLimit;
-        }
-        try {
-            return Integer.valueOf(doc.select("meta[property=\"og:restrictions:age\"]")
-                    .attr(CONTENT).replace("+", ""));
-        } catch (Exception e) {
-            throw new ParsingException("Could not get age restriction");
-        }
     }
 
     @Override
@@ -316,68 +264,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public long getViewCount() throws ParsingException {
         assertPageFetched();
+        String views = null;
+
         try {
-            if (getStreamType().equals(StreamType.LIVE_STREAM)) {
+            views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
-                return getLiveStreamWatchingCount();
+                    .getObject("videoViewCountRenderer").getObject("viewCount"));
-            } else {
+        } catch (Exception ignored) {}
-                return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount"));
+
-            }
+        if (views == null) {
-        } catch (Exception e) {
             try {
-                return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
+                views = playerResponse.getObject("videoDetails").getString("viewCount");
-            } catch (Exception ignored) {
+            } catch (Exception ignored) {}
-                throw new ParsingException("Could not get view count", e);
+
-            }
+            if (views == null) throw new ParsingException("Could not get view count");
-        }
         }
 
-    private long getLiveStreamWatchingCount() throws ExtractionException, IOException, JsonParserException {
+        return Long.parseLong(Utils.removeNonDigitCharacters(views));
-        // https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=
-        String innerTubeKey = null, clientVersion = null;
-        if (playerArgs != null && !playerArgs.isEmpty()) {
-            innerTubeKey = playerArgs.getString("innertube_api_key");
-            clientVersion = playerArgs.getString("innertube_context_client_version");
-        } else if (!videoInfoPage.isEmpty()) {
-            innerTubeKey = videoInfoPage.get("innertube_api_key");
-            clientVersion = videoInfoPage.get("innertube_context_client_version");
-        }
-
-        if (innerTubeKey == null || innerTubeKey.isEmpty()) {
-            throw new ExtractionException("Couldn't get innerTube key");
-        }
-
-        if (clientVersion == null || clientVersion.isEmpty()) {
-            throw new ExtractionException("Couldn't get innerTube client version");
-        }
-
-        final String metadataUrl = "https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=" + innerTubeKey;
-        final byte[] dataBody = ("{\"context\":{\"client\":{\"clientName\":1,\"clientVersion\":\"" + clientVersion + "\"}}" +
-                ",\"videoId\":\"" + getId() + "\"}").getBytes("UTF-8");
-        final Response response = getDownloader().execute(Request.newBuilder()
-                .post(metadataUrl, dataBody)
-                .addHeader("Content-Type", "application/json")
-                .build());
-        final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
-
-        for (Object actionEntry : jsonObject.getArray("actions")) {
-            if (!(actionEntry instanceof JsonObject)) continue;
-            final JsonObject entry = (JsonObject) actionEntry;
-
-            final JsonObject updateViewershipAction = entry.getObject("updateViewershipAction", null);
-            if (updateViewershipAction == null) continue;
-
-            final JsonArray viewCountRuns = JsonUtils.getArray(updateViewershipAction, "viewership.videoViewCountRenderer.viewCount.runs");
-            if (viewCountRuns.isEmpty()) continue;
-
-            final JsonObject textObject = viewCountRuns.getObject(0);
-            if (!textObject.has("text")) {
-                throw new ExtractionException("Response don't have \"text\" element");
-            }
-
-            return Long.parseLong(Utils.removeNonDigitCharacters(textObject.getString("text")));
-        }
-
-        throw new ExtractionException("Could not find correct results in response");
     }
 
     @Override
@@ -385,9 +287,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         assertPageFetched();
         String likesString = "";
         try {
-            Element button = doc.select("button.like-button-renderer-like-button").first();
             try {
-                likesString = button.select("span.yt-uix-button-content").first().text();
+                likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
+                        .getObject("sentimentBarRenderer").getString("tooltip").split("/")[0];
             } catch (NullPointerException e) {
                 //if this kicks in our button has no content and therefore ratings must be disabled
                 if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
@@ -408,9 +310,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         assertPageFetched();
         String dislikesString = "";
         try {
-            Element button = doc.select("button.like-button-renderer-dislike-button").first();
             try {
-                dislikesString = button.select("span.yt-uix-button-content").first().text();
+                dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
+                        .getObject("sentimentBarRenderer").getString("tooltip").split("/")[1];
             } catch (NullPointerException e) {
                 //if this kicks in our button has no content and therefore ratings must be disabled
                 if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
@@ -431,59 +333,52 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getUploaderUrl() throws ParsingException {
         assertPageFetched();
         try {
-            return "https://www.youtube.com/channel/" +
+            String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer()
-                    playerResponse.getObject("videoDetails").getString("channelId");
+                    .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint"));
-        } catch (Exception e) {
+            if (uploaderUrl != null) return uploaderUrl;
-            String uploaderUrl = null;
-            try {
-                uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children()
-                        .select("a").first().attr("abs:href");
         } catch (Exception ignored) {}
-
+        try {
-            if (uploaderUrl == null) {
+            String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
-                throw new ParsingException("Could not get channel link", e);
+            if (uploaderId != null)
-            }
+                return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
-            return uploaderUrl;
+        } catch (Exception ignored) {}
-        }
+        throw new ParsingException("Could not get uploader url");
     }
 
     @Nonnull
     @Override
     public String getUploaderName() throws ParsingException {
         assertPageFetched();
+        String uploaderName = null;
+
         try {
-            return playerResponse.getObject("videoDetails").getString("author");
+            uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
-        } catch (Exception e) {
+                    .getObject("videoOwnerRenderer").getObject("title"));
-            String name = null;
-            try {
-                name = doc.select("div.yt-user-info").first().text();
         } catch (Exception ignored) {}
 
-            if (name == null) {
+        if (uploaderName == null) {
-                throw new ParsingException("Could not get uploader name");
+            try {
-            }
+                uploaderName = playerResponse.getObject("videoDetails").getString("author");
-            return name;
+            } catch (Exception ignored) {}
+
+            if (uploaderName == null) throw new ParsingException("Could not get uploader name");
         }
+
+        return uploaderName;
     }
 
     @Nonnull
     @Override
     public String getUploaderAvatarUrl() throws ParsingException {
         assertPageFetched();
-
-        String uploaderAvatarUrl = null;
         try {
-            uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first()
+            String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
-                    .select("img").first()
+                    .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
-                    .attr("abs:data-thumb");
+
-        } catch (Exception e) {//todo: add fallback method
+            return fixThumbnailUrl(url);
+        } catch (Exception e) {
             throw new ParsingException("Could not get uploader avatar url", e);
         }
-
-        if (uploaderAvatarUrl == null) {
-            throw new ParsingException("Could not get uploader avatar url");
-        }
-        return uploaderAvatarUrl;
     }
 
     @Nonnull
@@ -592,13 +487,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Override
     @Nonnull
-    public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
+    public List<SubtitlesStream> getSubtitlesDefault() {
         return getSubtitles(MediaFormat.TTML);
     }
 
     @Override
     @Nonnull
-    public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
+    public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
         assertPageFetched();
         List<SubtitlesStream> subtitles = new ArrayList<>();
         for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) {
@@ -622,18 +517,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public StreamInfoItem getNextStream() throws IOException, ExtractionException {
+    public StreamInfoItem getNextStream() throws ExtractionException {
         assertPageFetched();
-        try {
-            StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-            final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
-            Elements watch = doc.select("div[class=\"watch-sidebar-section\"]");
+        if (getAgeLimit() != NO_AGE_LIMIT) return null;
-            if (watch.size() < 1) {
+
-                return null;// prevent the snackbar notification "report error" on age-restricted videos
+        try {
+            final JsonObject firstWatchNextItem = initialData.getObject("contents")
+                    .getObject("twoColumnWatchNextResults").getObject("secondaryResults")
+                    .getObject("secondaryResults").getArray("results").getObject(0);
+
+            if (!firstWatchNextItem.has("compactAutoplayRenderer")) {
+                // there is no "next" stream
+                return null;
             }
 
-            collector.commit(extractVideoPreviewInfo(watch.first().select("li").first(), timeAgoParser));
+            final JsonObject videoInfo = firstWatchNextItem.getObject("compactAutoplayRenderer")
+                    .getArray("contents").getObject(0).getObject("compactVideoRenderer");
+
+            final TimeAgoParser timeAgoParser = getTimeAgoParser();
+            StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+
+            collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
             return collector.getItems().get(0);
         } catch (Exception e) {
             throw new ParsingException("Could not get next video", e);
@@ -641,20 +546,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
+    public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
         assertPageFetched();
+
+        if (getAgeLimit() != NO_AGE_LIMIT) return null;
+
         try {
             StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+            JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
+                    .getObject("secondaryResults").getObject("secondaryResults").getArray("results");
+
             final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
-            Element ul = doc.select("ul[id=\"watch-related\"]").first();
+            for (Object ul : results) {
-            if (ul != null) {
+                final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer");
-                for (Element li : ul.children()) {
+
-                    // first check if we have a playlist. If so leave them out
+                if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
-                    if (li.select("a[class*=\"content-link\"]").first() != null) {
-                        collector.commit(extractVideoPreviewInfo(li, timeAgoParser));
-                    }
-                }
             }
             return collector;
         } catch (Exception e) {
@@ -667,25 +574,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
      */
     @Override
     public String getErrorMessage() {
-        StringBuilder errorReason;
+        try {
-        Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
+            return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
-
+                    .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
-        if (errorElement == null) {
+        } catch (ParsingException e) {
-            errorReason = null;
+            return null;
-        } else {
-            String errorMessage = errorElement.text();
-            if (errorMessage == null || errorMessage.isEmpty()) {
-                errorReason = null;
-            } else {
-                errorReason = new StringBuilder(errorMessage);
-                errorReason.append("  ");
-                errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
         }
     }
 
-        return errorReason != null ? errorReason.toString() : "";
-    }
-
     /*//////////////////////////////////////////////////////////////////////////
     // Fetch page
     //////////////////////////////////////////////////////////////////////////*/
@@ -693,11 +589,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private static final String FORMATS = "formats";
     private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
     private static final String HTTPS = "https:";
-    private static final String CONTENT = "content";
     private static final String DECRYPTION_FUNC_NAME = "decrypt";
 
-    private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
-
     private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX =
             "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
     private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 =
@@ -709,32 +602,41 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     private volatile String decryptionCode = "";
 
-    private String pageHtml = null;
-
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
+        final String url = getUrl() + "&pbj=1";
-        final Response response = downloader.get(verifiedUrl, getExtractorLocalization());
+
-        pageHtml = response.responseBody();
+        initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
-        doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
 
         final String playerUrl;
-        // Check if the video is age restricted
+
-        if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) {
+        if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos
+            initialData = initialAjaxJson.getObject(2).getObject("response");
+            ageLimit = 18;
+
             final EmbeddedInfo info = getEmbeddedInfo();
             final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
             final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
             videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
             playerUrl = info.url;
-            isAgeRestricted = true;
         } else {
-            final JsonObject ytPlayerConfig = getPlayerConfig();
+            initialData = initialAjaxJson.getObject(3).getObject("response");
-            playerArgs = getPlayerArgs(ytPlayerConfig);
+            ageLimit = NO_AGE_LIMIT;
-            playerUrl = getPlayerUrl(ytPlayerConfig);
+
-            isAgeRestricted = false;
+            playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player"));
+            playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player"));
         }
+
         playerResponse = getPlayerResponse();
 
+        final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY);
+        final String status = playabilityStatus.getString("status");
+        // If status exist, and is not "OK", throw a ContentNotAvailableException with the reason.
+        if (status != null && !status.toLowerCase().equals("ok")) {
+            final String reason = playabilityStatus.getString("reason");
+            throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
+        }
+
         if (decryptionCode.isEmpty()) {
             decryptionCode = loadDecryptionCode(playerUrl);
         }
@@ -744,23 +646,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
-    private JsonObject getPlayerConfig() throws ParsingException {
-        try {
-            String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml);
-            return JsonParser.object().from(ytPlayerConfigRaw);
-        } catch (Parser.RegexException e) {
-            String errorReason = getErrorMessage();
-            switch (errorReason) {
-                case "":
-                    throw new ContentNotAvailableException("Content not available: player config empty", e);
-                default:
-                    throw new ContentNotAvailableException("Content not available", e);
-            }
-        } catch (Exception e) {
-            throw new ParsingException("Could not parse yt player config", e);
-        }
-    }
-
     private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
         JsonObject playerArgs;
 
@@ -910,9 +795,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Nonnull
-    private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesException {
+    private List<SubtitlesInfo> getAvailableSubtitlesInfo() {
         // If the video is age restricted getPlayerConfig will fail
-        if (isAgeRestricted) return Collections.emptyList();
+        if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList();
 
         final JsonObject captions;
         if (!playerResponse.has("captions")) {
@@ -924,7 +809,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
         final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
         // todo: use this to apply auto translation to different language from a source language
-        final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray());
+//        final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray());
 
         // This check is necessary since there may be cases where subtitles metadata do not contain caption track info
         // e.g. https://www.youtube.com/watch?v=-Vpwatutnko
@@ -981,6 +866,50 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     // Utils
     //////////////////////////////////////////////////////////////////////////*/
 
+    private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
+        if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer;
+
+        JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
+                .getObject("results").getObject("results").getArray("contents");
+        JsonObject videoPrimaryInfoRenderer = null;
+
+        for (Object content : contents) {
+            if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) {
+                videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer");
+                break;
+            }
+        }
+
+        if (videoPrimaryInfoRenderer == null) {
+            throw new ParsingException("Could not find videoPrimaryInfoRenderer");
+        }
+
+        this.videoPrimaryInfoRenderer = videoPrimaryInfoRenderer;
+        return videoPrimaryInfoRenderer;
+    }
+
+    private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
+        if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer;
+
+        JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
+                .getObject("results").getObject("results").getArray("contents");
+        JsonObject videoSecondaryInfoRenderer = null;
+
+        for (Object content : contents) {
+            if (((JsonObject) content).getObject("videoSecondaryInfoRenderer") != null) {
+                videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer");
+                break;
+            }
+        }
+
+        if (videoSecondaryInfoRenderer == null) {
+            throw new ParsingException("Could not find videoSecondaryInfoRenderer");
+        }
+
+        this.videoSecondaryInfoRenderer = videoSecondaryInfoRenderer;
+        return videoSecondaryInfoRenderer;
+    }
+
     @Nonnull
     private static String getVideoInfoUrl(final String id, final String sts) {
         return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
@@ -1015,84 +944,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
                         urlAndItags.put(streamUrl, itagItem);
                     }
-                } catch (UnsupportedEncodingException ignored) {
+                } catch (UnsupportedEncodingException ignored) {}
-
-                }
             }
         }
 
         return urlAndItags;
     }
 
-    /**
-     * Provides information about links to other videos on the video page, such as related videos.
-     * This is encapsulated in a StreamInfoItem object, which is a subset of the fields in a full StreamInfo.
-     */
-    private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li, final TimeAgoParser timeAgoParser) {
-        return new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
-
-            @Override
-            public String getUrl() throws ParsingException {
-                return li.select("a.content-link").first().attr("abs:href");
-            }
-
-            @Override
-            public String getName() throws ParsingException {
-                //todo: check NullPointerException causing
-                return li.select("span.title").first().text();
-                //this page causes the NullPointerException, after finding it by searching for "tjvg":
-                //https://www.youtube.com/watch?v=Uqg0aEhLFAg
-            }
-
-            @Override
-            public String getUploaderName() throws ParsingException {
-                return li.select("span[class*=\"attribution\"").first()
-                        .select("span").first().text();
-            }
-
-            @Override
-            public String getUploaderUrl() throws ParsingException {
-                return ""; // The uploader is not linked
-            }
-
-            @Override
-            public String getTextualUploadDate() throws ParsingException {
-                return "";
-            }
-
-            @Override
-            public String getThumbnailUrl() throws ParsingException {
-                Element img = li.select("img").first();
-                String thumbnailUrl = img.attr("abs:src");
-                // Sometimes youtube sends links to gif files which somehow seem to not exist
-                // anymore. Items with such gif also offer a secondary image source. So we are going
-                // to use that if we caught such an item.
-                if (thumbnailUrl.contains(".gif")) {
-                    thumbnailUrl = img.attr("data-thumb");
-                }
-                if (thumbnailUrl.startsWith("//")) {
-                    thumbnailUrl = HTTPS + thumbnailUrl;
-                }
-                return thumbnailUrl;
-            }
-        };
-    }
-
     @Nonnull
     @Override
     public List<Frameset> getFrames() throws ExtractionException {
         try {
-			final String script = doc.select("#player-api").first().siblingElements().select("script").html();
+            JsonObject jo = initialAjaxJson.getObject(2).getObject("player");
-			int p = script.indexOf("ytplayer.config");
-			if (p == -1) {
-				return Collections.emptyList();
-			}
-			p = script.indexOf('{', p);
-			int e = script.indexOf("ytplayer.load", p);
-			if (e == -1) {
-				return Collections.emptyList();
-			}
-			JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
             final String resp = jo.getObject("args").getString("player_response");
             jo = JsonParser.object().from(resp);
             final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
@@ -1134,4 +997,45 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             throw new ExtractionException(e);
         }
     }
+
+    @Nonnull
+    @Override
+    public String getHost() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getPrivacy() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getCategory() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getLicence() {
+        return "";
+    }
+
+    @Override
+    public Locale getLanguageInfo() {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -1,11 +1,12 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import org.jsoup.nodes.Element;
+import com.grack.nanojson.JsonArray;
-import org.jsoup.select.Elements;
+import com.grack.nanojson.JsonObject;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -15,6 +16,8 @@ import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
 
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*;
+
 /*
  * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
  * YoutubeStreamInfoItemExtractor.java is part of NewPipe.
@@ -35,113 +38,151 @@ import java.util.Date;
 
 public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
 
-    private final Element item;
+    private JsonObject videoInfo;
     private final TimeAgoParser timeAgoParser;
-
+    private StreamType cachedStreamType;
-    private String cachedUploadDate;
 
     /**
      * Creates an extractor of StreamInfoItems from a YouTube page.
-     * @param item          The page element
+     *
+     * @param videoInfoItem The JSON page element
      * @param timeAgoParser A parser of the textual dates or {@code null}.
      */
-    public YoutubeStreamInfoItemExtractor(Element item, @Nullable TimeAgoParser timeAgoParser) {
+    public YoutubeStreamInfoItemExtractor(JsonObject videoInfoItem, @Nullable TimeAgoParser timeAgoParser) {
-        this.item = item;
+        this.videoInfo = videoInfoItem;
         this.timeAgoParser = timeAgoParser;
     }
 
     @Override
-    public StreamType getStreamType() throws ParsingException {
+    public StreamType getStreamType() {
-        if (isLiveStream(item)) {
+        if (cachedStreamType != null) {
-            return StreamType.LIVE_STREAM;
+            return cachedStreamType;
-        } else {
-            return StreamType.VIDEO_STREAM;
         }
+
+        try {
+            JsonArray badges = videoInfo.getArray("badges");
+            for (Object badge : badges) {
+                if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) {
+                    return cachedStreamType = StreamType.LIVE_STREAM;
+                }
+            }
+
+        } catch (Exception ignored) {}
+
+        try {
+            final String style = videoInfo.getArray("thumbnailOverlays").getObject(0)
+                    .getObject("thumbnailOverlayTimeStatusRenderer").getString("style");
+            if (style.equalsIgnoreCase("LIVE")) {
+                return cachedStreamType = StreamType.LIVE_STREAM;
+            }
+        } catch (Exception ignored) {}
+
+        return cachedStreamType = StreamType.VIDEO_STREAM;
     }
 
     @Override
     public boolean isAd() throws ParsingException {
-        return !item.select("span[class*=\"icon-not-available\"]").isEmpty()
+        return isPremium() || getName().equals("[Private video]") || getName().equals("[Deleted video]");
-                || !item.select("span[class*=\"yt-badge-ad\"]").isEmpty()
-                || isPremiumVideo();
-    }
-
-    private boolean isPremiumVideo() {
-        Element premiumSpan = item.select("span[class=\"standalone-collection-badge-renderer-red-text\"]").first();
-        if(premiumSpan == null) return false;
-
-        // if this span has text it most likely says ("Free Video") so we can play this
-        if(premiumSpan.hasText()) return false;
-        return true;
     }
 
     @Override
     public String getUrl() throws ParsingException {
         try {
-            Element el = item.select("div[class*=\"yt-lockup-video\"]").first();
+            String videoId = videoInfo.getString("videoId");
-            Element dl = el.select("h3").first().select("a").first();
+            return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
-            return dl.attr("abs:href");
         } catch (Exception e) {
-            throw new ParsingException("Could not get web page url for the video", e);
+            throw new ParsingException("Could not get url", e);
         }
     }
 
     @Override
     public String getName() throws ParsingException {
-        try {
+        String name = getTextFromObject(videoInfo.getObject("title"));
-            Element el = item.select("div[class*=\"yt-lockup-video\"]").first();
+        if (name != null && !name.isEmpty()) return name;
-            Element dl = el.select("h3").first().select("a").first();
+        throw new ParsingException("Could not get name");
-            return dl.text();
-        } catch (Exception e) {
-            throw new ParsingException("Could not get title", e);
-        }
     }
 
     @Override
     public long getDuration() throws ParsingException {
-        try {
+        if (getStreamType() == StreamType.LIVE_STREAM || isPremiere()) {
-            if (getStreamType() == StreamType.LIVE_STREAM) return -1;
+            return -1;
-
-            final Element duration = item.select("span[class*=\"video-time\"]").first();
-            // apparently on youtube, video-time element will not show up if the video has a duration of 00:00
-            // see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf
-            return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text());
-        } catch (Exception e) {
-            throw new ParsingException("Could not get Duration: " + getUrl(), e);
         }
+
+        String duration = null;
+
+        try {
+            duration = getTextFromObject(videoInfo.getObject("lengthText"));
+        } catch (Exception ignored) {}
+
+        if (duration == null) {
+            try {
+                for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
+                    if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) {
+                        duration = getTextFromObject(((JsonObject) thumbnailOverlay)
+                                .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
+                    }
+                }
+            } catch (Exception ignored) {}
+
+            if (duration == null) throw new ParsingException("Could not get duration");
+        }
+
+        return YoutubeParsingHelper.parseDurationString(duration);
     }
 
     @Override
     public String getUploaderName() throws ParsingException {
+        String name = null;
+
         try {
-            return item.select("div[class=\"yt-lockup-byline\"]").first()
+            name = getTextFromObject(videoInfo.getObject("longBylineText"));
-                    .select("a").first()
+        } catch (Exception ignored) {}
-                    .text();
+
-        } catch (Exception e) {
+        if (name == null) {
-            throw new ParsingException("Could not get uploader", e);
+            try {
+                name = getTextFromObject(videoInfo.getObject("ownerText"));
+            } catch (Exception ignored) {}
+
+            if (name == null) {
+                try {
+                    name = getTextFromObject(videoInfo.getObject("shortBylineText"));
+                } catch (Exception ignored) {}
+
+                if (name == null) throw new ParsingException("Could not get uploader name");
             }
         }
 
+        return name;
+    }
+
     @Override
     public String getUploaderUrl() throws ParsingException {
-        // this url is not always in the form "/channel/..."
+        String url = null;
-        // sometimes Youtube provides urls in the from "/user/..."
-        try {
-            try {
-                return item.select("div[class=\"yt-lockup-byline\"]").first()
-                        .select("a").first()
-                        .attr("abs:href");
-            } catch (Exception e){}
 
-            // try this if the first didn't work
+        try {
-            return item.select("span[class=\"title\"")
+            url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")
-                    .text().split(" - ")[0];
+                    .getArray("runs").getObject(0).getObject("navigationEndpoint"));
-        } catch (Exception e) {
+        } catch (Exception ignored) {}
-            System.out.println(item.html());
+
-            throw new ParsingException("Could not get uploader url", e);
+        if (url == null) {
+            try {
+                url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText")
+                        .getArray("runs").getObject(0).getObject("navigationEndpoint"));
+            } catch (Exception ignored) {}
+
+            if (url == null) {
+                try {
+                    url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
+                            .getArray("runs").getObject(0).getObject("navigationEndpoint"));
+                } catch (Exception ignored) {}
+
+                if (url == null) throw new ParsingException("Could not get uploader url");
             }
         }
 
+        return url;
+    }
+
     @Nullable
     @Override
     public String getTextualUploadDate() throws ParsingException {
@@ -149,29 +190,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
             return null;
         }
 
-        if (cachedUploadDate != null) {
+        if (isPremiere()) {
-            return cachedUploadDate;
+            final Date date = getDateFromPremiere().getTime();
+            return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
         }
 
         try {
-            if (isVideoReminder()) {
+            return getTextFromObject(videoInfo.getObject("publishedTimeText"));
-                final Calendar calendar = getDateFromReminder();
-                if (calendar != null) {
-                    return cachedUploadDate = new SimpleDateFormat("yyyy-MM-dd HH:mm")
-                            .format(calendar.getTime());
-                }
-            }
-
-
-            Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
-            if (meta == null) return "";
-
-            final Elements li = meta.select("li");
-            if (li.isEmpty()) return "";
-
-            return cachedUploadDate = li.first().text();
         } catch (Exception e) {
-            throw new ParsingException("Could not get upload date", e);
+            // upload date is not always available, e.g. in playlists
+            return null;
         }
     }
 
@@ -182,115 +210,89 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
             return null;
         }
 
-        if (isVideoReminder()) {
+        if (isPremiere()) {
-            return new DateWrapper(getDateFromReminder());
+            return new DateWrapper(getDateFromPremiere());
         }
 
-        String textualUploadDate = getTextualUploadDate();
+        final String textualUploadDate = getTextualUploadDate();
         if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) {
+            try {
                 return timeAgoParser.parse(textualUploadDate);
-        } else {
+            } catch (ParsingException e) {
-            return null;
+                throw new ParsingException("Could not get upload date", e);
             }
         }
+        return null;
+    }
 
     @Override
     public long getViewCount() throws ParsingException {
-        String input;
-
-        final Element spanViewCount = item.select("span.view-count").first();
-        if (spanViewCount != null) {
-            input = spanViewCount.text();
-
-        } else if (getStreamType().equals(StreamType.LIVE_STREAM)) {
-            Element meta = item.select("ul.yt-lockup-meta-info").first();
-            if (meta == null) return 0;
-
-            final Elements li = meta.select("li");
-            if (li.isEmpty()) return 0;
-
-            input = li.first().text();
-        } else {
         try {
-                Element meta = item.select("div.yt-lockup-meta").first();
+            if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) {
-                if (meta == null) return -1;
+                return -1;
-
-                // This case can happen if google releases a special video
-                if (meta.select("li").size() < 2) return -1;
-
-                input = meta.select("li").get(1).text();
-            } catch (IndexOutOfBoundsException e) {
-                throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
-            }
             }
 
-        if (input == null) {
+            final JsonObject viewCountObject = videoInfo.getObject("viewCountText");
-            throw new ParsingException("Input is null");
+            if (viewCountObject == null) {
+                // This object is null when a video has its views hidden.
+                return -1;
             }
 
-        try {
+            final String viewCount = getTextFromObject(viewCountObject);
 
-            return Long.parseLong(Utils.removeNonDigitCharacters(input));
+            if (viewCount.toLowerCase().contains("no views")) {
-        } catch (NumberFormatException e) {
-            // if this happens the video probably has no views
-            if (!input.isEmpty()){
                 return 0;
+            } else if (viewCount.toLowerCase().contains("recommended")) {
+                return -1;
             }
 
-            throw new ParsingException("Could not handle input: " + input, e);
+            return Long.parseLong(Utils.removeNonDigitCharacters(viewCount));
+        } catch (Exception e) {
+            throw new ParsingException("Could not get view count", e);
         }
     }
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
         try {
-            String url;
+            // TODO: Don't simply get the first item, but look at all thumbnails and their resolution
-            Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
+            String url = videoInfo.getObject("thumbnail").getArray("thumbnails")
-                    .select("img").first();
+                    .getObject(0).getString("url");
-            url = te.attr("abs:src");
+
-            // Sometimes youtube sends links to gif files which somehow seem to not exist
+            return fixThumbnailUrl(url);
-            // anymore. Items with such gif also offer a secondary image source. So we are going
-            // to use that if we've caught such an item.
-            if (url.contains(".gif")) {
-                url = te.attr("abs:data-thumb");
-            }
-            return url;
         } catch (Exception e) {
             throw new ParsingException("Could not get thumbnail url", e);
         }
     }
 
-
+    private boolean isPremium() {
-    private boolean isVideoReminder() {
-        return !item.select("span.yt-uix-livereminder").isEmpty();
-    }
-
-    private Calendar getDateFromReminder() throws ParsingException {
-        final Element timeFuture = item.select("span.yt-badge.localized-date").first();
-
-        if (timeFuture == null) {
-            throw new ParsingException("Span timeFuture is null");
-        }
-
-        final String timestamp = timeFuture.attr("data-timestamp");
-        if (!timestamp.isEmpty()) {
         try {
+            JsonArray badges = videoInfo.getArray("badges");
+            for (Object badge : badges) {
+                if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("Premium")) {
+                    return true;
+                }
+            }
+        } catch (Exception ignored) {
+        }
+        return false;
+    }
+
+    private boolean isPremiere() {
+        return videoInfo.has("upcomingEventData");
+    }
+
+    private Calendar getDateFromPremiere() throws ParsingException {
+        final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData");
+        final String startTime = upcomingEventData.getString("startTime");
+
+        try {
+            final long startTimeTimestamp = Long.parseLong(startTime);
             final Calendar calendar = Calendar.getInstance();
-                calendar.setTime(new Date(Long.parseLong(timestamp) * 1000L));
+            calendar.setTime(new Date(startTimeTimestamp * 1000L));
             return calendar;
         } catch (Exception e) {
-                throw new ParsingException("Could not parse = \"" + timestamp + "\"");
+            throw new ParsingException("Could not parse date from premiere:  \"" + startTime + "\"");
         }
     }
-
-        throw new ParsingException("Could not parse date from reminder element: \"" + timeFuture + "\"");
-    }
-
-    /**
-     * Generic method that checks if the element contains any clues that it's a livestream item
-     */
-    protected static boolean isLiveStream(Element item) {
-        return !item.select("span[class*=\"yt-badge-live\"]").isEmpty()
-                || !item.select("span[class*=\"video-time-overlay-live\"]").isEmpty();
-    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java

@@ -3,9 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -20,27 +20,28 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import org.jsoup.nodes.Document;
+import com.grack.nanojson.JsonArray;
-import org.jsoup.nodes.Element;
+import com.grack.nanojson.JsonObject;
-import org.jsoup.select.Elements;
+
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 
-public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
+import javax.annotation.Nonnull;
 
-    private Document doc;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
+
+public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
+    private JsonObject initialData;
 
     public YoutubeTrendingExtractor(StreamingService service,
                                     ListLinkHandler linkHandler,
@@ -50,11 +51,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        final String url = getUrl() +
+        final String url = getUrl() + "?pbj=1&gl="
-                "?gl=" + getExtractorContentCountry().getCountryCode();
+                + getExtractorContentCountry().getCountryCode();
 
-        final Response response = downloader.get(url, getExtractorLocalization());
+        final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
-        doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
+
+        initialData = ajaxJson.getObject(1).getObject("response");
     }
 
     @Override
@@ -70,99 +72,39 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
     @Nonnull
     @Override
     public String getName() throws ParsingException {
+        String name;
         try {
-            Element a = doc.select("a[href*=\"/feed/trending\"]").first();
+            name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
-            Element span = a.select("span[class*=\"display-name\"]").first();
-            Element nameSpan = span.select("span").first();
-            return nameSpan.text();
         } catch (Exception e) {
             throw new ParsingException("Could not get Trending name", e);
         }
+        if (name != null && !name.isEmpty()) {
+            return name;
+        }
+        throw new ParsingException("Could not get Trending name");
     }
 
     @Nonnull
     @Override
-    public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
+    public InfoItemsPage<StreamInfoItem> getInitialPage() {
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]");
-
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
+        JsonArray itemSectionRenderers = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
+                .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
+                .getObject("sectionListRenderer").getArray("contents");
 
-        for(Element ul : uls) {
+        for (Object itemSectionRenderer : itemSectionRenderers) {
-            for(final Element li : ul.children()) {
+            JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer")
-                final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
+                    .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content")
-                collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
+                    .getObject("expandedShelfContentsRenderer");
-                    @Override
+            if (expandedShelfContentsRenderer != null) {
-                    public String getUrl() throws ParsingException {
+                for (Object ul : expandedShelfContentsRenderer.getArray("items")) {
-                        try {
+                    final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
-                            Element dl = el.select("h3").first().select("a").first();
+                    collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
-                            return dl.attr("abs:href");
-                        } catch (Exception e) {
-                            throw new ParsingException("Could not get web page url for the video", e);
                 }
             }
-
-                    @Override
-                    public String getName() throws ParsingException {
-                        try {
-                            Element dl = el.select("h3").first().select("a").first();
-                            return dl.text();
-                        } catch (Exception e) {
-                            throw new ParsingException("Could not get web page url for the video", e);
         }
-                    }
-
-                    @Override
-                    public String getUploaderUrl() throws ParsingException {
-                        try {
-                            String link = getUploaderLink().attr("abs:href");
-                            if (link.isEmpty()) {
-                                throw new IllegalArgumentException("is empty");
-                            }
-                            return link;
-                        } catch (Exception e) {
-                            throw new ParsingException("Could not get Uploader name");
-                        }
-                    }
-
-                    private Element getUploaderLink() {
-                        // this url is not always in the form "/channel/..."
-                        // sometimes Youtube provides urls in the from "/user/..."
-                        Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
-                        return uploaderEl.select("a").first();
-                    }
-
-                    @Override
-                    public String getUploaderName() throws ParsingException {
-                        try {
-                            return getUploaderLink().text();
-                        } catch (Exception e) {
-                            throw new ParsingException("Could not get Uploader name");
-                        }
-                    }
-
-                    @Override
-                    public String getThumbnailUrl() throws ParsingException {
-                        try {
-                            String url;
-                            Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
-                                    .select("img").first();
-                            url = te.attr("abs:src");
-                            // Sometimes youtube sends links to gif files which somehow seem to not exist
-                            // anymore. Items with such gif also offer a secondary image source. So we are going
-                            // to use that if we've caught such an item.
-                            if (url.contains(".gif")) {
-                                url = te.attr("abs:data-thumb");
-                            }
-                            return url;
-                        } catch (Exception e) {
-                            throw new ParsingException("Could not get thumbnail url", e);
-                        }
-                    }
-                });
-            }
-        }
-
         return new InfoItemsPage<>(collector, getNextPageUrl());
+
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java

@@ -35,6 +35,14 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
         return instance;
     }
 
+    /**
+     * Returns URL to channel from an ID
+     *
+     * @param id Channel ID including e.g. 'channel/'
+     * @param contentFilters
+     * @param searchFilter
+     * @return URL to channel
+     */
     @Override
     public String getUrl(String id, List<String> contentFilters, String searchFilter) {
         return "https://www.youtube.com/" + id;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java

@@ -1,7 +1,10 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL;
+
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 
 import java.util.List;
@@ -14,6 +17,15 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
         return instance;
     }
 
+    @Override
+    public ListLinkHandler fromUrl(String url) throws ParsingException {
+        if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){
+            return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL);
+        } else {
+            return super.fromUrl(url);
+        }
+    }
+
     @Override
     public String getUrl(String id) {
         return "https://m.youtube.com/watch?v=" + id;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java

@@ -1,17 +1,32 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
 
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.localization.Localization;
+import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.Utils;
 
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.URL;
+import java.net.URLDecoder;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.util.Calendar;
+import java.util.*;
-import java.util.Date;
+
+import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
+import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
+import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
 
 /*
  * Created by Christian Schabesberger on 02.03.16.
@@ -38,6 +53,15 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
+    /**
+     * The official youtube app supports intents in this format, where after the ':' is the videoId.
+     * Accordingly there are other apps sharing streams in this format.
+     */
+    public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube";
+
+    private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
+    private static String clientVersion;
+
     private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
     private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
 
@@ -143,4 +167,245 @@ public class YoutubeParsingHelper {
         uploadDate.setTime(date);
         return uploadDate;
     }
+
+    public static JsonObject getInitialData(String html) throws ParsingException {
+        try {
+            String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
+            return JsonParser.object().from(initialData);
+        } catch (JsonParserException | Parser.RegexException e) {
+            throw new ParsingException("Could not get ytInitialData", e);
+        }
+    }
+
+    public static boolean isHardcodedClientVersionValid() throws IOException, ExtractionException {
+        final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
+
+        Map<String, List<String>> headers = new HashMap<>();
+        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
+        headers.put("X-YouTube-Client-Version",
+                Collections.singletonList(HARDCODED_CLIENT_VERSION));
+        final String response = getDownloader().get(url, headers).responseBody();
+
+        return response.length() > 50; // ensure to have a valid response
+    }
+
+    /**
+     * Get the client version from a page
+     * @return
+     * @throws ParsingException
+     */
+    public static String getClientVersion() throws IOException, ExtractionException {
+        if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
+
+        if (isHardcodedClientVersionValid()) {
+            clientVersion = HARDCODED_CLIENT_VERSION;
+            return clientVersion;
+        }
+
+        final String url = "https://www.youtube.com/results?search_query=test";
+        final String html = getDownloader().get(url).responseBody();
+        JsonObject initialData = getInitialData(html);
+        JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
+        String shortClientVersion = null;
+
+        // try to get version from initial data first
+        for (Object service : serviceTrackingParams) {
+            JsonObject s = (JsonObject) service;
+            if (s.getString("service").equals("CSI")) {
+                JsonArray params = s.getArray("params");
+                for (Object param : params) {
+                    JsonObject p = (JsonObject) param;
+                    String key = p.getString("key");
+                    if (key != null && key.equals("cver")) {
+                        clientVersion = p.getString("value");
+                        return clientVersion;
+                    }
+                }
+            } else if (s.getString("service").equals("ECATCHER")) {
+                // fallback to get a shortened client version which does not contain the last two digits
+                JsonArray params = s.getArray("params");
+                for (Object param : params) {
+                    JsonObject p = (JsonObject) param;
+                    String key = p.getString("key");
+                    if (key != null && key.equals("client.version")) {
+                        shortClientVersion = p.getString("value");
+                    }
+                }
+            }
+        }
+
+        String contextClientVersion;
+        String[] patterns = {
+                "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
+                "innertube_context_client_version\":\"([0-9\\.]+?)\"",
+                "client.version=([0-9\\.]+)"
+        };
+        for (String pattern : patterns) {
+            try {
+                contextClientVersion = Parser.matchGroup1(pattern, html);
+                if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
+                    clientVersion = contextClientVersion;
+                    return clientVersion;
+                }
+            } catch (Exception ignored) {
+            }
+        }
+
+        if (shortClientVersion != null) {
+            clientVersion = shortClientVersion;
+            return clientVersion;
+        }
+
+        throw new ParsingException("Could not get client version");
+    }
+
+    public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
+        if (navigationEndpoint.getObject("urlEndpoint") != null) {
+            String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
+            if (internUrl.startsWith("/redirect?")) {
+                // q parameter can be the first parameter
+                internUrl = internUrl.substring(10);
+                String[] params = internUrl.split("&");
+                for (String param : params) {
+                    if (param.split("=")[0].equals("q")) {
+                        String url;
+                        try {
+                            url = URLDecoder.decode(param.split("=")[1], "UTF-8");
+                        } catch (UnsupportedEncodingException e) {
+                            return null;
+                        }
+                        return url;
+                    }
+                }
+            } else if (internUrl.startsWith("http")) {
+                return internUrl;
+            }
+        } else if (navigationEndpoint.getObject("browseEndpoint") != null) {
+            final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");
+            final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl");
+            final String browseId = browseEndpoint.getString("browseId");
+
+            // All channel ids are prefixed with UC
+            if (browseId != null && browseId.startsWith("UC")) {
+                return "https://www.youtube.com/channel/" + browseId;
+            }
+
+            if (canonicalBaseUrl != null && !canonicalBaseUrl.isEmpty()) {
+                return "https://www.youtube.com" + canonicalBaseUrl;
+            }
+
+            throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")");
+        } else if (navigationEndpoint.getObject("watchEndpoint") != null) {
+            StringBuilder url = new StringBuilder();
+            url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
+            if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
+                url.append("&amp;list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
+            if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
+                url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
+            return url.toString();
+        }
+        return null;
+    }
+
+    public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException {
+        if (textObject.has("simpleText")) return textObject.getString("simpleText");
+
+        StringBuilder textBuilder = new StringBuilder();
+        for (Object textPart : textObject.getArray("runs")) {
+            String text = ((JsonObject) textPart).getString("text");
+            if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) {
+                String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
+                if (url != null && !url.isEmpty()) {
+                    textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
+                    continue;
+                }
+            }
+            textBuilder.append(text);
+        }
+
+        String text = textBuilder.toString();
+
+        if (html) {
+            text = text.replaceAll("\\n", "<br>");
+            text = text.replaceAll("  ", " &nbsp;");
+        }
+
+        return text;
+    }
+
+    public static String getTextFromObject(JsonObject textObject) throws ParsingException {
+        return getTextFromObject(textObject, false);
+    }
+
+    public static String fixThumbnailUrl(String thumbnailUrl) {
+        if (thumbnailUrl.startsWith("//")) {
+            thumbnailUrl = thumbnailUrl.substring(2);
+        }
+
+        if (thumbnailUrl.startsWith(HTTP)) {
+            thumbnailUrl = Utils.replaceHttpWithHttps(thumbnailUrl);
+        } else if (!thumbnailUrl.startsWith(HTTPS)) {
+            thumbnailUrl = "https://" + thumbnailUrl;
+        }
+
+        return thumbnailUrl;
+    }
+
+    public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException {
+        Map<String, List<String>> headers = new HashMap<>();
+        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
+        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
+        final Response response = getDownloader().get(url, headers, localization);
+
+        if (response.responseCode() == 404) {
+            throw new ContentNotAvailableException("Not found" +
+                    " (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
+        }
+
+        final String responseBody = response.responseBody();
+        if (responseBody.length() < 50) { // ensure to have a valid response
+            throw new ParsingException("JSON response is too short");
+        }
+
+        // Check if the request was redirected to the error page.
+        final URL latestUrl = new URL(response.latestUrl());
+        if (latestUrl.getHost().equalsIgnoreCase("www.youtube.com")) {
+            final String path = latestUrl.getPath();
+            if (path.equalsIgnoreCase("/oops") || path.equalsIgnoreCase("/error")) {
+                throw new ContentNotAvailableException("Content unavailable");
+            }
+        }
+
+        final String responseContentType = response.getHeader("Content-Type");
+        if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) {
+            throw new ParsingException("Got HTML document, expected JSON response" +
+                    " (latest url was: \"" + response.latestUrl() + "\")");
+        }
+
+        try {
+            return JsonParser.array().from(responseBody);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse JSON", e);
+        }
+    }
+
+    /**
+     * Shared alert detection function, multiple endpoints return the error similarly structured.
+     * <p>
+     * Will check if the object has an alert of the type "ERROR".
+     *
+     * @param initialData the object which will be checked if an alert is present
+     * @throws ContentNotAvailableException if an alert is detected
+     */
+    public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException {
+        final JsonArray alerts = initialData.getArray("alerts");
+        if (alerts != null && !alerts.isEmpty()) {
+            final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer");
+            final String alertText = alertRenderer.getObject("text").getString("simpleText");
+            final String alertType = alertRenderer.getString("type");
+            if (alertType.equalsIgnoreCase("ERROR")) {
+                throw new ContentNotAvailableException("Got error: \"" + alertText + "\"");
+            }
+        }
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java

@@ -24,13 +24,13 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
     public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
         try {
             final String url = "https://www.youtube.com/results"
-                    + "?q=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
+                    + "?search_query=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
 
             if (contentFilters.size() > 0) {
                 switch (contentFilters.get(0)) {
-                    case VIDEOS: return url + "&sp=EgIQAVAU";
+                    case VIDEOS: return url + "&sp=EgIQAQ%253D%253D";
-                    case CHANNELS: return url + "&sp=EgIQAlAU";
+                    case CHANNELS: return url + "&sp=EgIQAg%253D%253D";
-                    case PLAYLISTS: return url + "&sp=EgIQA1AU";
+                    case PLAYLISTS: return url + "&sp=EgIQAw%253D%253D";
                     case ALL:
                     default:
                 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java

@@ -1,7 +1,10 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL;
+
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -49,6 +52,15 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
         return id;
     }
 
+    @Override
+    public LinkHandler fromUrl(String url) throws ParsingException {
+        if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){
+            return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL);
+        } else {
+            return super.fromUrl(url);
+        }
+    }
+
     @Override
     public String getUrl(String id) {
         return "https://www.youtube.com/watch?v=" + id;
@@ -190,7 +202,12 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
                     return assertIsID(id);
                 }
 
-                break;
+                String viewQueryValue = Utils.getQueryValue(url, "v");
+                if (viewQueryValue != null) {
+                    return assertIsID(viewQueryValue);
+                }
+
+                return assertIsID(path);
             }
         }
 

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java

@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.stream;
+
+import java.io.Serializable;
+
+public class Description implements Serializable {
+
+    public static final int HTML = 1;
+    public static final int MARKDOWN = 2;
+    public static final int PLAIN_TEXT = 3;
+    public static final Description emptyDescription = new Description("", PLAIN_TEXT);
+
+    private String content;
+    private int type;
+
+    public Description(String content, int type) {
+        this.type = type;
+        if (content == null) {
+            this.content = "";
+        } else {
+            this.content = content;
+        }
+    }
+
+    public String getContent() {
+        return content;
+    }
+
+    public int getType() {
+        return type;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java

@@ -1,7 +1,5 @@
 package org.schabi.newpipe.extractor.stream;
 
-import javax.annotation.Nullable;
-import java.util.Collection;
 import java.util.List;
 
 public final class Frameset {

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java

@@ -1,10 +1,10 @@
 package org.schabi.newpipe.extractor.stream;
 
+import org.schabi.newpipe.extractor.MediaFormat;
+
 import java.io.Serializable;
 import java.util.List;
 
-import org.schabi.newpipe.extractor.MediaFormat;
-
 /**
  * Creates a stream object from url, format and optional torrent url
  */

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -34,6 +34,7 @@ import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 
 /**
  * Scrapes information from a video/audio streaming service (eg, YouTube).
@@ -75,6 +76,7 @@ public abstract class StreamExtractor extends Extractor {
 
     /**
      * This will return the url to the thumbnail of the stream. Try to return the medium resolution here.
+     *
      * @return The url of the thumbnail.
      * @throws ParsingException
      */
@@ -82,15 +84,17 @@ public abstract class StreamExtractor extends Extractor {
     public abstract String getThumbnailUrl() throws ParsingException;
 
     /**
-     * This is the stream description. On YouTube this is the video description. You can return simple HTML here.
+     * This is the stream description.
-     * @return The description of the stream/video.
+     *
+     * @return The description of the stream/video or Description.emptyDescription if the description is empty.
      * @throws ParsingException
      */
     @Nonnull
-    public abstract String getDescription() throws ParsingException;
+    public abstract Description getDescription() throws ParsingException;
 
     /**
      * Get the age limit.
+     *
      * @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit
      * @throws ParsingException if an error occurs while parsing
      */
@@ -98,6 +102,7 @@ public abstract class StreamExtractor extends Extractor {
 
     /**
      * This should return the length of a video in seconds.
+     *
      * @return The length of the stream in seconds.
      * @throws ParsingException
      */
@@ -107,6 +112,7 @@ public abstract class StreamExtractor extends Extractor {
      * If the url you are currently handling contains a time stamp/seek, you can return the
      * position it represents here.
      * If the url has no time stamp simply return zero.
+     *
      * @return the timestamp in seconds
      * @throws ParsingException
      */
@@ -115,22 +121,25 @@ public abstract class StreamExtractor extends Extractor {
     /**
      * The count of how many people have watched the video/listened to the audio stream.
      * If the current stream has no view count or its not available simply return -1
+     *
      * @return amount of views.
      * @throws ParsingException
      */
     public abstract long getViewCount() throws ParsingException;
 
     /**
-     * The Amount of likes a video/audio stream got.
+     * The amount of likes a video/audio stream got.
      * If the current stream has no likes or its not available simply return -1
+     *
      * @return the amount of likes the stream got
      * @throws ParsingException
      */
     public abstract long getLikeCount() throws ParsingException;
 
     /**
-     * The Amount of dislikes a video/audio stream got.
+     * The amount of dislikes a video/audio stream got.
      * If the current stream has no dislikes or its not available simply return -1
+     *
      * @return the amount of likes the stream got
      * @throws ParsingException
      */
@@ -142,6 +151,7 @@ public abstract class StreamExtractor extends Extractor {
      * <a href="https://teamnewpipe.github.io/documentation/03_Implement_a_service/#channel">ChannelExtractor</a>,
      * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects
      * this url.
+     *
      * @return the url to the page of the creator/uploader of the stream or an empty String
      * @throws ParsingException
      */
@@ -151,6 +161,7 @@ public abstract class StreamExtractor extends Extractor {
     /**
      * The name of the creator/uploader of the stream.
      * If the name is not available you can simply return an empty string.
+     *
      * @return the name of the creator/uploader of the stream or an empty String
      * @throws ParsingException
      */
@@ -160,6 +171,7 @@ public abstract class StreamExtractor extends Extractor {
     /**
      * The url to the image file/profile picture/avatar of the creator/uploader of the stream.
      * If the url is not available you can return an empty String.
+     *
      * @return The url of the image file of the uploader or an empty String
      * @throws ParsingException
      */
@@ -169,20 +181,24 @@ public abstract class StreamExtractor extends Extractor {
     /**
      * Get the dash mpd url. If you don't know what a dash MPD is you can read about it
      * <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">here</a>.
+     *
      * @return the url as a string or an empty string
      * @throws ParsingException if an error occurs while reading
      */
-    @Nonnull public abstract String getDashMpdUrl() throws ParsingException;
+    @Nonnull
+    public abstract String getDashMpdUrl() throws ParsingException;
 
     /**
      * I am not sure if this is in use, and how this is used. However the frontend is missing support
      * for HLS streams. Prove me if I am wrong. Please open an
      * <a href="https://github.com/teamnewpipe/newpipe/issues">issue</a>,
      * or fix this description if you know whats up with this.
+     *
      * @return The Url to the hls stream.
      * @throws ParsingException
      */
-    @Nonnull public abstract String getHlsUrl() throws ParsingException;
+    @Nonnull
+    public abstract String getHlsUrl() throws ParsingException;
 
     /**
      * This should return a list of available
@@ -190,6 +206,7 @@ public abstract class StreamExtractor extends Extractor {
      * You can also return null or an empty list, however be aware that if you don't return anything
      * in getVideoStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as
      * a failed extraction procedure.
+     *
      * @return a list of audio only streams in the format of AudioStream
      * @throws IOException
      * @throws ExtractionException
@@ -203,6 +220,7 @@ public abstract class StreamExtractor extends Extractor {
      * You can also return null or an empty list, however be aware that if you don't return anything
      * in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as
      * a failed extraction procedure.
+     *
      * @return a list of combined video and streams in the format of AudioStream
      * @throws IOException
      * @throws ExtractionException
@@ -216,6 +234,7 @@ public abstract class StreamExtractor extends Extractor {
      * You can also return null or an empty list, however be aware that if you don't return anything
      * in getAudioStreams(), getVideoStreams() and getDashMpdUrl() either the Collector will handle this as
      * a failed extraction procedure.
+     *
      * @return a list of video and streams in the format of AudioStream
      * @throws IOException
      * @throws ExtractionException
@@ -226,6 +245,7 @@ public abstract class StreamExtractor extends Extractor {
      * This will return a list of available
      * <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/Subtitles.html">Subtitles</a>s.
      * If no subtitles are available an empty list can returned.
+     *
      * @return a list of available subtitles or an empty list
      * @throws IOException
      * @throws ExtractionException
@@ -238,6 +258,7 @@ public abstract class StreamExtractor extends Extractor {
      * <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/Subtitles.html">Subtitles</a>s.
      * given by a specific type.
      * If no subtitles in that specific format are available an empty list can returned.
+     *
      * @param format the media format by which the subtitles should be filtered
      * @return a list of available subtitles or an empty list
      * @throws IOException
@@ -248,15 +269,17 @@ public abstract class StreamExtractor extends Extractor {
 
     /**
      * Get the <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/">StreamType</a>.
+     *
      * @return the type of the stream
      * @throws ParsingException
      */
     public abstract StreamType getStreamType() throws ParsingException;
 
     /**
-     * should return the url of the next stream. NewPipe will automatically play
+     * Should return the url of the next stream. NewPipe will automatically play
      * the next stream if the user wants that.
      * If the next stream is is not available simply return null
+     *
      * @return the InfoItem of the next stream
      * @throws IOException
      * @throws ExtractionException
@@ -268,7 +291,8 @@ public abstract class StreamExtractor extends Extractor {
      * streams. If you don't like suggested streams you should implement them anyway since they can
      * be disabled by the user later in the frontend.
      * This list MUST NOT contain the next available video as this should be return through getNextStream()
-     * If  is is not available simply return null
+     * If it is not available simply return null
+     *
      * @return a list of InfoItems showing the related videos/streams
      * @throws IOException
      * @throws ExtractionException
@@ -277,6 +301,7 @@ public abstract class StreamExtractor extends Extractor {
 
     /**
      * Should return a list of Frameset object that contains preview of stream frames
+     *
      * @return list of preview frames or empty list if frames preview is not supported or not found for specified stream
      * @throws IOException
      * @throws ExtractionException
@@ -299,9 +324,10 @@ public abstract class StreamExtractor extends Extractor {
 
     /**
      * Override this function if the format of time stamp in the url is not the same format as that form youtube.
-     * Honestly I don't even know the time stamp fromat of youtube.
+     * Honestly I don't even know the time stamp format of YouTube.
+     *
      * @param regexPattern
-     * @return the sime stamp/seek for the video in seconds
+     * @return the time stamp/seek for the video in seconds
      * @throws ParsingException
      */
     protected long getTimestampSeconds(String regexPattern) throws ParsingException {
@@ -309,10 +335,10 @@ public abstract class StreamExtractor extends Extractor {
         try {
             timeStamp = Parser.matchGroup1(regexPattern, getOriginalUrl());
         } catch (Parser.RegexException e) {
-            // catch this instantly since an url does not necessarily have to have a time stamp
+            // catch this instantly since a url does not necessarily have a timestamp
 
-            // -2 because well the testing system will then know its the regex that failed :/
+            // -2 because the testing system will consequently know that the regex failed
-            // not good i know
+            // not good, I know
             return -2;
         }
 
@@ -349,4 +375,81 @@ public abstract class StreamExtractor extends Extractor {
             return 0;
         }
     }
+
+    /**
+     * The host of the stream (Eg. peertube.cpy.re).
+     * If the host is not available, or if the service doesn't use
+     * a federated system, but a centralised system,
+     * you can simply return an empty string.
+     *
+     * @return the host of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getHost() throws ParsingException;
+
+    /**
+     * The privacy of the stream (Eg. Public, Private, Unlisted…).
+     * If the privacy is not available you can simply return an empty string.
+     *
+     * @return the privacy of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getPrivacy() throws ParsingException;
+
+    /**
+     * The name of the category of the stream.
+     * If the category is not available you can simply return an empty string.
+     *
+     * @return the category of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getCategory() throws ParsingException;
+
+    /**
+     * The name of the licence of the stream.
+     * If the licence is not available you can simply return an empty string.
+     *
+     * @return the licence of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getLicence() throws ParsingException;
+
+    /**
+     * The locale language of the stream.
+     * If the language is not available you can simply return null.
+     * If the language is provided by a language code, you can return
+     * new Locale(language_code);
+     *
+     * @return the locale language of the stream or null.
+     * @throws ParsingException
+     */
+    @Nullable
+    public abstract Locale getLanguageInfo() throws ParsingException;
+
+    /**
+     * The list of tags of the stream.
+     * If the tag list is not available you can simply return an empty list.
+     *
+     * @return the list of tags of the stream or an empty list.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract List<String> getTags() throws ParsingException;
+
+    /**
+     * The support information of the stream.
+     * see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37
+     * (support button).
+     * If the support information are not available,
+     * you can simply return an empty String.
+     *
+     * @return the support information of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getSupportInfo() throws ParsingException;
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 /*
  * Created by Christian Schabesberger on 26.08.15.
@@ -270,6 +271,43 @@ public class StreamInfo extends Info {
             streamInfo.addError(e);
         }
 
+        //additional info
+        try {
+            streamInfo.setHost(extractor.getHost());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setPrivacy(extractor.getPrivacy());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setCategory(extractor.getCategory());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setLicence(extractor.getLicence());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setLanguageInfo(extractor.getLanguageInfo());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setTags(extractor.getTags());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setSupportInfo(extractor.getSupportInfo());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+
         streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
 
         return streamInfo;
@@ -281,7 +319,7 @@ public class StreamInfo extends Info {
     private DateWrapper uploadDate;
     private long duration = -1;
     private int ageLimit = -1;
-    private String description;
+    private Description description;
 
     private long viewCount = -1;
     private long likeCount = -1;
@@ -308,6 +346,14 @@ public class StreamInfo extends Info {
     private long startPosition = 0;
     private List<SubtitlesStream> subtitles = new ArrayList<>();
 
+    private String host = "";
+    private String privacy = "";
+    private String category = "";
+    private String licence = "";
+    private String support = "";
+    private Locale language = null;
+    private List<String> tags = new ArrayList<>();
+
     /**
      * Get the stream type
      *
@@ -371,11 +417,11 @@ public class StreamInfo extends Info {
         this.ageLimit = ageLimit;
     }
 
-    public String getDescription() {
+    public Description getDescription() {
         return description;
     }
 
-    public void setDescription(String description) {
+    public void setDescription(Description description) {
         this.description = description;
     }
 
@@ -533,4 +579,59 @@ public class StreamInfo extends Info {
         this.subtitles = subtitles;
     }
 
+    public String getHost() {
+        return this.host;
+    }
+
+    public void setHost(String str) {
+        this.host = str;
+    }
+
+    public String getPrivacy() {
+        return this.privacy;
+    }
+
+    public void setPrivacy(String str) {
+        this.privacy = str;
+    }
+
+    public String getCategory() {
+        return this.category;
+    }
+
+    public void setCategory(String cat) {
+        this.category = cat;
+    }
+
+    public String getLicence() {
+        return this.licence;
+    }
+
+    public void setLicence(String str) {
+        this.licence = str;
+    }
+
+    public Locale getLanguageInfo() {
+        return this.language;
+    }
+
+    public void setLanguageInfo(Locale lang) {
+        this.language = lang;
+    }
+
+    public List<String> getTags() {
+        return this.tags;
+    }
+
+    public void setTags(List<String> tags) {
+        this.tags = tags;
+    }
+
+    public void setSupportInfo(String support) {
+        this.support = support;
+    }
+
+    public String getSupportInfo() {
+        return this.support;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java

@@ -31,6 +31,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
 
     /**
      * Get the stream type
+     *
      * @return the stream type
      * @throws ParsingException thrown if there is an error in the extraction
      */
@@ -38,6 +39,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
 
     /**
      * Check if the stream is an ad.
+     *
      * @return {@code true} if the stream is an ad.
      * @throws ParsingException thrown if there is an error in the extraction
      */
@@ -45,6 +47,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
 
     /**
      * Get the stream duration in seconds
+     *
      * @return the stream duration in seconds
      * @throws ParsingException thrown if there is an error in the extraction
      */
@@ -52,6 +55,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
 
     /**
      * Parses the number of views
+     *
      * @return the number of views or -1 for live streams
      * @throws ParsingException thrown if there is an error in the extraction
      */
@@ -59,6 +63,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
 
     /**
      * Get the uploader name
+     *
      * @return the uploader name
      * @throws ParsingException if parsing fails
      */

extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java

@@ -56,6 +56,7 @@ public class VideoStream extends Stream {
 
     /**
      * Get the video resolution
+     *
      * @return the video resolution
      */
     public String getResolution() {
@@ -64,8 +65,9 @@ public class VideoStream extends Stream {
 
     /**
      * Check if the video is video only.
-     *
+     * <p>
      * Video only streams have no audio
+     *
      * @return {@code true} if this stream is vid
      */
     public boolean isVideoOnly() {

extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java

@@ -1,8 +1,8 @@
 package org.schabi.newpipe.extractor.utils;
 
-import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
@@ -109,11 +109,11 @@ public class DashMpdParser {
      * <p>
      * It has video, video only and audio streams and will only add to the list if it don't
      * find a similar stream in the respective lists (calling {@link Stream#equalStats}).
-     *
+     * <p>
      * Info about dash MPD can be found here
-     * @see <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">www.brendanlog.com</a>
      *
      * @param streamInfo where the parsed streams will be added
+     * @see <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">www.brendanlog.com</a>
      */
     public static ParserResult getStreams(final StreamInfo streamInfo)
             throws DashMpdParsingException, ReCaptchaException {

extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java

@@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.InfoItemsCollector;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
-import org.schabi.newpipe.extractor.comments.CommentsInfo;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfo;
 

extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java

@@ -1,18 +1,17 @@
 package org.schabi.newpipe.extractor.utils;
 
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-
-import com.grack.nanojson.JsonArray;
-import com.grack.nanojson.JsonObject;
-
 public class JsonUtils {
+    public static final JsonObject DEFAULT_EMPTY = new JsonObject();
 
     private JsonUtils() {
     }
@@ -38,6 +37,16 @@ public class JsonUtils {
         }
     }
 
+    @Nonnull
+    public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
+        Object value = getValue(object, path);
+        if (value instanceof Boolean) {
+            return (Boolean) value;
+        } else {
+            throw new ParsingException("Unable to get " + path);
+        }
+    }
+
     @Nonnull
     public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
         Object value = getValue(object, path);

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java

@@ -1,5 +1,10 @@
 package org.schabi.newpipe.extractor.utils;
 
+import org.nibor.autolink.LinkExtractor;
+import org.nibor.autolink.LinkSpan;
+import org.nibor.autolink.LinkType;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
 import java.util.ArrayList;
@@ -9,11 +14,6 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.nibor.autolink.LinkExtractor;
-import org.nibor.autolink.LinkSpan;
-import org.nibor.autolink.LinkType;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-
 /*
  * Created by Christian Schabesberger on 02.02.16.
  *

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -1,15 +1,18 @@
 package org.schabi.newpipe.extractor.utils;
 
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.util.List;
 
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-
 public class Utils {
 
+    public static final String HTTP = "http://";
+    public static final String HTTPS = "https://";
+
     private Utils() {
         //no instance
     }
@@ -35,6 +38,7 @@ public class Utils {
      *     <li>1.23K -&gt; 1230</li>
      *     <li>1.23M -&gt; 1230000</li>
      * </ul>
+     *
      * @param numberWord string to be converted to a long
      * @return a long
      * @throws NumberFormatException
@@ -82,9 +86,6 @@ public class Utils {
         }
     }
 
-    private static final String HTTP = "http://";
-    private static final String HTTPS = "https://";
-
     public static String replaceHttpWithHttps(final String url) {
         if (url == null) return null;
 
@@ -185,5 +186,4 @@ public class Utils {
         }
         return uri.getProtocol() + "://" + uri.getAuthority();
     }
-    
 }

extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java

@@ -20,7 +20,7 @@ import java.util.Map;
 
 public class DownloaderTestImpl extends Downloader {
 
-    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
+    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Firefox/68.0";
     private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en";
 
     private static DownloaderTestImpl instance = null;
@@ -99,19 +99,25 @@ public class DownloaderTestImpl extends Downloader {
             final int responseCode = connection.getResponseCode();
             final String responseMessage = connection.getResponseMessage();
             final Map<String, List<String>> responseHeaders = connection.getHeaderFields();
+            final String latestUrl = connection.getURL().toString();
 
-            return new Response(responseCode, responseMessage, responseHeaders, response.toString());
+            return new Response(responseCode, responseMessage, responseHeaders, response.toString(), latestUrl);
         } catch (Exception e) {
+            final int responseCode = connection.getResponseCode();
+
             /*
              * HTTP 429 == Too Many Request
              * Receive from Youtube.com = ReCaptcha challenge request
              * See : https://github.com/rg3/youtube-dl/issues/5138
              */
-            if (connection.getResponseCode() == 429) {
+            if (responseCode == 429) {
                 throw new ReCaptchaException("reCaptcha Challenge requested", url);
+            } else if (responseCode != -1) {
+                final String latestUrl = connection.getURL().toString();
+                return new Response(responseCode, connection.getResponseMessage(), connection.getHeaderFields(), null, latestUrl);
             }
 
-            throw new IOException(connection.getResponseCode() + " " + connection.getResponseMessage(), e);
+            throw new IOException("Error occurred while fetching the content", e);
         } finally {
             if (outputStream != null) outputStream.close();
             if (input != null) input.close();

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java

@@ -2,20 +2,29 @@ package org.schabi.newpipe.extractor.services;
 
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.util.Calendar;
 import java.util.List;
 
+import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.*;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.*;
+import static org.schabi.newpipe.extractor.StreamingService.*;
 
 public final class DefaultTests {
-    public static void defaultTestListOfItems(int expectedServiceId, List<? extends InfoItem> itemsList, List<Throwable> errors) {
+    public static void defaultTestListOfItems(StreamingService expectedService, List<? extends InfoItem> itemsList, List<Throwable> errors) throws ParsingException {
-        assertTrue("List of items is empty", !itemsList.isEmpty());
+        assertFalse("List of items is empty", itemsList.isEmpty());
         assertFalse("List of items contains a null element", itemsList.contains(null));
-        assertEmptyErrors("Errors during stream list extraction", errors);
+        assertEmptyErrors("Errors during extraction", errors);
 
         for (InfoItem item : itemsList) {
             assertIsSecureUrl(item.getUrl());
@@ -23,12 +32,17 @@ public final class DefaultTests {
                 assertIsSecureUrl(item.getThumbnailUrl());
             }
             assertNotNull("InfoItem type not set: " + item, item.getInfoType());
-            assertEquals("Service id doesn't match: " + item, expectedServiceId, item.getServiceId());
+            assertEquals("Unexpected item service id", expectedService.getServiceId(), item.getServiceId());
+            assertNotEmpty("Item name not set: " + item, item.getName());
 
             if (item instanceof StreamInfoItem) {
                 StreamInfoItem streamInfoItem = (StreamInfoItem) item;
                 assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
                 assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
+                assertIsSecureUrl(streamInfoItem.getUploaderUrl());
+
+                assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM);
+                assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL);
 
                 final String textualUploadDate = streamInfoItem.getTextualUploadDate();
                 if (textualUploadDate != null && !textualUploadDate.isEmpty()) {
@@ -37,34 +51,56 @@ public final class DefaultTests {
                     assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance()));
                 }
 
+            } else if (item instanceof ChannelInfoItem) {
+                final ChannelInfoItem channelInfoItem = (ChannelInfoItem) item;
+                assertExpectedLinkType(expectedService, channelInfoItem.getUrl(), LinkType.CHANNEL);
+
+            } else if (item instanceof PlaylistInfoItem) {
+                final PlaylistInfoItem playlistInfoItem = (PlaylistInfoItem) item;
+                assertExpectedLinkType(expectedService, playlistInfoItem.getUrl(), LinkType.PLAYLIST);
             }
         }
     }
 
-    public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestRelatedItems(ListExtractor<T> extractor, int expectedServiceId) throws Exception {
+    private static void assertExpectedLinkType(StreamingService expectedService, String url, LinkType expectedLinkType) throws ParsingException {
+        final LinkType linkTypeByUrl = expectedService.getLinkTypeByUrl(url);
+
+        assertNotEquals("Url is not recognized by its own service: \"" + url + "\"",
+                LinkType.NONE, linkTypeByUrl);
+        assertEquals("Service returned wrong link type for: \"" + url + "\"",
+                expectedLinkType, linkTypeByUrl);
+    }
+
+    public static <T extends InfoItem> void assertNoMoreItems(ListExtractor<T> extractor) throws Exception {
+        assertFalse("More items available when it shouldn't", extractor.hasNextPage());
+        final String nextPageUrl = extractor.getNextPageUrl();
+        assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty());
+    }
+
+    public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestRelatedItems(ListExtractor<T> extractor) throws Exception {
         final ListExtractor.InfoItemsPage<T> page = extractor.getInitialPage();
         final List<T> itemsList = page.getItems();
         List<Throwable> errors = page.getErrors();
 
-        defaultTestListOfItems(expectedServiceId, itemsList, errors);
+        defaultTestListOfItems(extractor.getService(), itemsList, errors);
         return page;
     }
 
-    public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestMoreItems(ListExtractor<T> extractor, int expectedServiceId) throws Exception {
+    public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestMoreItems(ListExtractor<T> extractor) throws Exception {
         assertTrue("Doesn't have more items", extractor.hasNextPage());
         ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(extractor.getNextPageUrl());
         final List<T> items = nextPage.getItems();
-        assertTrue("Next page is empty", !items.isEmpty());
+        assertFalse("Next page is empty", items.isEmpty());
         assertEmptyErrors("Next page have errors", nextPage.getErrors());
 
-        defaultTestListOfItems(expectedServiceId, nextPage.getItems(), nextPage.getErrors());
+        defaultTestListOfItems(extractor.getService(), nextPage.getItems(), nextPage.getErrors());
         return nextPage;
     }
 
-    public static void defaultTestGetPageInNewExtractor(ListExtractor<? extends InfoItem> extractor, ListExtractor<? extends InfoItem> newExtractor, int expectedServiceId) throws Exception {
+    public static void defaultTestGetPageInNewExtractor(ListExtractor<? extends InfoItem> extractor, ListExtractor<? extends InfoItem> newExtractor) throws Exception {
         final String nextPageUrl = extractor.getNextPageUrl();
 
         final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPageUrl);
-        defaultTestListOfItems(expectedServiceId, page.getItems(), page.getErrors());
+        defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java

@@ -4,22 +4,24 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
 
 import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
 
 /**
  * Test {@link MediaCCCConferenceExtractor}
  */
 public class MediaCCCConferenceExtractorTest {
-    private static ChannelExtractor extractor;
+
+    public static class FrOSCon2017 {
+        private static MediaCCCConferenceExtractor extractor;
 
         @BeforeClass
         public static void setUpClass() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
-        extractor = MediaCCC.getChannelExtractor("https://api.media.ccc.de/public/conferences/froscon2017");
+            extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/froscon2017");
             extractor.fetchPage();
         }
 
@@ -48,3 +50,40 @@ public class MediaCCCConferenceExtractorTest {
             assertEquals(97, extractor.getInitialPage().getItems().size());
         }
     }
+
+    public static class Oscal2019 {
+        private static MediaCCCConferenceExtractor extractor;
+
+        @BeforeClass
+        public static void setUpClass() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/oscal19");
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testName() throws Exception {
+            assertEquals("Open Source Conference Albania 2019", extractor.getName());
+        }
+
+        @Test
+        public void testGetUrl() throws Exception {
+            assertEquals("https://api.media.ccc.de/public/conferences/oscal19", extractor.getUrl());
+        }
+
+        @Test
+        public void testGetOriginalUrl() throws Exception {
+            assertEquals("https://media.ccc.de/c/oscal19", extractor.getOriginalUrl());
+        }
+
+        @Test
+        public void testGetThumbnailUrl() throws Exception {
+            assertEquals("https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", extractor.getAvatarUrl());
+        }
+
+        @Test
+        public void testGetInitalPage() throws Exception {
+            assertTrue(extractor.getInitialPage().getItems().size() >= 21);
+        }
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java

@@ -6,59 +6,67 @@ import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.services.BaseExtractorTest;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
-import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.utils.UtilsTest;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.List;
 
 import static java.util.Objects.requireNonNull;
 import static junit.framework.TestCase.assertEquals;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
 
 /**
  * Test {@link MediaCCCStreamExtractor}
  */
-public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
+public class MediaCCCStreamExtractorTest {
-    private static StreamExtractor extractor;
+
+    public static class Gpn18Tmux {
+        private static MediaCCCStreamExtractor extractor;
 
         @BeforeClass
         public static void setUpClass() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
 
-        extractor =  MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d");
+            extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht");
             extractor.fetchPage();
         }
 
-    @Override
+        @Test
         public void testServiceId() throws Exception {
             assertEquals(2, extractor.getServiceId());
         }
 
-    @Override
+        @Test
         public void testName() throws Exception {
             assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName());
         }
 
-    @Override
+        @Test
         public void testId() throws Exception {
-        assertEquals("", extractor.getId());
+            assertEquals("gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getId());
         }
 
-    @Override
+        @Test
         public void testUrl() throws Exception {
-        assertEquals("", extractor.getUrl());
+            assertIsSecureUrl(extractor.getUrl());
+            assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl());
         }
 
-    @Override
+        @Test
         public void testOriginalUrl() throws Exception {
-        assertEquals("", extractor.getOriginalUrl());
+            assertIsSecureUrl(extractor.getOriginalUrl());
+            assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl());
         }
 
         @Test
         public void testThumbnail() throws Exception {
+            assertIsSecureUrl(extractor.getThumbnailUrl());
             assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl());
         }
 
@@ -69,22 +77,32 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
 
         @Test
         public void testUploaderUrl() throws Exception {
+            assertIsSecureUrl(extractor.getUploaderUrl());
             assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl());
         }
 
         @Test
         public void testUploaderAvatarUrl() throws Exception {
+            assertIsSecureUrl(extractor.getUploaderAvatarUrl());
             assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl());
         }
 
         @Test
         public void testVideoStreams() throws Exception {
-        assertEquals(4, extractor.getVideoStreams().size());
+            List<VideoStream> videoStreamList = extractor.getVideoStreams();
+            assertEquals(4, videoStreamList.size());
+            for (VideoStream stream : videoStreamList) {
+                assertIsSecureUrl(stream.getUrl());
+            }
         }
 
         @Test
         public void testAudioStreams() throws Exception {
-        assertEquals(2, extractor.getAudioStreams().size());
+            List<AudioStream> audioStreamList = extractor.getAudioStreams();
+            assertEquals(2, audioStreamList.size());
+            for (AudioStream stream : audioStreamList) {
+                assertIsSecureUrl(stream.getUrl());
+            }
         }
 
         @Test
@@ -99,3 +117,90 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
             assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
         }
     }
+
+    public static class _36c3PrivacyMessaging {
+        private static MediaCCCStreamExtractor extractor;
+
+        @BeforeClass
+        public static void setUpClass() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging");
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testName() throws Exception {
+            assertEquals("What's left for private messaging?", extractor.getName());
+        }
+
+        @Test
+        public void testId() throws Exception {
+            assertEquals("36c3-10565-what_s_left_for_private_messaging", extractor.getId());
+        }
+
+        @Test
+        public void testUrl() throws Exception {
+            assertIsSecureUrl(extractor.getUrl());
+            assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl());
+        }
+
+        @Test
+        public void testOriginalUrl() throws Exception {
+            assertIsSecureUrl(extractor.getOriginalUrl());
+            assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl());
+        }
+
+        @Test
+        public void testThumbnail() throws Exception {
+            assertIsSecureUrl(extractor.getThumbnailUrl());
+            assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl());
+        }
+
+        @Test
+        public void testUploaderName() throws Exception {
+            assertEquals("36c3", extractor.getUploaderName());
+        }
+
+        @Test
+        public void testUploaderUrl() throws Exception {
+            assertIsSecureUrl(extractor.getUploaderUrl());
+            assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl());
+        }
+
+        @Test
+        public void testUploaderAvatarUrl() throws Exception {
+            assertIsSecureUrl(extractor.getUploaderAvatarUrl());
+            assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl());
+        }
+
+        @Test
+        public void testVideoStreams() throws Exception {
+            List<VideoStream> videoStreamList = extractor.getVideoStreams();
+            assertEquals(8, videoStreamList.size());
+            for (VideoStream stream : videoStreamList) {
+                assertIsSecureUrl(stream.getUrl());
+            }
+        }
+
+        @Test
+        public void testAudioStreams() throws Exception {
+            List<AudioStream> audioStreamList = extractor.getAudioStreams();
+            assertEquals(2, audioStreamList.size());
+            for (AudioStream stream : audioStreamList) {
+                assertIsSecureUrl(stream.getUrl());
+            }
+        }
+
+        @Test
+        public void testGetTextualUploadDate() throws ParsingException {
+            Assert.assertEquals("2020-01-11T01:00:00.000+01:00", extractor.getTextualUploadDate());
+        }
+
+        @Test
+        public void testGetUploadDate() throws ParsingException, ParseException {
+            final Calendar instance = Calendar.getInstance();
+            instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-11"));
+            assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
+        }
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java

@@ -1,15 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
-
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -20,6 +10,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
 
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.*;
+
 /**
  * Test for {@link PeertubeChannelExtractor}
  */
@@ -72,12 +68,12 @@ public class PeertubeChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, PeerTube.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, PeerTube.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -131,7 +127,7 @@ public class PeertubeChannelExtractorTest {
         @Test
         public void testGetPageInNewExtractor() throws Exception {
             final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
-            defaultTestGetPageInNewExtractor(extractor, newExtractor, PeerTube.getServiceId());
+            defaultTestGetPageInNewExtractor(extractor, newExtractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -169,12 +165,12 @@ public class PeertubeChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, PeerTube.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, PeerTube.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -204,7 +200,7 @@ public class PeertubeChannelExtractorTest {
 
         @Test
         public void testSubscriberCount() throws ParsingException {
-            assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
+            assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1);
         }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java

@@ -1,8 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
@@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 /**
  * Test for {@link PeertubeChannelLinkHandlerFactory}
  */

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java

@@ -1,12 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
-
-import java.io.IOException;
-import java.util.List;
-
 import org.jsoup.helper.StringUtil;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -18,6 +11,13 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
 
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+
 public class PeertubeCommentsExtractorTest {
 
     private static PeertubeCommentsExtractor extractor;
@@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest {
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
         extractor = (PeertubeCommentsExtractor) PeerTube
-                .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
+                .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
     }
 
     @Test
@@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest {
     @Test
     public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
         boolean result = false;
-        CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
+        CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
         assertTrue("Comments".equals(commentsInfo.getName()));
         result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java

@@ -1,8 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
@@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 /**
  * Test for {@link PeertubeCommentsLinkHandlerFactory}
  */

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java

@@ -1,8 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
@@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 /**
  * Test for {@link PeertubePlaylistLinkHandlerFactory}
  */

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java

@@ -1,17 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static java.util.Objects.requireNonNull;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -24,21 +12,45 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.TimeZone;
+
+import static java.util.Objects.requireNonNull;
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+
 /**
  * Test for {@link StreamExtractor}
  */
 public class PeertubeStreamExtractorDefaultTest {
     private static PeertubeStreamExtractor extractor;
+    private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/";
+    private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary";
 
     @BeforeClass
     public static void setUp() throws Exception {
         NewPipe.init(DownloaderTestImpl.getInstance());
         // setting instance might break test when running in parallel
-        PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
+        PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube"));
-        extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
+        extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
         extractor.fetchPage();
     }
 
+    @Test
+    public void testGetUploadDate() throws ParsingException, ParseException {
+        final Calendar instance = Calendar.getInstance();
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
+        sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+        instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z"));
+        assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
+
+    }
+
     @Test
     public void testGetInvalidTimeStamp() throws ParsingException {
         assertTrue(extractor.getTimeStamp() + "",
@@ -47,22 +59,37 @@ public class PeertubeStreamExtractorDefaultTest {
 
     @Test
     public void testGetTitle() throws ParsingException {
-        assertEquals(extractor.getName(), "Power Corrupts the Best");
+        assertEquals("What is PeerTube?", extractor.getName());
     }
 
     @Test
-    public void testGetDescription() throws ParsingException {
+    public void testGetLargeDescription() throws ParsingException {
-        assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
+        assertEquals(expectedLargeDescription, extractor.getDescription().getContent());
+    }
+
+    @Test
+    public void testGetEmptyDescription() throws Exception {
+        PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d");
+        extractorEmpty.fetchPage();
+        assertEquals("", extractorEmpty.getDescription().getContent());
+    }
+
+    @Test
+    public void testGetSmallDescription() throws Exception {
+        PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server"));
+        PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea");
+        extractorSmall.fetchPage();
+        assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent());
     }
 
     @Test
     public void testGetUploaderName() throws ParsingException {
-        assertEquals(extractor.getUploaderName(), "Rowsedower");
+        assertEquals("Framasoft", extractor.getUploaderName());
     }
 
     @Test
     public void testGetLength() throws ParsingException {
-        assertEquals(extractor.getLength(), 269);
+        assertEquals(113, extractor.getLength());
     }
 
     @Test
@@ -71,18 +98,10 @@ public class PeertubeStreamExtractorDefaultTest {
                 extractor.getViewCount() > 10);
     }
 
-    @Test
-    public void testGetUploadDate() throws ParsingException, ParseException {
-        final Calendar instance = Calendar.getInstance();
-        instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z"));
-        assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
-
-    }
-
     @Test
     public void testGetUploaderUrl() throws ParsingException {
         assertIsSecureUrl(extractor.getUploaderUrl());
-        assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
+        assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl());
     }
 
     @Test
@@ -115,11 +134,31 @@ public class PeertubeStreamExtractorDefaultTest {
 
     @Test
     public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
-        assertTrue(extractor.getSubtitlesDefault().isEmpty());
+        assertFalse(extractor.getSubtitlesDefault().isEmpty());
     }
 
     @Test
     public void testGetSubtitlesList() throws IOException, ExtractionException {
-        assertTrue(extractor.getSubtitlesDefault().isEmpty());
+        assertFalse(extractor.getSubtitlesDefault().isEmpty());
+    }
+
+    @Test
+    public void testGetAgeLimit() throws ExtractionException, IOException {
+        assertEquals(0, extractor.getAgeLimit());
+        PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/3c0da7fb-e4d9-442e-84e3-a8c47004ee28");
+        ageLimit.fetchPage();
+        assertEquals(18, ageLimit.getAgeLimit());
+    }
+
+    @Test
+    public void testGetSupportInformation() throws ExtractionException, IOException {
+        PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37");
+        supportInfoExtractor.fetchPage();
+        assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo());
+    }
+
+    @Test
+    public void testGetLanguageInformation() throws ParsingException {
+        assertEquals(new Locale("en"), extractor.getLanguageInfo());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java

@@ -1,8 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
@@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 /**
  * Test for {@link PeertubeStreamLinkHandlerFactory}
  */

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java

@@ -1,97 +1,79 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
-
-import java.util.List;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
+import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudChartsExtractor;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
-/**
+import java.util.List;
- * Test for {@link PeertubeTrendingExtractor}
- */
-public class PeertubeTrendingExtractorTest {
 
-    static KioskExtractor extractor;
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ServiceList.*;
+import static org.schabi.newpipe.extractor.services.DefaultTests.*;
+
+public class PeertubeTrendingExtractorTest {
+    public static class Trending implements BaseListExtractorTest {
+        private static PeertubeTrendingExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
             // setting instance might break test when running in parallel
             PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
-        extractor = PeerTube
+            extractor = (PeertubeTrendingExtractor) PeerTube.getKioskList()
-                .getKioskList()
                     .getExtractorById("Trending", null);
             extractor.fetchPage();
         }
 
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
         @Test
-    public void testGetDownloader() throws Exception {
+        public void testServiceId() {
-        assertNotNull(NewPipe.getDownloader());
+            assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
         }
 
         @Test
-    public void testGetName() throws Exception {
+        public void testName() throws Exception {
-        assertEquals(extractor.getName(), "Trending");
+            assertEquals("Trending", extractor.getName());
         }
 
         @Test
-    public void testId() {
+        public void testId() throws Exception {
-        assertEquals(extractor.getId(), "Trending");
+            assertEquals("Trending", extractor.getId());
         }
 
         @Test
-    public void testGetStreams() throws Exception {
+        public void testUrl() throws ParsingException {
-        ListExtractor.InfoItemsPage<StreamInfoItem> page = extractor.getInitialPage();
+            assertEquals("https://peertube.mastodon.host/api/v1/videos?sort=-trending", extractor.getUrl());
-        if(!page.getErrors().isEmpty()) {
-            System.err.println("----------");
-            List<Throwable> errors = page.getErrors();
-            for(Throwable e: errors) {
-                e.printStackTrace();
-                System.err.println("----------");
-            }
-        }
-        assertTrue("no streams are received",
-                !page.getItems().isEmpty()
-                        && page.getErrors().isEmpty());
         }
 
         @Test
-    public void testGetStreamsErrors() throws Exception {
+        public void testOriginalUrl() throws ParsingException {
-        assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty());
+            assertEquals("https://peertube.mastodon.host/api/v1/videos?sort=-trending", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ListExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
-    public void testHasMoreStreams() throws Exception {
+        public void testMoreRelatedItems() throws Exception {
-        // Setup the streams
+            defaultTestMoreItems(extractor);
-        extractor.getInitialPage();
+        }
-        assertTrue("has more streams", extractor.hasNextPage());
-    }
-
-    @Test
-    public void testGetNextPageUrl() throws Exception {
-        assertTrue(extractor.hasNextPage());
-    }
-
-    @Test
-    public void testGetNextPage() throws Exception {
-        extractor.getInitialPage().getItems();
-        assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
-                || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty());
-    }
-
-    @Test
-    public void testGetCleanUrl() throws Exception {
-        assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending");
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java

@@ -1,9 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
@@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+
 /**
  * Test for {@link PeertubeTrendingLinkHandlerFactory}
  */

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java

@@ -1,12 +1,12 @@
 package org.schabi.newpipe.extractor.services.peertube.search;
 
-import static org.junit.Assert.assertTrue;
-
 import org.junit.Test;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
 
+import static org.junit.Assert.assertTrue;
+
 /**
  * Test for {@link PeertubeSearchExtractor}
  */

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java

@@ -1,10 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube.search;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
@@ -15,6 +10,9 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+
 /**
  * Test for {@link PeertubeSearchExtractor}
  */

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java

@@ -1,12 +1,12 @@
 package org.schabi.newpipe.extractor.services.peertube.search;
 
-import static org.junit.Assert.assertEquals;
-import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
 
+import static org.junit.Assert.assertEquals;
+import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+
 public class PeertubeSearchQHTest {
 
     @BeforeClass

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java

@@ -64,12 +64,12 @@ public class SoundcloudChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, SoundCloud.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -120,7 +120,7 @@ public class SoundcloudChannelExtractorTest {
         @Test
         public void testGetPageInNewExtractor() throws Exception {
             final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl());
-            defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
+            defaultTestGetPageInNewExtractor(extractor, newExtractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -158,12 +158,12 @@ public class SoundcloudChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, SoundCloud.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java

@@ -1,92 +1,131 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.util.List;
 
 import static org.junit.Assert.*;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.*;
 
-/**
- * Test for {@link SoundcloudChartsLinkHandlerFactory}
- */
 public class SoundcloudChartsExtractorTest {
-
+    public static class NewAndHot implements BaseListExtractorTest {
-    static KioskExtractor extractor;
+        private static SoundcloudChartsExtractor extractor;
 
         @BeforeClass
         public static void setUp() throws Exception {
             NewPipe.init(DownloaderTestImpl.getInstance());
-        extractor = SoundCloud
+            extractor = (SoundcloudChartsExtractor) SoundCloud.getKioskList()
-                .getKioskList()
+                    .getExtractorById("New & hot", null);
-                .getExtractorById("Top 50", null);
             extractor.fetchPage();
         }
 
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
         @Test
-    public void testGetDownloader() throws Exception {
+        public void testServiceId() {
-        assertNotNull(NewPipe.getDownloader());
+            assertEquals(SoundCloud.getServiceId(), extractor.getServiceId());
         }
 
         @Test
-    public void testGetName() throws Exception {
+        public void testName() {
-        assertEquals(extractor.getName(), "Top 50");
+            assertEquals("New & hot", extractor.getName());
         }
 
         @Test
         public void testId() {
-        assertEquals(extractor.getId(), "Top 50");
+            assertEquals("New & hot", extractor.getId());
         }
 
         @Test
-    public void testGetStreams() throws Exception {
+        public void testUrl() throws ParsingException {
-        ListExtractor.InfoItemsPage<StreamInfoItem> page = extractor.getInitialPage();
+            assertEquals("https://soundcloud.com/charts/new", extractor.getUrl());
-        if(!page.getErrors().isEmpty()) {
-            System.err.println("----------");
-            List<Throwable> errors = page.getErrors();
-            for(Throwable e: errors) {
-                e.printStackTrace();
-                System.err.println("----------");
-            }
-        }
-        assertTrue("no streams are received",
-                !page.getItems().isEmpty()
-                        && page.getErrors().isEmpty());
         }
 
         @Test
-    public void testGetStreamsErrors() throws Exception {
+        public void testOriginalUrl() throws ParsingException {
-        assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty());
+            assertEquals("https://soundcloud.com/charts/new", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ListExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
-    public void testHasMoreStreams() throws Exception {
+        public void testMoreRelatedItems() throws Exception {
-        // Setup the streams
+            defaultTestMoreItems(extractor);
-        extractor.getInitialPage();
+        }
-        assertTrue("has more streams", extractor.hasNextPage());
+    }
+
+    public static class Top50Charts implements BaseListExtractorTest {
+        private static SoundcloudChartsExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = (SoundcloudChartsExtractor) SoundCloud.getKioskList()
+                    .getExtractorById("Top 50", null);
+            extractor.fetchPage();
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testServiceId() {
+            assertEquals(SoundCloud.getServiceId(), extractor.getServiceId());
         }
 
         @Test
-    public void testGetNextPageUrl() throws Exception {
+        public void testName() {
-        assertTrue(extractor.hasNextPage());
+            assertEquals("Top 50", extractor.getName());
         }
 
         @Test
-    public void testGetNextPage() throws Exception {
+        public void testId() {
-        extractor.getInitialPage().getItems();
+            assertEquals("Top 50", extractor.getId());
-        assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
-                || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty());
         }
 
         @Test
-    public void testGetCleanUrl() throws Exception {
+        public void testUrl() throws ParsingException {
-        assertEquals(extractor.getUrl(), "https://soundcloud.com/charts/top");
+            assertEquals("https://soundcloud.com/charts/top", extractor.getUrl());
+        }
+
+        @Test
+        public void testOriginalUrl() throws ParsingException {
+            assertEquals("https://soundcloud.com/charts/top", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ListExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor);
+        }
+
+        @Test
+        public void testMoreRelatedItems() throws Exception {
+            defaultTestMoreItems(extractor);
+        }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java

@@ -1,10 +1,12 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.junit.*;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
 
 public class SoundcloudParsingHelperTest {
     @BeforeClass

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -7,7 +7,6 @@ import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@@ -67,13 +66,13 @@ public class SoundcloudPlaylistExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() {
             try {
-                defaultTestMoreItems(extractor, SoundCloud.getServiceId());
+                defaultTestMoreItems(extractor);
             } catch (Throwable ignored) {
                 return;
             }
@@ -165,12 +164,12 @@ public class SoundcloudPlaylistExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, SoundCloud.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -226,11 +225,10 @@ public class SoundcloudPlaylistExtractorTest {
         // Additional Testing
         //////////////////////////////////////////////////////////////////////////*/
 
-        @Ignore
         @Test
         public void testGetPageInNewExtractor() throws Exception {
             final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
-            defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
+            defaultTestGetPageInNewExtractor(extractor, newExtractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -269,18 +267,18 @@ public class SoundcloudPlaylistExtractorTest {
         @Ignore
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         //TODO: FUCK THIS: This triggers a 500 at sever
         @Ignore
         @Test
         public void testMoreRelatedItems() throws Exception {
-            ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId());
+            ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor);
             // Test for 2 more levels
             for (int i = 0; i < 2; i++) {
                 currentPage = extractor.getPage(currentPage.getNextPageUrl());
-                defaultTestListOfItems(SoundCloud.getServiceId(), currentPage.getItems(), currentPage.getErrors());
+                defaultTestListOfItems(SoundCloud, currentPage.getItems(), currentPage.getErrors());
             }
         }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java

@@ -53,7 +53,7 @@ public class SoundcloudStreamExtractorDefaultTest {
 
     @Test
     public void testGetDescription() throws ParsingException {
-        assertEquals("The Perfect LUV Tape®️", extractor.getDescription());
+        assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -1,29 +1,46 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
-import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
 
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.DefaultTests.*;
+
 /**
  * Test for {@link ChannelExtractor}
  */
 public class YoutubeChannelExtractorTest {
+
+    public static class NotAvailable {
+        @BeforeClass
+        public static void setUp() {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+        }
+
+        @Test(expected = ContentNotAvailableException.class)
+        public void deletedFetch() throws Exception {
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCAUc4iz6edWerIjlnL8OSSw");
+            extractor.fetchPage();
+        }
+
+        @Test(expected = ContentNotAvailableException.class)
+        public void nonExistentFetch() throws Exception {
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST");
+            extractor.fetchPage();
+        }
+    }
+
     public static class Gronkh implements BaseChannelExtractorTest {
         private static YoutubeChannelExtractor extractor;
 
@@ -70,12 +87,12 @@ public class YoutubeChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
          /*//////////////////////////////////////////////////////////////////////////
@@ -160,12 +177,12 @@ public class YoutubeChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
          /*//////////////////////////////////////////////////////////////////////////
@@ -175,7 +192,7 @@ public class YoutubeChannelExtractorTest {
         @Test
         public void testDescription() throws Exception {
             assertTrue("What it actually was: " + extractor.getDescription(),
-                    extractor.getDescription().contains("Our World is Amazing. Questions? Ideas? Tweet me:"));
+                    extractor.getDescription().contains("Our World is Amazing. \n\nQuestions? Ideas? Tweet me:"));
         }
 
         @Test
@@ -223,7 +240,7 @@ public class YoutubeChannelExtractorTest {
         @Test
         public void testGetPageInNewExtractor() throws Exception {
             final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
-            defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId());
+            defaultTestGetPageInNewExtractor(extractor, newExtractor);
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -262,12 +279,12 @@ public class YoutubeChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
          /*//////////////////////////////////////////////////////////////////////////
@@ -353,12 +370,12 @@ public class YoutubeChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
          /*//////////////////////////////////////////////////////////////////////////
@@ -443,12 +460,12 @@ public class YoutubeChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() throws Exception {
-            defaultTestMoreItems(extractor, YouTube.getServiceId());
+            defaultTestMoreItems(extractor);
         }
 
          /*//////////////////////////////////////////////////////////////////////////
@@ -489,7 +506,6 @@ public class YoutubeChannelExtractorTest {
     }
 
 
-
     public static class RandomChannel implements BaseChannelExtractorTest {
         private static YoutubeChannelExtractor extractor;
 
@@ -536,13 +552,13 @@ public class YoutubeChannelExtractorTest {
 
         @Test
         public void testRelatedItems() throws Exception {
-            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+            defaultTestRelatedItems(extractor);
         }
 
         @Test
         public void testMoreRelatedItems() {
             try {
-                defaultTestMoreItems(extractor, YouTube.getServiceId());
+                defaultTestMoreItems(extractor);
             } catch (Throwable ignored) {
                 return;
             }