Merge pull request #27 from TeamNewPipe/kiosk

add Kiosk support

src/main/java/org/schabi/newpipe/extractor/InfoItem.java

@@ -1,7 +1,7 @@
 package org.schabi.newpipe.extractor;
 
 /*
- * Created by the-scrabi on 11.02.17.
+ * Created by Christian Schabesberger on 11.02.17.
  *
  * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
  * InfoItem.java is part of NewPipe.

src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor;
 
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.kiosk.KioskList;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchEngine;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
@@ -48,6 +49,7 @@ public abstract class StreamingService {
     public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
     public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
     public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
+    public abstract KioskList getKioskList() throws ExtractionException;
 
     public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
         return getChannelExtractor(url, null);

src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java

@@ -0,0 +1,71 @@
+package org.schabi.newpipe.extractor.kiosk;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * KioskExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.io.IOException;
+
+public abstract class KioskExtractor extends ListExtractor {
+    private String contentCountry = null;
+
+    public KioskExtractor(StreamingService streamingService,
+                          String url,
+                          String nextStreamsUrl)
+        throws IOException, ExtractionException {
+        super(streamingService, url, nextStreamsUrl);
+        this.contentCountry = contentCountry;
+    }
+
+    /**
+     * For certain Websites the content of a kiosk will be different depending
+     * on the country you want to poen the website in. Therefore you should
+     * set the contentCountry.
+     * @param contentCountry Set the country decoded as Country Code: http://www.1728.org/countries.htm
+     */
+    public void setContentCountry(String contentCountry) {
+        this.contentCountry = contentCountry;
+    }
+
+    /**
+     * Returns the type of the kiosk.
+     * eg. Trending, Top & Hot, Top last 24 hours
+     * @return type of kiosk
+     */
+    public abstract String getType() throws ParsingException;
+
+    @Override
+    public String getId() throws ParsingException {
+        return getType();
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return getType();
+    }
+
+    public String getContentCountry() {
+        return contentCountry;
+    }
+}

src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java

@@ -0,0 +1,74 @@
+package org.schabi.newpipe.extractor.kiosk;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * KioskInfo.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
+
+import java.io.IOException;
+
+public class KioskInfo extends ListInfo {
+    public String type;
+
+    public static KioskInfo getInfo(String url,
+                                    String contentCountry) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url, contentCountry);
+    }
+
+    public static KioskInfo getInfo(ServiceList serviceItem,
+                                    String url,
+                                    String contentContry) throws IOException, ExtractionException {
+        return getInfo(serviceItem.getService(), url, contentContry);
+    }
+
+    public static KioskInfo getInfo(StreamingService service,
+                                    String url,
+                                    String contentCountry) throws IOException, ExtractionException {
+        KioskList kl = service.getKioskList();
+        KioskExtractor extractor = kl.getExtryctorByUrl(url);
+        return getInfo(extractor, contentCountry);
+    }
+
+    public static KioskInfo getInfo(KioskExtractor extractor,
+                                    String contentCountry) throws IOException, ExtractionException {
+        KioskInfo info = new KioskInfo();
+        extractor.setContentCountry(contentCountry);
+        extractor.fetchPage();
+        info.type = extractor.getType();
+        info.name = extractor.getName();
+        info.id = extractor.getId();
+
+        try {
+            StreamInfoItemCollector c = extractor.getStreams();
+            info.related_streams = c.getItemList();
+            info.errors.addAll(c.getErrors());
+        } catch (Exception e) {
+            info.errors.add(e);
+        }
+
+        return info;
+    }
+}

src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java

@@ -0,0 +1,57 @@
+package org.schabi.newpipe.extractor.kiosk;
+
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public  class KioskList {
+    private int service_id;
+    private HashMap<String, KioskEntry> kioskList = new HashMap<>();
+
+    private class KioskEntry {
+        public KioskEntry(KioskExtractor e, UrlIdHandler h) {
+            extractor = e;
+            handler = h;
+        }
+        KioskExtractor extractor;
+        UrlIdHandler handler;
+    }
+
+    public KioskList(int service_id) {
+        this.service_id = service_id;
+    }
+
+    public void addKioskEntry(KioskExtractor extractor, UrlIdHandler handler)
+        throws Exception {
+        if(kioskList.get(extractor.getType()) != null) {
+            throw new Exception("Kiosk with type " + extractor.getType() + " already exists.");
+        }
+        kioskList.put(extractor.getType(), new KioskEntry(extractor, handler));
+    }
+
+    public KioskExtractor getExtractorByType(String kioskType) throws ExtractionException {
+        KioskEntry ke = kioskList.get(kioskType);
+        if(ke == null) {
+            throw new ExtractionException("No kiosk found with the type: " + kioskType);
+        } else {
+            return ke.extractor;
+        }
+    }
+
+    public Set<String> getAvailableKisokTypes() {
+        return kioskList.keySet();
+    }
+
+    public KioskExtractor getExtryctorByUrl(String url) throws ExtractionException {
+        for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
+            KioskEntry ke = e.getValue();
+            if(ke.handler.acceptUrl(url)) {
+                return getExtractorByType(e.getKey());
+            }
+        }
+        throw new ExtractionException("Could not find a kiosk that fits to the url: " + url);
+    }
+}

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java

@@ -0,0 +1,71 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
+
+public class SoundcloudChartsExtractor extends KioskExtractor {
+	private String url;
+
+    public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        super(service, url, nextStreamsUrl);
+        this.url = url;
+    }
+
+    @Override
+    public void fetchPage() {
+    }
+
+    @Override
+    public String getType() throws ParsingException {
+        return getUrlIdHandler().getId(url);
+    }
+
+    @Override
+    public UrlIdHandler getUrlIdHandler() {
+        return new SoundcloudChartsUrlIdHandler();
+    }
+
+    @Override
+    public NextItemsResult getNextStreams() throws IOException, ExtractionException {
+        if (!hasMoreStreams()) {
+            throw new ExtractionException("Chart doesn't have more streams");
+        }
+
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
+        nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true);
+
+        return new NextItemsResult(collector, nextStreamsUrl);
+    }
+
+    @Override
+    public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
+
+        String apiUrl = "https://api-v2.soundcloud.com/charts" +
+                "?genre=soundcloud:genres:all-music" +
+                "&client_id=" + SoundcloudParsingHelper.clientId();
+
+        if (getType().equals("Top 50")) {
+            apiUrl += "&kind=top";
+        } else {
+            apiUrl += "&kind=new";
+        }
+
+        List<String> supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US");
+        String contentCountry = getContentCountry();
+        if (supportedCountries.contains(contentCountry)) {
+            apiUrl += "&region=soundcloud:regions:" + contentCountry;
+        }
+
+        nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
+        return collector;
+    }
+}

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java

@@ -0,0 +1,37 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
+    public String getUrl(String id) {
+        if (id.equals("Top 50")) {
+            return "https://soundcloud.com/charts/top";
+        } else {
+            return "https://soundcloud.com/charts/new";
+        }
+    }
+
+    @Override
+    public String getId(String url) {
+        if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
+            return "Top 50";
+        } else {
+            return "New & hot";
+        }
+    }
+
+    @Override
+    public String cleanUrl(String url) {
+        if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
+            return "https://soundcloud.com/charts/top";
+        } else {
+            return "https://soundcloud.com/charts/new";
+        }
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        return Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$", url.toLowerCase());
+    }
+}

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -127,7 +127,7 @@ public class SoundcloudParsingHelper {
      *
      * @return the next streams url, empty if don't have
      */
-    public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
+    public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
         String response = NewPipe.getDownloader().download(apiUrl);
         JsonObject responseObject;
         try {
@@ -151,4 +151,8 @@ public class SoundcloudParsingHelper {
 
         return nextStreamsUrl;
     }
+
+    public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
+        return getStreamsFromApi(collector, apiUrl, false);
+    }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -1,16 +1,17 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
+import java.io.IOException;
+
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.SuggestionExtractor;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.kiosk.KioskList;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchEngine;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 
-import java.io.IOException;
-
 public class SoundcloudService extends StreamingService {
 
     public SoundcloudService(int id, String name) {
@@ -57,4 +58,20 @@ public class SoundcloudService extends StreamingService {
     public SuggestionExtractor getSuggestionExtractor() {
         return new SoundcloudSuggestionExtractor(getServiceId());
     }
+
+    @Override
+    public KioskList getKioskList() throws ExtractionException {
+        KioskList list = new KioskList(getServiceId());
+
+        // add kiosks here e.g.:
+        SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler();
+        try {
+            list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("Top 50"), null), h);
+            list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("New & hot"), null), h);
+        } catch (Exception e) {
+            throw new ExtractionException(e);
+        }
+
+        return list;
+    }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java

@@ -30,12 +30,13 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
 
     @Override
     public String getUploaderName() {
-        return searchResult.getObject("user").getString("username");
+        //return searchResult.getObject("user").getString("username");
+        return searchResult.getObject("track").getObject("user").getString("username");
     }
 
     @Override
     public String getUploadDate() throws ParsingException {
-        return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
+        return SoundcloudParsingHelper.toDateString(searchResult.getObject("track").getString("created_at"));
     }
 
     @Override

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.SuggestionExtractor;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.kiosk.KioskList;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchEngine;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
@@ -78,4 +79,19 @@ public class YoutubeService extends StreamingService {
     public SuggestionExtractor getSuggestionExtractor() {
         return new YoutubeSuggestionExtractor(getServiceId());
     }
+
+    @Override
+    public KioskList getKioskList() throws ExtractionException {
+        KioskList list = new KioskList(getServiceId());
+
+        // add kiosks here e.g.:
+        YoutubeTrendingUrlIdHandler h = new YoutubeTrendingUrlIdHandler();
+        try {
+            list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), null), h);
+        } catch (Exception e) {
+            throw new ExtractionException(e);
+        }
+
+        return list;
+    }
 }

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java

@@ -0,0 +1,132 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.*;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
+
+import java.io.IOException;
+
+public class YoutubeTrendingExtractor extends KioskExtractor {
+
+    private Document doc;
+
+    public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl)
+        throws IOException, ExtractionException {
+        super(service, url, nextStreamsUrl);
+    }
+
+    @Override
+    public void fetchPage() throws IOException, ExtractionException {
+        Downloader downloader = NewPipe.getDownloader();
+
+        final String contentCountry = getContentCountry();
+        String url = getCleanUrl();
+        if(contentCountry != null && !contentCountry.isEmpty()) {
+            url += "?gl=" + contentCountry;
+        }
+
+        String pageContent = downloader.download(url);
+        doc = Jsoup.parse(pageContent, url);
+    }
+
+    @Override
+    public String getType() {
+        return "Trending";
+    }
+
+    @Override
+    public UrlIdHandler getUrlIdHandler() {
+        return new YoutubeTrendingUrlIdHandler();
+    }
+
+    @Override
+    public ListExtractor.NextItemsResult getNextStreams() {
+        return null;
+    }
+
+    @Override
+    public StreamInfoItemCollector getStreams() throws ParsingException {
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
+        Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first();
+        for(final Element li : ul.children()) {
+            final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
+            collector.commit(new YoutubeStreamInfoItemExtractor(li) {
+                @Override
+                public String getUrl() throws ParsingException {
+                    try {
+                        Element dl = el.select("h3").first().select("a").first();
+                        return dl.attr("abs:href");
+                    } catch (Exception e) {
+                        throw new ParsingException("Could not get web page url for the video", e);
+                    }
+                }
+
+                @Override
+                public String getName() throws ParsingException {
+                    try {
+                        Element dl = el.select("h3").first().select("a").first();
+                        return dl.text();
+                    } catch (Exception e) {
+                        throw new ParsingException("Could not get web page url for the video", e);
+                    }
+                }
+
+                @Override
+                public String getUploaderName() throws ParsingException {
+                    try {
+                        Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
+                        return uploaderEl.select("a").text();
+                    } catch (Exception e) {
+                        throw new ParsingException("Could not get Uploader name");
+                    }
+                }
+
+                @Override
+                public String getThumbnailUrl() throws ParsingException {
+                    try {
+                        String url;
+                        Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
+                                .select("img").first();
+                        url = te.attr("abs:src");
+                        // Sometimes youtube sends links to gif files which somehow seem to not exist
+                        // anymore. Items with such gif also offer a secondary image source. So we are going
+                        // to use that if we've caught such an item.
+                        if (url.contains(".gif")) {
+                            url = te.attr("abs:data-thumb");
+                        }
+                        return url;
+                    } catch (Exception e) {
+                        throw new ParsingException("Could not get thumbnail url", e);
+                    }
+                }
+            });
+        }
+
+        return collector;
+    }
+}

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java

@@ -0,0 +1,46 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingUrlIdHandler.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+public class YoutubeTrendingUrlIdHandler implements UrlIdHandler {
+
+    public String getUrl(String id) {
+        return "https://www.youtube.com/feed/trending";
+    }
+
+    @Override
+    public String getId(String url) {
+        return "Trending";
+    }
+
+    @Override
+    public String cleanUrl(String url) {
+        return getUrl("");
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url);
+    }
+}

src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java

@@ -1,9 +1,13 @@
 package org.schabi.newpipe.extractor.stream;
 
+import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.InfoItemCollector;
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
+import java.util.List;
+import java.util.Vector;
+
 /*
  * Created by Christian Schabesberger on 28.02.16.
  *
@@ -80,4 +84,14 @@ public class StreamInfoItemCollector extends InfoItemCollector {
             addError(e);
         }
     }
+
+    public List<StreamInfoItem> getStreamInfoItemList() {
+        List<StreamInfoItem> siiList = new Vector<>();
+        for(InfoItem ii : super.getItemList()) {
+            if(ii instanceof StreamInfoItem) {
+                siiList.add((StreamInfoItem) ii);
+            }
+        }
+        return siiList;
+    }
 }

src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java

@@ -0,0 +1,85 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.InfoItemCollector;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+
+/**
+ * Test for {@link SoundcloudChartsUrlIdHandler}
+ */
+public class SoundcloudChartsExtractorTest {
+
+    KioskExtractor extractor;
+
+    @Before
+    public void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        extractor = SoundCloud.getService()
+                .getKioskList()
+                .getExtractorByType("Top 50");
+        extractor.fetchPage();
+    }
+
+    @Test
+    public void testGetDownloader() throws Exception {
+        assertNotNull(NewPipe.getDownloader());
+    }
+
+    @Test
+    public void testGetName() throws Exception {
+        assertEquals(extractor.getName(), "Top 50");
+    }
+
+    @Test
+    public void testId() throws Exception {
+        assertEquals(extractor.getId(), "Top 50");
+    }
+
+    @Test
+    public void testGetStreams() throws Exception {
+        InfoItemCollector collector = extractor.getStreams();
+        if(!collector.getErrors().isEmpty()) {
+            System.err.println("----------");
+            for(Throwable e : collector.getErrors()) {
+                e.printStackTrace();
+                System.err.println("----------");
+            }
+        }
+        assertTrue("no streams are received",
+                !collector.getItemList().isEmpty()
+                        && collector.getErrors().isEmpty());
+    }
+
+    @Test
+    public void testGetStreamsErrors() throws Exception {
+        assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
+    }
+
+    @Test
+    public void testHasMoreStreams() throws Exception {
+        // Setup the streams
+        extractor.getStreams();
+        assertTrue("has more streams", extractor.hasMoreStreams());
+    }
+
+    @Test
+    public void testGetNextStreams() throws Exception {
+        extractor.getStreams();
+        assertFalse("extractor has next streams", extractor.getNextStreams() == null
+                || extractor.getNextStreams().nextItemsList.isEmpty());
+    }
+
+    @Test
+    public void testGetCleanUrl() throws Exception {
+        assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top");
+    }
+}

src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java

@@ -0,0 +1,49 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+
+/**
+ * Test for {@link SoundcloudChartsUrlIdHandler}
+ */
+public class SoundcloudChartsUrlIdHandlerTest {
+    private SoundcloudChartsUrlIdHandler urlIdHandler;
+
+    @Before
+    public void setUp() throws Exception {
+        urlIdHandler = new SoundcloudChartsUrlIdHandler();
+        NewPipe.init(Downloader.getInstance());
+    }
+
+    @Test
+    public void getUrl() {
+        assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top");
+        assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new");
+    }
+
+    @Test
+    public void getId() {
+        assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50");
+        assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot");
+    }
+
+    @Test
+    public void acceptUrl() {
+        assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts"));
+        assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/"));
+        assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new"));
+        assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/charts/top?genre=all-music"));
+        assertTrue(urlIdHandler.acceptUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"));
+
+        assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia"));
+        assertFalse(urlIdHandler.acceptUrl("soundcloud.com/charts askjkf"));
+        assertFalse(urlIdHandler.acceptUrl("    soundcloud.com/charts"));
+        assertFalse(urlIdHandler.acceptUrl(""));
+    }
+}

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -115,5 +115,4 @@ public class YoutubeChannelExtractorTest {
         assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
         assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
     }
-
 }

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java

@@ -1,17 +1,5 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-import org.junit.Before;
-import org.junit.Test;
-import org.schabi.newpipe.Downloader;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.SuggestionExtractor;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-
-import java.io.IOException;
-
-import static org.junit.Assert.assertFalse;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-
 /*
  * Created by Christian Schabesberger on 18.11.16.
  *
@@ -32,6 +20,18 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.SuggestionExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
 /**
  * Test for {@link SuggestionExtractor}
  */

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java

@@ -0,0 +1,108 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingExtractorTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.InfoItemCollector;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+
+import java.util.List;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+
+/**
+ * Test for {@link YoutubeTrendingUrlIdHandler}
+ */
+public class YoutubeTrendingExtractorTest {
+
+    KioskExtractor extractor;
+
+    @Before
+    public void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        extractor = YouTube.getService()
+                .getKioskList()
+                .getExtractorByType("Trending");
+        extractor.fetchPage();
+    }
+
+    @Test
+    public void testGetDownloader() throws Exception {
+        assertNotNull(NewPipe.getDownloader());
+    }
+
+    @Test
+    public void testGetName() throws Exception {
+        assertEquals(extractor.getName(), "Trending");
+    }
+
+    @Test
+    public void testId() throws Exception {
+        assertEquals(extractor.getId(), "Trending");
+    }
+
+    @Test
+    public void testGetStreams() throws Exception {
+        InfoItemCollector collector = extractor.getStreams();
+        if(!collector.getErrors().isEmpty()) {
+            System.err.println("----------");
+            for(Throwable e : collector.getErrors()) {
+                e.printStackTrace();
+                System.err.println("----------");
+            }
+        }
+        assertTrue("no streams are received",
+                !collector.getItemList().isEmpty()
+                        && collector.getErrors().isEmpty());
+    }
+
+    @Test
+    public void testGetStreamsErrors() throws Exception {
+        assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
+    }
+
+    @Test
+    public void testHasMoreStreams() throws Exception {
+        // Setup the streams
+        extractor.getStreams();
+        assertFalse("has more streams", extractor.hasMoreStreams());
+    }
+
+    @Test
+    public void testGetNextStreams() throws Exception {
+        assertTrue("extractor has next streams", extractor.getNextStreams() == null
+                || extractor.getNextStreams().nextItemsList.isEmpty());
+    }
+
+    @Test
+    public void testGetCleanUrl() throws Exception {
+        assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending");
+    }
+}

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java

@@ -0,0 +1,77 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingUrlIdHandlerTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for {@link YoutubeTrendingUrlIdHandler}
+ */
+public class YoutubeTrendingUrlIdHandlerTest {
+    private YoutubeTrendingUrlIdHandler urlIdHandler;
+
+    @Before
+    public void setUp() throws Exception {
+        urlIdHandler = new YoutubeTrendingUrlIdHandler();
+        NewPipe.init(Downloader.getInstance());
+    }
+
+    @Test
+    public void getUrl() {
+        assertEquals(urlIdHandler.getUrl(""), "https://www.youtube.com/feed/trending");
+    }
+
+    @Test
+    public void getId() {
+        assertEquals(urlIdHandler.getId(""), "Trending");
+    }
+
+    @Test
+    public void acceptUrl() {
+        assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending"));
+        assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending?adsf=fjaj#fhe"));
+        assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/feed/trending"));
+        assertTrue(urlIdHandler.acceptUrl("www.youtube.com/feed/trending"));
+        assertTrue(urlIdHandler.acceptUrl("youtube.com/feed/trending"));
+        assertTrue(urlIdHandler.acceptUrl("youtube.com/feed/trending?akdsakjf=dfije&kfj=dkjak"));
+        assertTrue(urlIdHandler.acceptUrl("https://youtube.com/feed/trending"));
+        assertTrue(urlIdHandler.acceptUrl("m.youtube.com/feed/trending"));
+
+        assertFalse(urlIdHandler.acceptUrl("https://youtu.be/feed/trending"));
+        assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia"));
+        assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/bullshit/feed/trending"));
+        assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending/bullshit"));
+        assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/bullshit/trending"));
+        assertFalse(urlIdHandler.acceptUrl("peter klaut aepferl youtube.com/feed/trending"));
+        assertFalse(urlIdHandler.acceptUrl("youtube.com/feed/trending askjkf"));
+        assertFalse(urlIdHandler.acceptUrl("askdjfi youtube.com/feed/trending askjkf"));
+        assertFalse(urlIdHandler.acceptUrl("    youtube.com/feed/trending"));
+        assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending.html"));
+        assertFalse(urlIdHandler.acceptUrl(""));
+    }
+}