diff --git a/src/main/java/org/schabi/newpipe/extractor/InfoItem.java b/src/main/java/org/schabi/newpipe/extractor/InfoItem.java index 44f97153f..ab6e37cb6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/InfoItem.java +++ b/src/main/java/org/schabi/newpipe/extractor/InfoItem.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor; /* - * Created by the-scrabi on 11.02.17. + * Created by Christian Schabesberger on 11.02.17. * * Copyright (C) Christian Schabesberger 2017 * InfoItem.java is part of NewPipe. diff --git a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index ae2241f30..53935ba98 100644 --- a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -48,6 +49,7 @@ public abstract class StreamingService { public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException; public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; + public abstract KioskList getKioskList() throws ExtractionException; public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException { return getChannelExtractor(url, null); diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java new file mode 100644 index 000000000..0081aa0b6 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -0,0 +1,71 @@ +package org.schabi.newpipe.extractor.kiosk; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * KioskExtractor.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.io.IOException; + +public abstract class KioskExtractor extends ListExtractor { + private String contentCountry = null; + + public KioskExtractor(StreamingService streamingService, + String url, + String nextStreamsUrl) + throws IOException, ExtractionException { + super(streamingService, url, nextStreamsUrl); + this.contentCountry = contentCountry; + } + + /** + * For certain Websites the content of a kiosk will be different depending + * on the country you want to poen the website in. Therefore you should + * set the contentCountry. + * @param contentCountry Set the country decoded as Country Code: http://www.1728.org/countries.htm + */ + public void setContentCountry(String contentCountry) { + this.contentCountry = contentCountry; + } + + /** + * Returns the type of the kiosk. + * eg. Trending, Top & Hot, Top last 24 hours + * @return type of kiosk + */ + public abstract String getType() throws ParsingException; + + @Override + public String getId() throws ParsingException { + return getType(); + } + + @Override + public String getName() throws ParsingException { + return getType(); + } + + public String getContentCountry() { + return contentCountry; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java new file mode 100644 index 000000000..c3fee602c --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -0,0 +1,74 @@ +package org.schabi.newpipe.extractor.kiosk; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * KioskInfo.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.schabi.newpipe.extractor.ListInfo; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + +import java.io.IOException; + +public class KioskInfo extends ListInfo { + public String type; + + public static KioskInfo getInfo(String url, + String contentCountry) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url, contentCountry); + } + + public static KioskInfo getInfo(ServiceList serviceItem, + String url, + String contentContry) throws IOException, ExtractionException { + return getInfo(serviceItem.getService(), url, contentContry); + } + + public static KioskInfo getInfo(StreamingService service, + String url, + String contentCountry) throws IOException, ExtractionException { + KioskList kl = service.getKioskList(); + KioskExtractor extractor = kl.getExtryctorByUrl(url); + return getInfo(extractor, contentCountry); + } + + public static KioskInfo getInfo(KioskExtractor extractor, + String contentCountry) throws IOException, ExtractionException { + KioskInfo info = new KioskInfo(); + extractor.setContentCountry(contentCountry); + extractor.fetchPage(); + info.type = extractor.getType(); + info.name = extractor.getName(); + info.id = extractor.getId(); + + try { + StreamInfoItemCollector c = extractor.getStreams(); + info.related_streams = c.getItemList(); + info.errors.addAll(c.getErrors()); + } catch (Exception e) { + info.errors.add(e); + } + + return info; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java new file mode 100644 index 000000000..c467a0b05 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -0,0 +1,57 @@ +package org.schabi.newpipe.extractor.kiosk; + +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class KioskList { + private int service_id; + private HashMap kioskList = new HashMap<>(); + + private class KioskEntry { + public KioskEntry(KioskExtractor e, UrlIdHandler h) { + extractor = e; + handler = h; + } + KioskExtractor extractor; + UrlIdHandler handler; + } + + public KioskList(int service_id) { + this.service_id = service_id; + } + + public void addKioskEntry(KioskExtractor extractor, UrlIdHandler handler) + throws Exception { + if(kioskList.get(extractor.getType()) != null) { + throw new Exception("Kiosk with type " + extractor.getType() + " already exists."); + } + kioskList.put(extractor.getType(), new KioskEntry(extractor, handler)); + } + + public KioskExtractor getExtractorByType(String kioskType) throws ExtractionException { + KioskEntry ke = kioskList.get(kioskType); + if(ke == null) { + throw new ExtractionException("No kiosk found with the type: " + kioskType); + } else { + return ke.extractor; + } + } + + public Set getAvailableKisokTypes() { + return kioskList.keySet(); + } + + public KioskExtractor getExtryctorByUrl(String url) throws ExtractionException { + for(Map.Entry e : kioskList.entrySet()) { + KioskEntry ke = e.getValue(); + if(ke.handler.acceptUrl(url)) { + return getExtractorByType(e.getKey()); + } + } + throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java new file mode 100644 index 000000000..44f1df552 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -0,0 +1,71 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + +public class SoundcloudChartsExtractor extends KioskExtractor { + private String url; + + public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + this.url = url; + } + + @Override + public void fetchPage() { + } + + @Override + public String getType() throws ParsingException { + return getUrlIdHandler().getId(url); + } + + @Override + public UrlIdHandler getUrlIdHandler() { + return new SoundcloudChartsUrlIdHandler(); + } + + @Override + public NextItemsResult getNextStreams() throws IOException, ExtractionException { + if (!hasMoreStreams()) { + throw new ExtractionException("Chart doesn't have more streams"); + } + + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true); + + return new NextItemsResult(collector, nextStreamsUrl); + } + + @Override + public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + + String apiUrl = "https://api-v2.soundcloud.com/charts" + + "?genre=soundcloud:genres:all-music" + + "&client_id=" + SoundcloudParsingHelper.clientId(); + + if (getType().equals("Top 50")) { + apiUrl += "&kind=top"; + } else { + apiUrl += "&kind=new"; + } + + List supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US"); + String contentCountry = getContentCountry(); + if (supportedCountries.contains(contentCountry)) { + apiUrl += "®ion=soundcloud:regions:" + contentCountry; + } + + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); + return collector; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java new file mode 100644 index 000000000..642e89c84 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.utils.Parser; + +public class SoundcloudChartsUrlIdHandler implements UrlIdHandler { + public String getUrl(String id) { + if (id.equals("Top 50")) { + return "https://soundcloud.com/charts/top"; + } else { + return "https://soundcloud.com/charts/new"; + } + } + + @Override + public String getId(String url) { + if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) { + return "Top 50"; + } else { + return "New & hot"; + } + } + + @Override + public String cleanUrl(String url) { + if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) { + return "https://soundcloud.com/charts/top"; + } else { + return "https://soundcloud.com/charts/new"; + } + } + + @Override + public boolean acceptUrl(String url) { + return Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$", url.toLowerCase()); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index b5cd3a6a6..29ffb7ec6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -127,7 +127,7 @@ public class SoundcloudParsingHelper { * * @return the next streams url, empty if don't have */ - public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException { String response = NewPipe.getDownloader().download(apiUrl); JsonObject responseObject; try { @@ -151,4 +151,8 @@ public class SoundcloudParsingHelper { return nextStreamsUrl; } + + public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException { + return getStreamsFromApi(collector, apiUrl, false); + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 80c565ac8..c2bb4f9c1 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,16 +1,17 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import java.io.IOException; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import java.io.IOException; - public class SoundcloudService extends StreamingService { public SoundcloudService(int id, String name) { @@ -57,4 +58,20 @@ public class SoundcloudService extends StreamingService { public SuggestionExtractor getSuggestionExtractor() { return new SoundcloudSuggestionExtractor(getServiceId()); } + + @Override + public KioskList getKioskList() throws ExtractionException { + KioskList list = new KioskList(getServiceId()); + + // add kiosks here e.g.: + SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler(); + try { + list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("Top 50"), null), h); + list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("New & hot"), null), h); + } catch (Exception e) { + throw new ExtractionException(e); + } + + return list; + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java index 8f273239a..ca405db6e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java @@ -30,12 +30,13 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto @Override public String getUploaderName() { - return searchResult.getObject("user").getString("username"); + //return searchResult.getObject("user").getString("username"); + return searchResult.getObject("track").getObject("user").getString("username"); } @Override public String getUploadDate() throws ParsingException { - return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at")); + return SoundcloudParsingHelper.toDateString(searchResult.getObject("track").getString("created_at")); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index eb8926024..c448d98ca 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -78,4 +79,19 @@ public class YoutubeService extends StreamingService { public SuggestionExtractor getSuggestionExtractor() { return new YoutubeSuggestionExtractor(getServiceId()); } + + @Override + public KioskList getKioskList() throws ExtractionException { + KioskList list = new KioskList(getServiceId()); + + // add kiosks here e.g.: + YoutubeTrendingUrlIdHandler h = new YoutubeTrendingUrlIdHandler(); + try { + list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), null), h); + } catch (Exception e) { + throw new ExtractionException(e); + } + + return list; + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java new file mode 100644 index 000000000..8c71f967a --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -0,0 +1,132 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingExtractor.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + +import java.io.IOException; + +public class YoutubeTrendingExtractor extends KioskExtractor { + + private Document doc; + + public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl) + throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + } + + @Override + public void fetchPage() throws IOException, ExtractionException { + Downloader downloader = NewPipe.getDownloader(); + + final String contentCountry = getContentCountry(); + String url = getCleanUrl(); + if(contentCountry != null && !contentCountry.isEmpty()) { + url += "?gl=" + contentCountry; + } + + String pageContent = downloader.download(url); + doc = Jsoup.parse(pageContent, url); + } + + @Override + public String getType() { + return "Trending"; + } + + @Override + public UrlIdHandler getUrlIdHandler() { + return new YoutubeTrendingUrlIdHandler(); + } + + @Override + public ListExtractor.NextItemsResult getNextStreams() { + return null; + } + + @Override + public StreamInfoItemCollector getStreams() throws ParsingException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first(); + for(final Element li : ul.children()) { + final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first(); + collector.commit(new YoutubeStreamInfoItemExtractor(li) { + @Override + public String getUrl() throws ParsingException { + try { + Element dl = el.select("h3").first().select("a").first(); + return dl.attr("abs:href"); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getName() throws ParsingException { + try { + Element dl = el.select("h3").first().select("a").first(); + return dl.text(); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getUploaderName() throws ParsingException { + try { + Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first(); + return uploaderEl.select("a").text(); + } catch (Exception e) { + throw new ParsingException("Could not get Uploader name"); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + String url; + Element te = li.select("span[class=\"yt-thumb-simple\"]").first() + .select("img").first(); + url = te.attr("abs:src"); + // Sometimes youtube sends links to gif files which somehow seem to not exist + // anymore. Items with such gif also offer a secondary image source. So we are going + // to use that if we've caught such an item. + if (url.contains(".gif")) { + url = te.attr("abs:data-thumb"); + } + return url; + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } + + return collector; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java new file mode 100644 index 000000000..63a35181e --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingUrlIdHandler.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.utils.Parser; + +public class YoutubeTrendingUrlIdHandler implements UrlIdHandler { + + public String getUrl(String id) { + return "https://www.youtube.com/feed/trending"; + } + + @Override + public String getId(String url) { + return "Trending"; + } + + @Override + public String cleanUrl(String url) { + return getUrl(""); + } + + @Override + public boolean acceptUrl(String url) { + return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java index 8b1863ba2..b0757b886 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java @@ -1,9 +1,13 @@ package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import java.util.List; +import java.util.Vector; + /* * Created by Christian Schabesberger on 28.02.16. * @@ -80,4 +84,14 @@ public class StreamInfoItemCollector extends InfoItemCollector { addError(e); } } + + public List getStreamInfoItemList() { + List siiList = new Vector<>(); + for(InfoItem ii : super.getItemList()) { + if(ii instanceof StreamInfoItem) { + siiList.add((StreamInfoItem) ii); + } + } + return siiList; + } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java new file mode 100644 index 000000000..061c37ff3 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -0,0 +1,85 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItemCollector; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; + +/** + * Test for {@link SoundcloudChartsUrlIdHandler} + */ +public class SoundcloudChartsExtractorTest { + + KioskExtractor extractor; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = SoundCloud.getService() + .getKioskList() + .getExtractorByType("Top 50"); + extractor.fetchPage(); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getName(), "Top 50"); + } + + @Test + public void testId() throws Exception { + assertEquals(extractor.getId(), "Top 50"); + } + + @Test + public void testGetStreams() throws Exception { + InfoItemCollector collector = extractor.getStreams(); + if(!collector.getErrors().isEmpty()) { + System.err.println("----------"); + for(Throwable e : collector.getErrors()) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !collector.getItemList().isEmpty() + && collector.getErrors().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertTrue("has more streams", extractor.hasMoreStreams()); + } + + @Test + public void testGetNextStreams() throws Exception { + extractor.getStreams(); + assertFalse("extractor has next streams", extractor.getNextStreams() == null + || extractor.getNextStreams().nextItemsList.isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top"); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java new file mode 100644 index 000000000..6429f6c3f --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java @@ -0,0 +1,49 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; + +/** + * Test for {@link SoundcloudChartsUrlIdHandler} + */ +public class SoundcloudChartsUrlIdHandlerTest { + private SoundcloudChartsUrlIdHandler urlIdHandler; + + @Before + public void setUp() throws Exception { + urlIdHandler = new SoundcloudChartsUrlIdHandler(); + NewPipe.init(Downloader.getInstance()); + } + + @Test + public void getUrl() { + assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top"); + assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new"); + } + + @Test + public void getId() { + assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50"); + assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot"); + } + + @Test + public void acceptUrl() { + assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts")); + assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/")); + assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new")); + assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/charts/top?genre=all-music")); + assertTrue(urlIdHandler.acceptUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries")); + + assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia")); + assertFalse(urlIdHandler.acceptUrl("soundcloud.com/charts askjkf")); + assertFalse(urlIdHandler.acceptUrl(" soundcloud.com/charts")); + assertFalse(urlIdHandler.acceptUrl("")); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 3df3add37..b57f61bad 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -115,5 +115,4 @@ public class YoutubeChannelExtractorTest { assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty()); assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); } - } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java index 58089081c..39caa42c7 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java @@ -1,17 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import org.junit.Before; -import org.junit.Test; -import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.SuggestionExtractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; - -import java.io.IOException; - -import static org.junit.Assert.assertFalse; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - /* * Created by Christian Schabesberger on 18.11.16. * @@ -32,6 +20,18 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; * along with NewPipe. If not, see . */ +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + /** * Test for {@link SuggestionExtractor} */ diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java new file mode 100644 index 000000000..4dd9879f7 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -0,0 +1,108 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingExtractorTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemCollector; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; + +import java.util.List; + +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + + +/** + * Test for {@link YoutubeTrendingUrlIdHandler} + */ +public class YoutubeTrendingExtractorTest { + + KioskExtractor extractor; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = YouTube.getService() + .getKioskList() + .getExtractorByType("Trending"); + extractor.fetchPage(); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getName(), "Trending"); + } + + @Test + public void testId() throws Exception { + assertEquals(extractor.getId(), "Trending"); + } + + @Test + public void testGetStreams() throws Exception { + InfoItemCollector collector = extractor.getStreams(); + if(!collector.getErrors().isEmpty()) { + System.err.println("----------"); + for(Throwable e : collector.getErrors()) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !collector.getItemList().isEmpty() + && collector.getErrors().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertFalse("has more streams", extractor.hasMoreStreams()); + } + + @Test + public void testGetNextStreams() throws Exception { + assertTrue("extractor has next streams", extractor.getNextStreams() == null + || extractor.getNextStreams().nextItemsList.isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending"); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java new file mode 100644 index 000000000..303d409d6 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java @@ -0,0 +1,77 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingUrlIdHandlerTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; + +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test for {@link YoutubeTrendingUrlIdHandler} + */ +public class YoutubeTrendingUrlIdHandlerTest { + private YoutubeTrendingUrlIdHandler urlIdHandler; + + @Before + public void setUp() throws Exception { + urlIdHandler = new YoutubeTrendingUrlIdHandler(); + NewPipe.init(Downloader.getInstance()); + } + + @Test + public void getUrl() { + assertEquals(urlIdHandler.getUrl(""), "https://www.youtube.com/feed/trending"); + } + + @Test + public void getId() { + assertEquals(urlIdHandler.getId(""), "Trending"); + } + + @Test + public void acceptUrl() { + assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending?adsf=fjaj#fhe")); + assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("www.youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("youtube.com/feed/trending?akdsakjf=dfije&kfj=dkjak")); + assertTrue(urlIdHandler.acceptUrl("https://youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("m.youtube.com/feed/trending")); + + assertFalse(urlIdHandler.acceptUrl("https://youtu.be/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/bullshit/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending/bullshit")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/bullshit/trending")); + assertFalse(urlIdHandler.acceptUrl("peter klaut aepferl youtube.com/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("youtube.com/feed/trending askjkf")); + assertFalse(urlIdHandler.acceptUrl("askdjfi youtube.com/feed/trending askjkf")); + assertFalse(urlIdHandler.acceptUrl(" youtube.com/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending.html")); + assertFalse(urlIdHandler.acceptUrl("")); + } +}