From 316b4791998bb4db0d16697ab2c4c7a268c2fb46 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 12 Aug 2017 17:29:28 +0200 Subject: [PATCH 01/10] add basic setup for kiosk --- .../schabi/newpipe/extractor/InfoItem.java | 2 +- .../newpipe/extractor/StreamingService.java | 2 + .../extractor/kiosk/KioskExtractor.java | 52 +++++++++++++++ .../newpipe/extractor/kiosk/KioskInfo.java | 64 +++++++++++++++++++ .../newpipe/extractor/kiosk/KioskList.java | 57 +++++++++++++++++ .../soundcloud/SoundcloudService.java | 13 ++++ .../services/youtube/YoutubeService.java | 11 ++++ 7 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java diff --git a/src/main/java/org/schabi/newpipe/extractor/InfoItem.java b/src/main/java/org/schabi/newpipe/extractor/InfoItem.java index 44f97153f..ab6e37cb6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/InfoItem.java +++ b/src/main/java/org/schabi/newpipe/extractor/InfoItem.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor; /* - * Created by the-scrabi on 11.02.17. + * Created by Christian Schabesberger on 11.02.17. * * Copyright (C) Christian Schabesberger 2017 * InfoItem.java is part of NewPipe. diff --git a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index ae2241f30..d4e3468bb 100644 --- a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -48,6 +49,7 @@ public abstract class StreamingService { public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException; public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; + public abstract KioskList getKioskList(); public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException { return getChannelExtractor(url, null); diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java new file mode 100644 index 000000000..6dad3acc7 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -0,0 +1,52 @@ +package org.schabi.newpipe.extractor.kiosk; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * KioskExtractor.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.io.IOException; + +public abstract class KioskExtractor extends ListExtractor { + public KioskExtractor(StreamingService streamingService, String url, String nextStreamsUrl) + throws IOException, ExtractionException { + super(streamingService, url, nextStreamsUrl); + } + + /** + * Returns the type of the kiosk. + * eg. Trending, Top & Hot, Top last 24 hours + * @return type of kiosk + */ + public abstract String getType() throws ParsingException; + + @Override + public String getId() throws ParsingException { + return getType(); + } + + @Override + public String getName() throws ParsingException { + return getType(); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java new file mode 100644 index 000000000..b196fecde --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -0,0 +1,64 @@ +package org.schabi.newpipe.extractor.kiosk; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * KioskInfo.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.schabi.newpipe.extractor.ListInfo; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + +import java.io.IOException; + +public class KioskInfo extends ListInfo { + public String type; + + public static KioskInfo getInfo(String url) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url); + } + + public static KioskInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException { + return getInfo(serviceItem.getService(), url); + } + + public static KioskInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException { + KioskList kl = service.getKioskList(); + KioskExtractor extractor = kl.getExtryctorByUrl(url); + return getInfo(extractor); + } + + public static KioskInfo getInfo(KioskExtractor extractor) throws ParsingException { + KioskInfo info = new KioskInfo(); + info.type = extractor.getType(); + + try { + StreamInfoItemCollector c = extractor.getStreams(); + info.related_streams = c.getItemList(); + info.errors.addAll(c.getErrors()); + } catch (Exception e) { + info.errors.add(e); + } + + return info; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java new file mode 100644 index 000000000..26cb4a93d --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -0,0 +1,57 @@ +package org.schabi.newpipe.extractor.kiosk; + +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class KioskList { + private int service_id; + private HashMap kioskList = new HashMap<>(); + + private class KioskEntry { + public KioskEntry(KioskExtractor e, UrlIdHandler h) { + extractor = e; + handler = h; + } + KioskExtractor extractor; + UrlIdHandler handler; + } + + public KioskList(int service_id) { + this.service_id = service_id; + } + + public void addKioskEntry(String kioskType, KioskExtractor extractor, UrlIdHandler handler) + throws Exception { + if(kioskList.get(kioskType) != null) { + throw new Exception("Kiosk with type " + kioskType + " already exists."); + } + kioskList.put(kioskType, new KioskEntry(extractor, handler)); + } + + public KioskExtractor getExtractorByType(String kioskType) throws ExtractionException { + KioskEntry ke = kioskList.get(kioskType); + if(ke == null) { + throw new ExtractionException("No kiosk found with the type: " + kioskType); + } else { + return ke.extractor; + } + } + + public Set getAvailableKisokTypes() { + return kioskList.keySet(); + } + + public KioskExtractor getExtryctorByUrl(String url) throws ExtractionException { + for(Map.Entry e : kioskList.entrySet()) { + KioskEntry ke = e.getValue(); + if(ke.handler.acceptUrl(url)) { + return getExtractorByType(e.getKey()); + } + } + throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 80c565ac8..90e794f19 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -57,4 +59,15 @@ public class SoundcloudService extends StreamingService { public SuggestionExtractor getSuggestionExtractor() { return new SoundcloudSuggestionExtractor(getServiceId()); } + + @Override + public KioskList getKioskList() { + KioskList list = new KioskList(getServiceId()); + + // add kiosks here e.g.: + //list.addKioskEntry("trinding", new TrendingKiosk(), new TrendingUrlIdHandler()); + + + return list; + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index eb8926024..b071ba5eb 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -78,4 +79,14 @@ public class YoutubeService extends StreamingService { public SuggestionExtractor getSuggestionExtractor() { return new YoutubeSuggestionExtractor(getServiceId()); } + + @Override + public KioskList getKioskList() { + KioskList list = new KioskList(getServiceId()); + + // add kiosks here e.g.: + //list.addKioskEntry("trinding", new TrendingKiosk(), new TrendingUrlIdHandler()); + + return list; + } } From a6c1728dac28102dc6d979b7289c172b055ab71c Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 12 Aug 2017 17:35:43 +0200 Subject: [PATCH 02/10] add copyright header to kiosklist --- .../newpipe/extractor/kiosk/KioskList.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 26cb4a93d..0cecbdcde 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -1,5 +1,25 @@ package org.schabi.newpipe.extractor.kiosk; +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * KioskList.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; From b89f5a9b42d31c22809396dc460fcc9aaf4003b7 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 12 Aug 2017 21:10:21 +0200 Subject: [PATCH 03/10] add youtube trending extractor --- .../newpipe/extractor/StreamingService.java | 2 +- .../newpipe/extractor/kiosk/KioskList.java | 30 ++------- .../services/youtube/YoutubeService.java | 9 ++- .../youtube/YoutubeTrendingExtractor.java | 63 +++++++++++++++++++ .../youtube/YoutubeTrendingUrlIdHandler.java | 45 +++++++++++++ 5 files changed, 121 insertions(+), 28 deletions(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java diff --git a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index d4e3468bb..53935ba98 100644 --- a/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -49,7 +49,7 @@ public abstract class StreamingService { public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException; public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; - public abstract KioskList getKioskList(); + public abstract KioskList getKioskList() throws ExtractionException; public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException { return getChannelExtractor(url, null); diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 0cecbdcde..c467a0b05 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -1,25 +1,5 @@ package org.schabi.newpipe.extractor.kiosk; -/* - * Created by Christian Schabesberger on 12.08.17. - * - * Copyright (C) Christian Schabesberger 2017 - * KioskList.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -27,7 +7,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -public class KioskList { +public class KioskList { private int service_id; private HashMap kioskList = new HashMap<>(); @@ -44,12 +24,12 @@ public class KioskList { this.service_id = service_id; } - public void addKioskEntry(String kioskType, KioskExtractor extractor, UrlIdHandler handler) + public void addKioskEntry(KioskExtractor extractor, UrlIdHandler handler) throws Exception { - if(kioskList.get(kioskType) != null) { - throw new Exception("Kiosk with type " + kioskType + " already exists."); + if(kioskList.get(extractor.getType()) != null) { + throw new Exception("Kiosk with type " + extractor.getType() + " already exists."); } - kioskList.put(kioskType, new KioskEntry(extractor, handler)); + kioskList.put(extractor.getType(), new KioskEntry(extractor, handler)); } public KioskExtractor getExtractorByType(String kioskType) throws ExtractionException { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index b071ba5eb..18aedbe90 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -81,11 +81,16 @@ public class YoutubeService extends StreamingService { } @Override - public KioskList getKioskList() { + public KioskList getKioskList() throws ExtractionException { KioskList list = new KioskList(getServiceId()); // add kiosks here e.g.: - //list.addKioskEntry("trinding", new TrendingKiosk(), new TrendingUrlIdHandler()); + YoutubeTrendingUrlIdHandler h = new YoutubeTrendingUrlIdHandler(); + try { + list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), h.getUrl("")), h); + } catch (Exception e) { + throw new ExtractionException(e); + } return list; } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java new file mode 100644 index 000000000..4e38e85a2 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -0,0 +1,63 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingExtractor.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; +import java.io.IOException; + +public class YoutubeTrendingExtractor extends KioskExtractor { + + public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl) + throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + } + + @Override + public void fetchPage() + throws IOException, ExtractionException { + + } + + @Override + public String getType() { + return "Treinding"; + } + + @Override + public UrlIdHandler getUrlIdHandler() { + return new YoutubeTrendingUrlIdHandler(); + } + + @Override + public ListExtractor.NextItemsResult getNextStreams() { + return null; + } + + @Override + public StreamInfoItemCollector getStreams() { + return null; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java new file mode 100644 index 000000000..d6083459f --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java @@ -0,0 +1,45 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingUrlIdHandler.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.schabi.newpipe.extractor.UrlIdHandler; + +public class YoutubeTrendingUrlIdHandler implements UrlIdHandler { + + public String getUrl(String id) { + return "https://www.youtube.com/feed/trending"; + } + + @Override + public String getId(String url) { + return "Trending"; + } + + @Override + public String cleanUrl(String url) { + return getUrl(""); + } + + @Override + public boolean acceptUrl(String url) { + return url.contains("feed/treinding"); + } +} From 8dabda293b7776521e902d6db755dbded81b9bed Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 12 Aug 2017 23:03:34 +0200 Subject: [PATCH 04/10] add test for YoutubeTrendingUrlIdHandler --- .../youtube/YoutubeTrendingUrlIdHandler.java | 3 +- .../YoutubeSuggestionExtractorTest.java | 24 +++--- .../youtube/YoutubeTrendingExtractorTest.java | 4 + .../YoutubeTrendingUrlIdHandlerTest.java | 76 +++++++++++++++++++ 4 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java index d6083459f..63a35181e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java @@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.services.youtube; */ import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.utils.Parser; public class YoutubeTrendingUrlIdHandler implements UrlIdHandler { @@ -40,6 +41,6 @@ public class YoutubeTrendingUrlIdHandler implements UrlIdHandler { @Override public boolean acceptUrl(String url) { - return url.contains("feed/treinding"); + return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java index 58089081c..39caa42c7 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java @@ -1,17 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import org.junit.Before; -import org.junit.Test; -import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.SuggestionExtractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; - -import java.io.IOException; - -import static org.junit.Assert.assertFalse; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - /* * Created by Christian Schabesberger on 18.11.16. * @@ -32,6 +20,18 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; * along with NewPipe. If not, see . */ +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + /** * Test for {@link SuggestionExtractor} */ diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java new file mode 100644 index 000000000..70e536cf2 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -0,0 +1,4 @@ +package org.schabi.newpipe.extractor.services.youtube; + +public class YoutubeTrendingExtractorTest { +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java new file mode 100644 index 000000000..feca28c4b --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java @@ -0,0 +1,76 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingUrlIdHandlerTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; + +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test for {@link YoutubeTrendingUrlIdHandler} + */ +public class YoutubeTrendingUrlIdHandlerTest { + private YoutubeTrendingUrlIdHandler urlIdHandler; + + @Before + public void setUp() throws Exception { + urlIdHandler = new YoutubeTrendingUrlIdHandler(); + NewPipe.init(Downloader.getInstance()); + } + + @Test + public void getUrl() { + assertEquals(urlIdHandler.getUrl(""), "https://www.youtube.com/feed/trending"); + } + + @Test + public void getId() { + assertEquals(urlIdHandler.getId(""), "Trending"); + } + + @Test + public void acceptUrl() { + assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending?adsf=fjaj#fhe")); + assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("www.youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("youtube.com/feed/trending?akdsakjf=dfije&kfj=dkjak")); + assertTrue(urlIdHandler.acceptUrl("https://youtube.com/feed/trending")); + assertTrue(urlIdHandler.acceptUrl("m.youtube.com/feed/trending")); + + assertFalse(urlIdHandler.acceptUrl("https://youtu.be/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/bullshit/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending/bullshit")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/bullshit/trending")); + assertFalse(urlIdHandler.acceptUrl("peter klaut aepferl youtube.com/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("youtube.com/feed/trending askjkf")); + assertFalse(urlIdHandler.acceptUrl("askdjfi youtube.com/feed/trending askjkf")); + assertFalse(urlIdHandler.acceptUrl(" youtube.com/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("")); + } +} From 88d2fff0916462986222027dd9f4d589d9c57fa7 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 12 Aug 2017 23:19:35 +0200 Subject: [PATCH 05/10] add test for YoutubeTrendingExtractor --- .../youtube/YoutubeChannelExtractorTest.java | 1 - .../youtube/YoutubeTrendingExtractorTest.java | 84 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 646c6be97..0ce256910 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -102,5 +102,4 @@ public class YoutubeChannelExtractorTest { assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); } - } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index 70e536cf2..74ea20435 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -1,4 +1,88 @@ package org.schabi.newpipe.extractor.services.youtube; +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingExtractorTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; + +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + + +/** + * Test for {@link YoutubeTrendingUrlIdHandler} + */ public class YoutubeTrendingExtractorTest { + + KioskExtractor extractor; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = YouTube.getService() + .getKioskList() + .getExtractorByType("Trending"); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getName(), "Trending"); + } + + @Test + public void testId() throws Exception { + assertEquals(extractor.getId(), "Trending"); + } + + @Test + public void testGetStreams() throws Exception { + assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertTrue("don't have more streams", extractor.hasMoreStreams()); + } + + @Test + public void testGetNextStreams() throws Exception { + assertFalse("extractor has next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); + assertFalse("extractor has more streams after getNextStreams", extractor.hasMoreStreams()); + } } From 7b7f6d2cbfbd884db30dc4b02005c0f07da23a50 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 13 Aug 2017 00:58:29 +0200 Subject: [PATCH 06/10] made YoutubeTrendingExtractor work --- .../services/youtube/YoutubeService.java | 2 +- .../youtube/YoutubeTrendingExtractor.java | 80 +++++++++++++++++-- .../stream/StreamInfoItemCollector.java | 14 ++++ .../youtube/YoutubeTrendingExtractorTest.java | 28 ++++++- .../YoutubeTrendingUrlIdHandlerTest.java | 1 + 5 files changed, 112 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 18aedbe90..c448d98ca 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -87,7 +87,7 @@ public class YoutubeService extends StreamingService { // add kiosks here e.g.: YoutubeTrendingUrlIdHandler h = new YoutubeTrendingUrlIdHandler(); try { - list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), h.getUrl("")), h); + list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), null), h); } catch (Exception e) { throw new ExtractionException(e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index 4e38e85a2..77cc2a135 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -20,30 +20,38 @@ package org.schabi.newpipe.extractor.services.youtube; * along with NewPipe. If not, see . */ -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + import java.io.IOException; public class YoutubeTrendingExtractor extends KioskExtractor { + private Document doc; + public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { super(service, url, nextStreamsUrl); } @Override - public void fetchPage() - throws IOException, ExtractionException { + public void fetchPage() throws IOException, ExtractionException { + Downloader downloader = NewPipe.getDownloader(); + String channelUrl = getCleanUrl(); + String pageContent = downloader.download(channelUrl); + doc = Jsoup.parse(pageContent, channelUrl); } @Override public String getType() { - return "Treinding"; + return "Trending"; } @Override @@ -57,7 +65,63 @@ public class YoutubeTrendingExtractor extends KioskExtractor { } @Override - public StreamInfoItemCollector getStreams() { - return null; + public StreamInfoItemCollector getStreams() throws ParsingException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first(); + for(final Element li : ul.children()) { + final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first(); + collector.commit(new YoutubeStreamInfoItemExtractor(li) { + @Override + public String getUrl() throws ParsingException { + try { + Element dl = el.select("h3").first().select("a").first(); + return dl.attr("abs:href"); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getName() throws ParsingException { + try { + Element dl = el.select("h3").first().select("a").first(); + return dl.text(); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getUploaderName() throws ParsingException { + try { + Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first(); + return uploaderEl.select("a").text(); + } catch (Exception e) { + throw new ParsingException("Could not get Uploader name"); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + String url; + Element te = li.select("span[class=\"yt-thumb-simple\"]").first() + .select("img").first(); + url = te.attr("abs:src"); + // Sometimes youtube sends links to gif files which somehow seem to not exist + // anymore. Items with such gif also offer a secondary image source. So we are going + // to use that if we've caught such an item. + if (url.contains(".gif")) { + url = te.attr("abs:data-thumb"); + } + return url; + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } + + return collector; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java index 8b1863ba2..b0757b886 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java @@ -1,9 +1,13 @@ package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import java.util.List; +import java.util.Vector; + /* * Created by Christian Schabesberger on 28.02.16. * @@ -80,4 +84,14 @@ public class StreamInfoItemCollector extends InfoItemCollector { addError(e); } } + + public List getStreamInfoItemList() { + List siiList = new Vector<>(); + for(InfoItem ii : super.getItemList()) { + if(ii instanceof StreamInfoItem) { + siiList.add((StreamInfoItem) ii); + } + } + return siiList; + } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index 74ea20435..4dd9879f7 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -23,9 +23,13 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import java.util.List; + import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -46,6 +50,7 @@ public class YoutubeTrendingExtractorTest { extractor = YouTube.getService() .getKioskList() .getExtractorByType("Trending"); + extractor.fetchPage(); } @Test @@ -65,7 +70,17 @@ public class YoutubeTrendingExtractorTest { @Test public void testGetStreams() throws Exception { - assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty()); + InfoItemCollector collector = extractor.getStreams(); + if(!collector.getErrors().isEmpty()) { + System.err.println("----------"); + for(Throwable e : collector.getErrors()) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !collector.getItemList().isEmpty() + && collector.getErrors().isEmpty()); } @Test @@ -77,12 +92,17 @@ public class YoutubeTrendingExtractorTest { public void testHasMoreStreams() throws Exception { // Setup the streams extractor.getStreams(); - assertTrue("don't have more streams", extractor.hasMoreStreams()); + assertFalse("has more streams", extractor.hasMoreStreams()); } @Test public void testGetNextStreams() throws Exception { - assertFalse("extractor has next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); - assertFalse("extractor has more streams after getNextStreams", extractor.hasMoreStreams()); + assertTrue("extractor has next streams", extractor.getNextStreams() == null + || extractor.getNextStreams().nextItemsList.isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending"); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java index feca28c4b..303d409d6 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java @@ -71,6 +71,7 @@ public class YoutubeTrendingUrlIdHandlerTest { assertFalse(urlIdHandler.acceptUrl("youtube.com/feed/trending askjkf")); assertFalse(urlIdHandler.acceptUrl("askdjfi youtube.com/feed/trending askjkf")); assertFalse(urlIdHandler.acceptUrl(" youtube.com/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending.html")); assertFalse(urlIdHandler.acceptUrl("")); } } From c76f39c81b7f59ca13e0537e42693cfecabb82d3 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 13 Aug 2017 01:10:22 +0200 Subject: [PATCH 07/10] fix minor details --- src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index b196fecde..68c04553e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -50,6 +50,8 @@ public class KioskInfo extends ListInfo { public static KioskInfo getInfo(KioskExtractor extractor) throws ParsingException { KioskInfo info = new KioskInfo(); info.type = extractor.getType(); + info.name = extractor.getName(); + info.id = extractor.getId(); try { StreamInfoItemCollector c = extractor.getStreams(); From 5119dabb63318b1307b867babd272b836f410082 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 14 Aug 2017 12:48:51 +0200 Subject: [PATCH 08/10] add contentCountry to Kiosk --- .../extractor/kiosk/KioskExtractor.java | 21 +++++++++++++++++- .../newpipe/extractor/kiosk/KioskInfo.java | 22 +++++++++++++------ .../youtube/YoutubeTrendingExtractor.java | 11 +++++++--- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 6dad3acc7..0081aa0b6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -28,9 +28,24 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.IOException; public abstract class KioskExtractor extends ListExtractor { - public KioskExtractor(StreamingService streamingService, String url, String nextStreamsUrl) + private String contentCountry = null; + + public KioskExtractor(StreamingService streamingService, + String url, + String nextStreamsUrl) throws IOException, ExtractionException { super(streamingService, url, nextStreamsUrl); + this.contentCountry = contentCountry; + } + + /** + * For certain Websites the content of a kiosk will be different depending + * on the country you want to poen the website in. Therefore you should + * set the contentCountry. + * @param contentCountry Set the country decoded as Country Code: http://www.1728.org/countries.htm + */ + public void setContentCountry(String contentCountry) { + this.contentCountry = contentCountry; } /** @@ -49,4 +64,8 @@ public abstract class KioskExtractor extends ListExtractor { public String getName() throws ParsingException { return getType(); } + + public String getContentCountry() { + return contentCountry; + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index 68c04553e..c3fee602c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -33,22 +33,30 @@ import java.io.IOException; public class KioskInfo extends ListInfo { public String type; - public static KioskInfo getInfo(String url) throws IOException, ExtractionException { - return getInfo(NewPipe.getServiceByUrl(url), url); + public static KioskInfo getInfo(String url, + String contentCountry) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url, contentCountry); } - public static KioskInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException { - return getInfo(serviceItem.getService(), url); + public static KioskInfo getInfo(ServiceList serviceItem, + String url, + String contentContry) throws IOException, ExtractionException { + return getInfo(serviceItem.getService(), url, contentContry); } - public static KioskInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException { + public static KioskInfo getInfo(StreamingService service, + String url, + String contentCountry) throws IOException, ExtractionException { KioskList kl = service.getKioskList(); KioskExtractor extractor = kl.getExtryctorByUrl(url); - return getInfo(extractor); + return getInfo(extractor, contentCountry); } - public static KioskInfo getInfo(KioskExtractor extractor) throws ParsingException { + public static KioskInfo getInfo(KioskExtractor extractor, + String contentCountry) throws IOException, ExtractionException { KioskInfo info = new KioskInfo(); + extractor.setContentCountry(contentCountry); + extractor.fetchPage(); info.type = extractor.getType(); info.name = extractor.getName(); info.id = extractor.getId(); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index 77cc2a135..8c71f967a 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -44,9 +44,14 @@ public class YoutubeTrendingExtractor extends KioskExtractor { public void fetchPage() throws IOException, ExtractionException { Downloader downloader = NewPipe.getDownloader(); - String channelUrl = getCleanUrl(); - String pageContent = downloader.download(channelUrl); - doc = Jsoup.parse(pageContent, channelUrl); + final String contentCountry = getContentCountry(); + String url = getCleanUrl(); + if(contentCountry != null && !contentCountry.isEmpty()) { + url += "?gl=" + contentCountry; + } + + String pageContent = downloader.download(url); + doc = Jsoup.parse(pageContent, url); } @Override From e2b7cb9c69bf229cac09a88be0d408a15f4856b5 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 20 Aug 2017 10:03:41 +0200 Subject: [PATCH 09/10] Add SoundcloudChartsExtractor --- .../soundcloud/SoundcloudChartsExtractor.java | 63 ++++++++++++++ .../SoundcloudChartsUrlIdHandler.java | 37 ++++++++ .../soundcloud/SoundcloudParsingHelper.java | 12 ++- .../soundcloud/SoundcloudService.java | 16 ++-- .../SoundcloudChartsExtractorTest.java | 85 +++++++++++++++++++ .../SoundcloudChartsUrlIdHandlerTest.java | 49 +++++++++++ 6 files changed, 254 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java new file mode 100644 index 000000000..f4aca4e2d --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -0,0 +1,63 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import java.io.IOException; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + +public class SoundcloudChartsExtractor extends KioskExtractor { + private String url; + + public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { + super(service, url, nextStreamsUrl); + this.url = url; + } + + @Override + public void fetchPage() { + } + + @Override + public String getType() throws ParsingException { + return getUrlIdHandler().getId(url); + } + + @Override + public UrlIdHandler getUrlIdHandler() { + return new SoundcloudChartsUrlIdHandler(); + } + + @Override + public NextItemsResult getNextStreams() throws IOException, ExtractionException { + if (!hasMoreStreams()) { + throw new ExtractionException("Chart doesn't have more streams"); + } + + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true); + + return new NextItemsResult(collector.getItemList(), nextStreamsUrl); + } + + @Override + public StreamInfoItemCollector getStreams() throws IOException, ExtractionException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + + String apiUrl = "https://api-v2.soundcloud.com/charts" + + "?genre=soundcloud:genres:all-music" + + "&client_id=" + SoundcloudParsingHelper.clientId(); + + if (getType().equals("Top 50")) { + apiUrl += "&kind=top"; + } else { + apiUrl += "&kind=new"; + } + + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); + return collector; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java new file mode 100644 index 000000000..642e89c84 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.utils.Parser; + +public class SoundcloudChartsUrlIdHandler implements UrlIdHandler { + public String getUrl(String id) { + if (id.equals("Top 50")) { + return "https://soundcloud.com/charts/top"; + } else { + return "https://soundcloud.com/charts/new"; + } + } + + @Override + public String getId(String url) { + if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) { + return "Top 50"; + } else { + return "New & hot"; + } + } + + @Override + public String cleanUrl(String url) { + if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) { + return "https://soundcloud.com/charts/top"; + } else { + return "https://soundcloud.com/charts/new"; + } + } + + @Override + public boolean acceptUrl(String url) { + return Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$", url.toLowerCase()); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 21b1686d0..fd20ada45 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -121,13 +121,17 @@ public class SoundcloudParsingHelper { * * @return the next streams url, empty if don't have */ - public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { + public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException { String response = NewPipe.getDownloader().download(apiUrl); JSONObject responseObject = new JSONObject(response); JSONArray responseCollection = responseObject.getJSONArray("collection"); for (int i = 0; i < responseCollection.length(); i++) { - collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i))); + if (charts) { + collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i).getJSONObject("track"))); + } else { + collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i))); + } } String nextStreamsUrl; @@ -140,4 +144,8 @@ public class SoundcloudParsingHelper { return nextStreamsUrl; } + + public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException { + return getStreamsFromApi(collector, apiUrl, false); + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 90e794f19..c2bb4f9c1 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,18 +1,17 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import java.io.IOException; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import java.io.IOException; - public class SoundcloudService extends StreamingService { public SoundcloudService(int id, String name) { @@ -61,12 +60,17 @@ public class SoundcloudService extends StreamingService { } @Override - public KioskList getKioskList() { + public KioskList getKioskList() throws ExtractionException { KioskList list = new KioskList(getServiceId()); // add kiosks here e.g.: - //list.addKioskEntry("trinding", new TrendingKiosk(), new TrendingUrlIdHandler()); - + SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler(); + try { + list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("Top 50"), null), h); + list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("New & hot"), null), h); + } catch (Exception e) { + throw new ExtractionException(e); + } return list; } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java new file mode 100644 index 000000000..061c37ff3 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -0,0 +1,85 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItemCollector; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; + +/** + * Test for {@link SoundcloudChartsUrlIdHandler} + */ +public class SoundcloudChartsExtractorTest { + + KioskExtractor extractor; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = SoundCloud.getService() + .getKioskList() + .getExtractorByType("Top 50"); + extractor.fetchPage(); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getName(), "Top 50"); + } + + @Test + public void testId() throws Exception { + assertEquals(extractor.getId(), "Top 50"); + } + + @Test + public void testGetStreams() throws Exception { + InfoItemCollector collector = extractor.getStreams(); + if(!collector.getErrors().isEmpty()) { + System.err.println("----------"); + for(Throwable e : collector.getErrors()) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !collector.getItemList().isEmpty() + && collector.getErrors().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getStreams(); + assertTrue("has more streams", extractor.hasMoreStreams()); + } + + @Test + public void testGetNextStreams() throws Exception { + extractor.getStreams(); + assertFalse("extractor has next streams", extractor.getNextStreams() == null + || extractor.getNextStreams().nextItemsList.isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top"); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java new file mode 100644 index 000000000..6429f6c3f --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java @@ -0,0 +1,49 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; + +/** + * Test for {@link SoundcloudChartsUrlIdHandler} + */ +public class SoundcloudChartsUrlIdHandlerTest { + private SoundcloudChartsUrlIdHandler urlIdHandler; + + @Before + public void setUp() throws Exception { + urlIdHandler = new SoundcloudChartsUrlIdHandler(); + NewPipe.init(Downloader.getInstance()); + } + + @Test + public void getUrl() { + assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top"); + assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new"); + } + + @Test + public void getId() { + assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50"); + assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot"); + } + + @Test + public void acceptUrl() { + assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts")); + assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/")); + assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new")); + assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/charts/top?genre=all-music")); + assertTrue(urlIdHandler.acceptUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries")); + + assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia")); + assertFalse(urlIdHandler.acceptUrl("soundcloud.com/charts askjkf")); + assertFalse(urlIdHandler.acceptUrl(" soundcloud.com/charts")); + assertFalse(urlIdHandler.acceptUrl("")); + } +} From d07f6cdef7ebc4a1d6f0cb977e474fcf5b904aca Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 22 Aug 2017 14:47:51 +0200 Subject: [PATCH 10/10] Use content country if supported by SoundCloud --- .../services/soundcloud/SoundcloudChartsExtractor.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index f4aca4e2d..cb83c72e3 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.soundcloud; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; @@ -57,6 +59,12 @@ public class SoundcloudChartsExtractor extends KioskExtractor { apiUrl += "&kind=new"; } + List supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US"); + String contentCountry = getContentCountry(); + if (supportedCountries.contains(contentCountry)) { + apiUrl += "®ion=soundcloud:regions:" + contentCountry; + } + nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); return collector; }