From 20f280cb5700d025059480d15d91f7b65c27cfec Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 12 Oct 2018 00:40:22 +0530 Subject: [PATCH 01/34] added peertube extractor --- .../schabi/newpipe/extractor/MediaFormat.java | 8 + .../schabi/newpipe/extractor/ServiceList.java | 6 +- .../newpipe/extractor/StreamingService.java | 9 +- .../services/peertube/PeertubeInstance.java | 76 +++++ .../peertube/PeertubeParsingHelper.java | 23 ++ .../services/peertube/PeertubeService.java | 175 ++++++++++++ .../extractors/PeertubeChannelExtractor.java | 181 ++++++++++++ .../extractors/PeertubeCommentsExtractor.java | 124 +++++++++ .../PeertubeCommentsInfoItemExtractor.java | 88 ++++++ .../extractors/PeertubePlaylistExtractor.java | 87 ++++++ .../extractors/PeertubeSearchExtractor.java | 131 +++++++++ .../extractors/PeertubeStreamExtractor.java | 260 ++++++++++++++++++ .../PeertubeStreamInfoItemExtractor.java | 78 ++++++ .../PeertubeSubscriptionExtractor.java | 21 ++ .../PeertubeSuggestionExtractor.java | 23 ++ .../extractors/PeertubeTrendingExtractor.java | 130 +++++++++ .../PeertubeChannelLinkHandlerFactory.java | 35 +++ .../PeertubeCommentsLinkHandlerFactory.java | 41 +++ .../PeertubePlaylistLinkHandlerFactory.java | 37 +++ .../PeertubeSearchQueryHandlerFactory.java | 38 +++ .../PeertubeStreamLinkHandlerFactory.java | 37 +++ .../PeertubeTrendingLinkHandlerFactory.java | 51 ++++ .../soundcloud/SoundcloudService.java | 5 + .../services/youtube/YoutubeService.java | 5 + .../newpipe/extractor/utils/JsonUtils.java | 20 ++ .../PeertubeChannelExtractorTest.java | 207 ++++++++++++++ ...PeertubeChannelLinkHandlerFactoryTest.java | 37 +++ .../PeertubeCommentsExtractorTest.java | 93 +++++++ ...eertubeCommentsLinkHandlerFactoryTest.java | 37 +++ ...eertubePlaylistLinkHandlerFactoryTest.java | 37 +++ .../PeertubeStreamExtractorDefaultTest.java | 113 ++++++++ .../PeertubeStreamLinkHandlerFactoryTest.java | 38 +++ .../PeertubeTrendingExtractorTest.java | 96 +++++++ ...eertubeTrendingLinkHandlerFactoryTest.java | 54 ++++ .../PeertubeSearchExtractorBaseTest.java | 28 ++ .../PeertubeSearchExtractorDefaultTest.java | 91 ++++++ .../peertube/search/PeertubeSearchQHTest.java | 18 ++ 37 files changed, 2536 insertions(+), 2 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index e39e6977a..c0e44f758 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -117,6 +117,13 @@ public enum MediaFormat { return null; } + public static MediaFormat getFromSuffix(String suffix) { + for (MediaFormat vf: values()) { + if (vf.suffix.equals(suffix)) return vf; + } + return null; + } + /** * Get the name of the format * @return the name of the format @@ -140,4 +147,5 @@ public enum MediaFormat { public String getMimeType() { return mimeType; } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 846355ede..4d3dffc3d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor; +import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; @@ -18,11 +19,14 @@ public final class ServiceList { public static final YoutubeService YouTube; public static final SoundcloudService SoundCloud; + public static final PeertubeService PeerTube; private static final List SERVICES = unmodifiableList( asList( YouTube = new YoutubeService(0), - SoundCloud = new SoundcloudService(1) + SoundCloud = new SoundcloudService(1), + PeerTube = new PeertubeService(2) + )); /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index bab6c7f4f..418a61e15 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -22,7 +22,8 @@ import org.schabi.newpipe.extractor.utils.Localization; public abstract class StreamingService { public static class ServiceInfo { - private final String name; + private String name; + private final List mediaCapabilities; public ServiceInfo(String name, List mediaCapabilities) { @@ -33,6 +34,10 @@ public abstract class StreamingService { public String getName() { return name; } + + public void setName(String name) { + this.name = name; + } public List getMediaCapabilities() { return mediaCapabilities; @@ -191,6 +196,8 @@ public abstract class StreamingService { } public abstract boolean isCommentsSupported(); + + public abstract String getBaseUrl(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java new file mode 100644 index 000000000..e08477509 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -0,0 +1,76 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeInstance { + + private final String url; + private String name; + public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + + public PeertubeInstance(String url) throws IOException { + this.url = url; + String response = validateInstance(url); + setInstanceMetaData(response); + } + + private PeertubeInstance(String url , String name) { + this.url = url; + this.name = name; + } + + public String getUrl() { + return url; + } + + private String validateInstance(String url) throws IOException { + Downloader downloader = NewPipe.getDownloader(); + DownloadResponse response = null; + + try { + response = downloader.get(url + "/api/v1/config"); + } catch (ReCaptchaException | IOException e) { + throw new IOException("unable to configure instance " + url, e); + } + + if(null == response || StringUtil.isBlank(response.getResponseBody())) { + throw new IOException("unable to configure instance " + url); + } + + return response.getResponseBody(); + } + + private void setInstanceMetaData(String responseBody) { + JsonObject json; + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + return; + } + + if(null == json) return; + + try { + this.name = JsonUtils.getString(json, "instance.name"); + } catch (ParsingException e) { + return; + } + } + + public String getName() { + return name; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java new file mode 100644 index 000000000..b50cede26 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -0,0 +1,23 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public class PeertubeParsingHelper { + + private PeertubeParsingHelper() { + } + + public static String toDateString(String time) throws ParsingException { + try { + Date date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(time); + SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + return newDateFormat.format(date); + } catch (ParseException e) { + throw new ParsingException(e.getMessage(), e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java new file mode 100644 index 000000000..66f5412cc --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -0,0 +1,175 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + +import java.io.IOException; +import java.util.List; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +public class PeertubeService extends StreamingService { + + private PeertubeInstance instance; + + public PeertubeService(int id) { + this(id, PeertubeInstance.defaultInstance); + } + + public PeertubeService(int id, PeertubeInstance instance) { + super(id, instance.getName(), singletonList(VIDEO)); + this.instance = instance; + } + + public PeertubeService(int id, String name, List capabilities) { + super(id, name, capabilities); + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return PeertubeStreamLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + return PeertubeChannelLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + // TODO Auto-generated method stub + return null; + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return PeertubeSearchQueryHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return PeertubeCommentsLinkHandlerFactory.getInstance(); + } + + @Override + public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization) { + return new PeertubeSearchExtractor(this, queryHandler, localization); + } + + @Override + public SuggestionExtractor getSuggestionExtractor(Localization localization) { + // TODO Auto-generated method stub + return null; + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + // TODO Auto-generated method stub + return null; + } + + @Override + public KioskList getKioskList(Localization localization) throws ExtractionException { + KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, + String url, + String id, + Localization local) + throws ExtractionException { + return new PeertubeTrendingExtractor(PeertubeService.this, + new PeertubeTrendingLinkHandlerFactory().fromId(id), id, local); + } + }; + + KioskList list = new KioskList(getServiceId(), localization); + + // add kiosks here e.g.: + final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory(); + try { + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL); + list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + } catch (Exception e) { + throw new ExtractionException(e); + } + + return list; + } + + @Override + public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) + throws ExtractionException { + return new PeertubeChannelExtractor(this, linkHandler, localization); + } + + @Override + public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) + throws ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) + throws ExtractionException { + return new PeertubeStreamExtractor(this, linkHandler, localization); + } + + @Override + public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, Localization localization) + throws ExtractionException { + return new PeertubeCommentsExtractor(this, linkHandler, localization); + } + + @Override + public boolean isCommentsSupported() { + return true; + } + + @Override + public String getBaseUrl() { + return instance.getUrl(); + } + + public void setInstance(String url) throws IOException { + this.instance = new PeertubeInstance(url); + if(!StringUtil.isBlank(instance.getName())) { + this.getServiceInfo().setName(instance.getName()); + }else { + this.getServiceInfo().setName("PeerTube"); + } + } + + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java new file mode 100644 index 000000000..efab4c63b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -0,0 +1,181 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeChannelExtractor extends ChannelExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + private JsonObject json; + + public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + } + + @Override + public String getAvatarUrl() throws ParsingException { + String value = JsonUtils.getString(json, "avatar.path"); + return ServiceList.PeerTube.getBaseUrl() + value; + } + + @Override + public String getBannerUrl() throws ParsingException { + return null; + } + + @Override + public String getFeedUrl() throws ParsingException { + return null; + } + + @Override + public long getSubscriberCount() throws ParsingException { + Number number = JsonUtils.getNumber(json, "followersCount"); + return number.longValue(); + } + + @Override + public String getDescription() throws ParsingException { + try { + return JsonUtils.getString(json, "description"); + }catch(ParsingException e) { + return "No description"; + } + } + + @Override + public String[] getDonationLinks() throws ParsingException { + return new String[0]; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract channel streams", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + DownloadResponse response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + json = JsonParser.object().from(response.getResponseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + DownloadResponse response = downloader.get(getUrl()); + if(null != response && null != response.getResponseBody()) { + setInitialData(response.getResponseBody()); + }else { + throw new ExtractionException("Unable to extract peertube channel data"); + } + + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract peertube channel data", e); + } + if(null == json) throw new ExtractionException("Unable to extract peertube channel data"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "displayName"); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java new file mode 100644 index 000000000..fde931fa4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -0,0 +1,124 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeCommentsExtractor extends CommentsExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler, Localization localization) { + super(service, uiHandler, localization); + } + + @Override + public String getName() throws ParsingException { + return "Comments"; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract comments info", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, pageUrl); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + DownloadResponse response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + json = JsonParser.object().from(response.getResponseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for comments info", e); + } + } + + CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube comments info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java new file mode 100644 index 000000000..ea65aae3c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -0,0 +1,88 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; + + +public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + + private final JsonObject item; + private final String url; + + public PeertubeCommentsInfoItemExtractor(JsonObject item, String url) { + this.item = item; + this.url = url; + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(item, "account.avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return ServiceList.PeerTube.getBaseUrl() + value; + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getPublishedTime() throws ParsingException { + return JsonUtils.getString(item, "createdAt"); + } + + @Override + public Integer getLikeCount() throws ParsingException { + return 0; + } + + @Override + public String getCommentText() throws ParsingException { + String htmlText = JsonUtils.getString(item, "text"); + return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); + } + + @Override + public String getCommentId() throws ParsingException { + Number value = JsonUtils.getNumber(item, "id"); + return value.toString(); + } + + @Override + public String getAuthorThumbnail() throws ParsingException { + String value; + try { + value = JsonUtils.getString(item, "account.avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return ServiceList.PeerTube.getBaseUrl() + value; + } + + @Override + public String getAuthorName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getAuthorEndpoint() throws ParsingException { + String name = JsonUtils.getString(item, "account.name"); + String host = JsonUtils.getString(item, "account.host"); + return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java new file mode 100644 index 000000000..a3ef309e4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -0,0 +1,87 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.Localization; + +public class PeertubePlaylistExtractor extends PlaylistExtractor{ + + public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + // TODO Auto-generated constructor stub + } + + @Override + public String getThumbnailUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getBannerUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderName() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderAvatarUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getStreamCount() throws ParsingException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + // TODO Auto-generated method stub + + } + + @Override + public String getName() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java new file mode 100644 index 000000000..61c83a2b2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -0,0 +1,131 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeSearchExtractor extends SearchExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, + Localization localization) { + super(service, linkHandler, localization); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + return null; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private InfoItemsCollector collectStreamsFrom(JsonObject json) throws ParsingException { + + final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract search info", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + collector.commit(extractor); + } + } + + return collector; + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + DownloadResponse response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + json = JsonParser.object().from(response.getResponseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for search info", e); + } + } + + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl)); + } else { + throw new ExtractionException("Unable to get peertube search info"); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java new file mode 100644 index 000000000..c35d7654e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -0,0 +1,260 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.Subtitles; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesFormat; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeStreamExtractor extends StreamExtractor { + + + private JsonObject json; + + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + } + + @Override + public String getUploadDate() throws ParsingException { + String date = JsonUtils.getString(json, "publishedAt"); + return PeertubeParsingHelper.toDateString(date); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return ServiceList.PeerTube.getBaseUrl() + JsonUtils.getString(json, "thumbnailPath"); + } + + @Override + public String getDescription() throws ParsingException { + return JsonUtils.getString(json, "description"); + } + + @Override + public int getAgeLimit() throws ParsingException { + return NO_AGE_LIMIT; + } + + @Override + public long getLength() throws ParsingException { + Number value = JsonUtils.getNumber(json, "duration"); + return value.longValue(); + } + + @Override + public long getTimeStamp() throws ParsingException { + //TODO fetch timestamp from url if present; + return 0; + } + + @Override + public long getViewCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "views"); + return value.longValue(); + } + + @Override + public long getLikeCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "likes"); + return value.longValue(); + } + + @Override + public long getDislikeCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "dislikes"); + return value.longValue(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String name = JsonUtils.getString(json, "account.name"); + String host = JsonUtils.getString(json, "account.host"); + return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + } + + @Override + public String getUploaderName() throws ParsingException { + return JsonUtils.getString(json, "account.displayName"); + } + + @Override + public String getUploaderAvatarUrl() throws ParsingException { + String avatarPath = JsonUtils.getString(json, "account.avatar.path"); + return ServiceList.PeerTube.getBaseUrl() + avatarPath; + } + + @Override + public String getDashMpdUrl() throws ParsingException { + return ""; + } + + @Override + public String getHlsUrl() throws ParsingException { + return ""; + } + + @Override + public List getAudioStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public List getVideoStreams() throws IOException, ExtractionException { + assertPageFetched(); + List videoStreams = new ArrayList<>(); + try { + JsonArray streams = json.getArray("files", new JsonArray()); + for(Object s: streams) { + if(!(s instanceof JsonObject)) continue; + JsonObject stream = (JsonObject) s; + String url = JsonUtils.getString(stream, "fileUrl"); + String resolution = JsonUtils.getString(stream, "resolution.label"); + String extension = url.substring(url.lastIndexOf(".") + 1); + MediaFormat format = MediaFormat.getFromSuffix(extension); + VideoStream videoStream = new VideoStream(url, format, resolution); + if (!Stream.containSimilarStream(videoStream, videoStreams)) { + videoStreams.add(videoStream); + } + } + } catch (Exception e) { + throw new ParsingException("Could not get video streams", e); + } + + return videoStreams; + } + + + @Override + public List getVideoOnlyStreams() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getSubtitlesDefault() throws IOException, ExtractionException { + return Collections.emptyList(); + } + + @Override + public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { + return Collections.emptyList(); + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public StreamInfoItem getNextVideo() throws IOException, ExtractionException { + return null; + } + + @Override + public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + //TODO fetch related videos not trending + String apiUrl = new PeertubeTrendingLinkHandlerFactory().getUrl(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + getStreamsFromApi(collector, apiUrl); + return collector; + } + + private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { + DownloadResponse response = getDownloader().get(apiUrl); + JsonObject relatedVideosJson = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + relatedVideosJson = JsonParser.object().from(response.getResponseBody()); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for related videos", e); + } + } + + if(relatedVideosJson != null) { + collectStreamsFrom(collector, relatedVideosJson); + } + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract related videos", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + collector.commit(extractor); + } + } + + } + + + @Override + public String getErrorMessage() { + return null; + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + DownloadResponse response = downloader.get(getUrl()); + if(null != response && null != response.getResponseBody()) { + setInitialData(response.getResponseBody()); + }else { + throw new ExtractionException("Unable to extract peertube channel data"); + } + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract peertube stream data", e); + } + if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "name"); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java new file mode 100644 index 000000000..b73fdc609 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -0,0 +1,78 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; + +public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { + + protected final JsonObject item; + + public PeertubeStreamInfoItemExtractor(JsonObject item) { + this.item = item; + } + + @Override + public String getUrl() throws ParsingException { + String uuid = JsonUtils.getString(item, "uuid"); + return PeertubeStreamLinkHandlerFactory.getInstance().fromId(uuid).getUrl(); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + String value = JsonUtils.getString(item, "thumbnailPath"); + return ServiceList.PeerTube.getBaseUrl() + value; + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(item, "name"); + } + + @Override + public boolean isAd() throws ParsingException { + return false; + } + + @Override + public long getViewCount() throws ParsingException { + Number value = JsonUtils.getNumber(item, "views"); + return value.longValue(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String name = JsonUtils.getString(item, "account.name"); + String host = JsonUtils.getString(item, "account.host"); + return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + } + + @Override + public String getUploaderName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getUploadDate() throws ParsingException { + String date = JsonUtils.getString(item, "publishedAt"); + return PeertubeParsingHelper.toDateString(date); + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public long getDuration() throws ParsingException { + Number value = JsonUtils.getNumber(item, "duration"); + return value.longValue(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java new file mode 100644 index 000000000..a2b246141 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java @@ -0,0 +1,21 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.util.List; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; + +public class PeertubeSubscriptionExtractor extends SubscriptionExtractor { + + public PeertubeSubscriptionExtractor(StreamingService service, List supportedSources) { + super(service, supportedSources); + // TODO Auto-generated constructor stub + } + + @Override + public String getRelatedUrl() { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java new file mode 100644 index 000000000..ec36fc83f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java @@ -0,0 +1,23 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; +import java.util.List; + +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.utils.Localization; + +public class PeertubeSuggestionExtractor extends SuggestionExtractor{ + + public PeertubeSuggestionExtractor(int serviceId, Localization localization) { + super(serviceId, localization); + // TODO Auto-generated constructor stub + } + + @Override + public List suggestionList(String query) throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java new file mode 100644 index 000000000..5aa5c0323 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -0,0 +1,130 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeTrendingExtractor extends KioskExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeTrendingExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId, + Localization localization) { + super(streamingService, linkHandler, kioskId, localization); + } + + @Override + public String getName() throws ParsingException { + return getId(); + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract kiosk info", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + DownloadResponse response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + json = JsonParser.object().from(response.getResponseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java new file mode 100644 index 000000000..2d93b9f58 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); + private static final String ID_PATTERN = "/accounts/([^/?&#]*)"; + private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/"; + + public static PeertubeChannelLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getId(String url) throws ParsingException { + return Parser.matchGroup1(ID_PATTERN, url); + } + + @Override + public String getUrl(String id, List contentFilters, String searchFilter) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return baseUrl + ACCOUNTS_ENDPOINT + id; + } + + @Override + public boolean onAcceptUrl(String url) { + return url.contains("/accounts/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java new file mode 100644 index 000000000..dcd0c3256 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java @@ -0,0 +1,41 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory(); + private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)"; + private static final String COMMENTS_ENDPOINT = "/api/v1/videos/%s/comment-threads"; + + public static PeertubeCommentsLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return baseUrl + String.format(COMMENTS_ENDPOINT, id); + } + + @Override + public String getId(String url) throws ParsingException, IllegalArgumentException { + return Parser.matchGroup(ID_PATTERN, url, 2); + } + + @Override + public boolean onAcceptUrl(final String url) throws FoundAdException { + return url.contains("/videos/"); + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + return getUrl(id); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java new file mode 100644 index 000000000..fa6413589 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory(); + private static final String ID_PATTERN = "/video-channels/([^/?&#]*)"; + private static final String VIDEO_CHANNELS_ENDPOINT = "/api/v1/video-channels/"; + + public static PeertubePlaylistLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id, List contentFilters, String sortFilter) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return baseUrl + VIDEO_CHANNELS_ENDPOINT + id; + } + + @Override + public String getId(String url) throws ParsingException { + return Parser.matchGroup1(ID_PATTERN, url); + } + + + @Override + public boolean onAcceptUrl(final String url) { + return url.contains("/video-channels/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java new file mode 100644 index 000000000..536d1bf42 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -0,0 +1,38 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; + +public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + + public static final String CHARSET_UTF_8 = "UTF-8"; + public static final String VIDEOS = "videos"; + private static final String SEARCH_ENDPOINT = "/api/v1/search/videos"; + + public static PeertubeSearchQueryHandlerFactory getInstance() { + return new PeertubeSearchQueryHandlerFactory(); + } + + @Override + public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + try { + final String url = baseUrl + SEARCH_ENDPOINT + + "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8); + + return url; + } catch (UnsupportedEncodingException e) { + throw new ParsingException("Could not encode query", e); + } + } + + @Override + public String[] getAvailableContentFilter() { + return new String[] { VIDEOS }; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java new file mode 100644 index 000000000..880c4d937 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { + + private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory(); + private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)"; + private static final String VIDEO_ENDPOINT = "/api/v1/videos/"; + + private PeertubeStreamLinkHandlerFactory() { + } + + public static PeertubeStreamLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return baseUrl + VIDEO_ENDPOINT + id; + } + + @Override + public String getId(String url) throws ParsingException, IllegalArgumentException { + return Parser.matchGroup(ID_PATTERN, url, 2); + } + + @Override + public boolean onAcceptUrl(final String url) throws FoundAdException { + return url.contains("/videos/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java new file mode 100644 index 000000000..c4358998c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -0,0 +1,51 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + + + public static final Map KIOSK_MAP; + public static final String KIOSK_TRENDING = "Trending"; + public static final String KIOSK_RECENT = "Recently added"; + public static final String KIOSK_LOCAL = "Local"; + + static { + Map map = new HashMap<>(); + map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-views"); + map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); + map.put(KIOSK_LOCAL, "%s/api/v1/videos?filter=local"); + KIOSK_MAP = Collections.unmodifiableMap(map); + } + + public String getUrl(String id, List contentFilters, String sortFilter) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return String.format(KIOSK_MAP.get(id), baseUrl); + } + + @Override + public String getId(String url) throws ParsingException { + + if(url.contains("/videos/trending")) { + return KIOSK_TRENDING; + }else if(url.contains("/videos/recently-added")) { + return KIOSK_RECENT; + }else if(url.contains("/videos/local")){ + return KIOSK_LOCAL; + }else { + throw new ParsingException("no id found for this url"); + } + } + + @Override + public boolean onAcceptUrl(final String url) { + return url.contains("/videos/trending") || url.contains("/videos/recently-added") || url.contains("/videos/local"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index a0008f765..fe63a966a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -117,5 +117,10 @@ public class SoundcloudService extends StreamingService { public boolean isCommentsSupported() { return false; } + + @Override + public String getBaseUrl() { + return "https://soundcloud.com"; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index a505d60d2..64a4565f9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -155,4 +155,9 @@ public class YoutubeService extends StreamingService { return true; } + @Override + public String getBaseUrl() { + return "https://youtube.com"; + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ddbf47c81..2844e6d2c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -28,6 +28,26 @@ public class JsonUtils { return result; } + @Nonnull + public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + Object value = getValue(object, path); + if(value instanceof String) { + return (String) getValue(object, path); + }else { + throw new ParsingException("Unable to get " + path); + } + } + + @Nonnull + public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + Object value = getValue(object, path); + if(value instanceof Number) { + return (Number) getValue(object, path); + }else { + throw new ParsingException("Unable to get " + path); + } + } + @Nonnull public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java new file mode 100644 index 000000000..573a51455 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -0,0 +1,207 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeChannelExtractor} + */ +public class PeertubeChannelExtractorTest { + public static class LilUzi implements BaseChannelExtractorTest { + private static PeertubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeChannelExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("Noureddine HADDAG", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("root@tube.openalgeria.org", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEmpty(extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + } + } + + public static class DubMatix implements BaseChannelExtractorTest { + private static PeertubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeChannelExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Additional Testing + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testGetPageInNewExtractor() throws Exception { + final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl()); + defaultTestGetPageInNewExtractor(extractor, newExtractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("France Inter", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("franceinter@tube.kdy.ch", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/franceinter@tube.kdy.ch", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEmpty(extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 75); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java new file mode 100644 index 000000000..d50e5774c --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeChannelLinkHandlerFactory} + */ +public class PeertubeChannelLinkHandlerFactoryTest { + + private static PeertubeChannelLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubeChannelLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); + assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java new file mode 100644 index 000000000..0074ea6e1 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -0,0 +1,93 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import java.io.IOException; +import java.util.List; + +import org.jsoup.helper.StringUtil; +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.comments.CommentsInfo; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +public class PeertubeCommentsExtractorTest { + + private static PeertubeCommentsExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeCommentsExtractor) PeerTube + .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + } + + @Test + public void testGetComments() throws IOException, ExtractionException { + boolean result = false; + InfoItemsPage comments = extractor.getInitialPage(); + result = findInComments(comments, "@root A great documentary on a great guy."); + + while (comments.hasNextPage() && !result) { + comments = extractor.getPage(comments.getNextPageUrl()); + result = findInComments(comments, "@root A great documentary on a great guy."); + } + + assertTrue(result); + } + + @Test + public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { + boolean result = false; + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + assertTrue("Comments".equals(commentsInfo.getName())); + result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); + + String nextPage = commentsInfo.getNextPageUrl(); + while (!StringUtil.isBlank(nextPage) && !result) { + InfoItemsPage moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage); + result = findInComments(moreItems.getItems(), "Loved it!!!"); + nextPage = moreItems.getNextPageUrl(); + } + + assertTrue(result); + } + + @Test + public void testGetCommentsAllData() throws IOException, ExtractionException { + InfoItemsPage comments = extractor.getInitialPage(); + for(CommentsInfoItem c: comments.getItems()) { + assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); + assertFalse(StringUtil.isBlank(c.getAuthorName())); + assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); + assertFalse(StringUtil.isBlank(c.getCommentId())); + assertFalse(StringUtil.isBlank(c.getCommentText())); + assertFalse(StringUtil.isBlank(c.getName())); + assertFalse(StringUtil.isBlank(c.getPublishedTime())); + assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); + assertFalse(StringUtil.isBlank(c.getUrl())); + assertFalse(c.getLikeCount() == null); + } + } + + private boolean findInComments(InfoItemsPage comments, String comment) { + return findInComments(comments.getItems(), comment); + } + + private boolean findInComments(List comments, String comment) { + for(CommentsInfoItem c: comments) { + if(c.getCommentText().contains(comment)) { + return true; + } + } + return false; + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java new file mode 100644 index 000000000..ce4fdd14a --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeCommentsLinkHandlerFactory} + */ +public class PeertubeCommentsLinkHandlerFactoryTest { + + private static PeertubeCommentsLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubeCommentsLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("19319", linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads").getId()); + assertEquals("19319", linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java new file mode 100644 index 000000000..9705efc63 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubePlaylistLinkHandlerFactory} + */ +public class PeertubePlaylistLinkHandlerFactoryTest { + + private static PeertubePlaylistLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubePlaylistLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33").getId()); + assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java new file mode 100644 index 000000000..d6e5494be --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -0,0 +1,113 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Localization; + +import java.io.IOException; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +/** + * Test for {@link StreamExtractor} + */ +public class PeertubeStreamExtractorDefaultTest { + private static PeertubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/86fe4f24-64c3-4ab4-9e7e-66177219ed21"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetTitle() throws ParsingException { + assertEquals(extractor.getName(), "Pažadėtoji 1 Sezonas 1050 Serija - Ziuri.me"); + } + + @Test + public void testGetDescription() throws ParsingException { + assertEquals(extractor.getDescription(), "Serialo veiksmas vyksta Radžastane. kur vis dar gyvos liaudies tradicijos. o jo centre - Anandi gimusi varguolių šeimoje. Mergaitė privalės ištekėti už turtingo paveldėtojo vos tik sulaukusi aštuonerių metų ir priprasti prie naujojo nuotakos vaidm..."); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertEquals(extractor.getUploaderName(), "djsets"); + } + + @Test + public void testGetLength() throws ParsingException { + assertEquals(extractor.getLength(), 1238); + } + + @Test + public void testGetViewCount() throws ParsingException { + assertTrue(Long.toString(extractor.getViewCount()), + extractor.getViewCount() > 700); + } + + @Test + public void testGetUploadDate() throws ParsingException { + assertEquals("2018-10-06", extractor.getUploadDate()); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/djsets@hostyour.tv", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetVideoStreams() throws IOException, ExtractionException { + assertFalse(extractor.getVideoStreams().isEmpty()); + } + + @Test + public void testStreamType() throws ParsingException { + assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java new file mode 100644 index 000000000..b82632b2f --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java @@ -0,0 +1,38 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeStreamLinkHandlerFactory} + */ +public class PeertubeStreamLinkHandlerFactoryTest { + private static PeertubeStreamLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() throws Exception { + linkHandler = PeertubeStreamLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void getId() throws Exception { + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId()); + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId()); + } + + + @Test + public void testAcceptUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa")); + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java new file mode 100644 index 000000000..e11bb5ee0 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -0,0 +1,96 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import java.util.List; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeTrendingExtractor} + */ +public class PeertubeTrendingExtractorTest { + + static KioskExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = PeerTube + .getKioskList() + .getExtractorById("Trending", null); + extractor.fetchPage(); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getName(), "Trending"); + } + + @Test + public void testId() { + assertEquals(extractor.getId(), "Trending"); + } + + @Test + public void testGetStreams() throws Exception { + ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + if(!page.getErrors().isEmpty()) { + System.err.println("----------"); + List errors = page.getErrors(); + for(Throwable e: errors) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !page.getItems().isEmpty() + && page.getErrors().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getInitialPage(); + assertTrue("has more streams", extractor.hasNextPage()); + } + + @Test + public void testGetNextPageUrl() throws Exception { + assertTrue(extractor.hasNextPage()); + } + + @Test + public void testGetNextPage() throws Exception { + extractor.getInitialPage().getItems(); + assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null + || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-views"); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java new file mode 100644 index 000000000..6febed8d0 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -0,0 +1,54 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeTrendingLinkHandlerFactory} + */ +public class PeertubeTrendingLinkHandlerFactoryTest { + private static LinkHandlerFactory LinkHandlerFactory; + + @BeforeClass + public static void setUp() throws Exception { + LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void getUrl() + throws Exception { + assertEquals(LinkHandlerFactory.fromId("Trending").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-views"); + assertEquals(LinkHandlerFactory.fromId("Recently added").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt"); + assertEquals(LinkHandlerFactory.fromId("Local").getUrl(), "https://peertube.mastodon.host/api/v1/videos?filter=local"); + } + + @Test + public void getId() + throws Exception { + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/trending").getId(), "Trending"); + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/recently-added").getId(), "Recently added"); + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/local").getId(), "Local"); + } + + @Test + public void acceptUrl() throws ParsingException { + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending?adsf=fjaj#fhe")); + + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added?adsf=fjaj#fhe")); + + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local?adsf=fjaj#fhe")); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java new file mode 100644 index 000000000..08dd7a0a9 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java @@ -0,0 +1,28 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; + +/** + * Test for {@link PeertubeSearchExtractor} + */ +public abstract class PeertubeSearchExtractorBaseTest { + + protected static PeertubeSearchExtractor extractor; + protected static ListExtractor.InfoItemsPage itemsPage; + + @Test + public void testResultListElementsLength() { + assertTrue(Integer.toString(itemsPage.getItems().size()), + itemsPage.getItems().size() >= 3); + } + + @Test + public void testUrl() throws Exception { + assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://peertube.mastodon.host/api/v1/search/videos")); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java new file mode 100644 index 000000000..439688257 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -0,0 +1,91 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeSearchExtractor} + */ +public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorBaseTest { + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet's own boy"); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testGetSecondPageUrl() throws Exception { + assertEquals("", extractor.getNextPageUrl()); + } + + @Test + public void testResultList_FirstElement() { + InfoItem firstInfoItem = itemsPage.getItems().get(0); + + // THe channel should be the first item + assertEquals("name", "The Internet's Own Boy", firstInfoItem.getName()); + assertEquals("url","https://peertube.mastodon.host/api/v1/videos/04af977f-4201-4697-be67-a8d8cae6fa7a", firstInfoItem.getUrl()); + } + + @Test + public void testResultListCheckIfContainsStreamItems() { + boolean hasStreams = false; + for(InfoItem item : itemsPage.getItems()) { + if(item instanceof StreamInfoItem) { + hasStreams = true; + } + } + assertTrue("Has no InfoItemStreams", hasStreams); + } + + @Test + public void testGetSecondPage() throws Exception { + extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); + itemsPage = extractor.getInitialPage(); + PeertubeSearchExtractor secondExtractor = + (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); + ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); + assertTrue(Integer.toString(secondPage.getItems().size()), + secondPage.getItems().size() >= 10); + + // check if its the same result + boolean equals = true; + for (int i = 0; i < secondPage.getItems().size() + && i < itemsPage.getItems().size(); i++) { + if(!secondPage.getItems().get(i).getUrl().equals( + itemsPage.getItems().get(i).getUrl())) { + equals = false; + } + } + assertFalse("First and second page are equal", equals); + + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12", + secondPage.getNextPageUrl()); + } + + + @Test + public void testId() throws Exception { + assertEquals("internet's own boy", extractor.getId()); + } + + @Test + public void testName() { + assertEquals("internet's own boy", extractor.getName()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java new file mode 100644 index 000000000..89680d609 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import org.junit.Test; + +public class PeertubeSearchQHTest { + + @Test + public void testRegularValues() throws Exception { + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans",PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + } +} From 9dee8d5fcd2054fd62e04161e9d75754f4f8d47a Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 12 Oct 2018 01:37:47 +0530 Subject: [PATCH 02/34] updated kiosk accept link --- .../linkHandler/PeertubeTrendingLinkHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index c4358998c..b3763ccb8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -46,6 +46,6 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { - return url.contains("/videos/trending") || url.contains("/videos/recently-added") || url.contains("/videos/local"); + return url.contains("/videos?") || url.contains("/videos/trending") || url.contains("/videos/recently-added") || url.contains("/videos/local"); } } From 26da3088ba09c57e1f515b6527487c711a8d9bf0 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 12 Oct 2018 01:59:13 +0530 Subject: [PATCH 03/34] fallback avatar and description --- .../extractors/PeertubeStreamExtractor.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index c35d7654e..58fea28d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.DownloadResponse; @@ -20,8 +19,6 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -60,7 +57,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getDescription() throws ParsingException { - return JsonUtils.getString(json, "description"); + try { + return JsonUtils.getString(json, "description"); + }catch(ParsingException e) { + return "No description"; + } } @Override @@ -112,8 +113,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getUploaderAvatarUrl() throws ParsingException { - String avatarPath = JsonUtils.getString(json, "account.avatar.path"); - return ServiceList.PeerTube.getBaseUrl() + avatarPath; + String value; + try { + value = JsonUtils.getString(json, "account.avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return ServiceList.PeerTube.getBaseUrl() + value; } @Override From c9fba9cb1e12a96266c6703c045a85a2f9c99ad9 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 12 Oct 2018 02:27:32 +0530 Subject: [PATCH 04/34] fallback avatar for channel --- .../peertube/extractors/PeertubeChannelExtractor.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index efab4c63b..b356f8495 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -41,7 +41,12 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() throws ParsingException { - String value = JsonUtils.getString(json, "avatar.path"); + String value; + try { + value = JsonUtils.getString(json, "avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } return ServiceList.PeerTube.getBaseUrl() + value; } From 706d4271a612332f651c07c0ec71fa859d7b8987 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 12 Oct 2018 12:45:09 +0530 Subject: [PATCH 05/34] empty suggestions for search --- .../peertube/extractors/PeertubeSuggestionExtractor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java index ec36fc83f..200d633ac 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import java.io.IOException; +import java.util.Collections; import java.util.List; import org.schabi.newpipe.extractor.SuggestionExtractor; @@ -11,13 +12,11 @@ public class PeertubeSuggestionExtractor extends SuggestionExtractor{ public PeertubeSuggestionExtractor(int serviceId, Localization localization) { super(serviceId, localization); - // TODO Auto-generated constructor stub } @Override public List suggestionList(String query) throws IOException, ExtractionException { - // TODO Auto-generated method stub - return null; + return Collections.emptyList(); } } From 4883b6fb480a42813ad105ac5e88c9dacdb0eeb7 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 12 Oct 2018 13:04:51 +0530 Subject: [PATCH 06/34] return suggestion extractor --- .../newpipe/extractor/services/peertube/PeertubeService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 66f5412cc..bea320b52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannel import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; @@ -87,8 +88,7 @@ public class PeertubeService extends StreamingService { @Override public SuggestionExtractor getSuggestionExtractor(Localization localization) { - // TODO Auto-generated method stub - return null; + return new PeertubeSuggestionExtractor(this.getServiceId(), localization); } @Override From 8755c25349c1497448acce64342a641ec07cfca0 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Wed, 26 Dec 2018 11:08:39 +0530 Subject: [PATCH 07/34] format date of comment published. changed default instance --- .../services/peertube/PeertubeInstance.java | 2 +- .../PeertubeCommentsInfoItemExtractor.java | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index e08477509..2bedc07cf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -18,7 +18,7 @@ public class PeertubeInstance { private final String url; private String name; - public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); public PeertubeInstance(String url) throws IOException { this.url = url; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index ea65aae3c..1e6df7708 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -1,5 +1,10 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Locale; + import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -42,7 +47,8 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac @Override public String getPublishedTime() throws ParsingException { - return JsonUtils.getString(item, "createdAt"); + String date = JsonUtils.getString(item, "createdAt"); + return getFormattedDate(date); } @Override @@ -84,5 +90,15 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String host = JsonUtils.getString(item, "account.host"); return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); } + + private String getFormattedDate(String date) { + DateFormat sourceDf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + DateFormat targetDf = new SimpleDateFormat("dd-MM-yyyy hh:mm a", Locale.ENGLISH); + try { + return targetDf.format(sourceDf.parse(date)); + } catch (ParseException e) { + return date; + } + } } From 14cd1dbcbb112faeeba78c775f4a0dd985fb7339 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Wed, 26 Dec 2018 12:50:43 +0530 Subject: [PATCH 08/34] fixed kiosk by url --- .../newpipe/extractor/kiosk/KioskList.java | 2 +- .../PeertubeTrendingLinkHandlerFactory.java | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 22a9f2ca8..1896a6a7d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -103,7 +103,7 @@ public class KioskList { for(Map.Entry e : kioskList.entrySet()) { KioskEntry ke = e.getValue(); if(ke.handlerFactory.acceptUrl(url)) { - return getExtractorById(e.getKey(), nextPageUrl, localization); + return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization); } } throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index b3763ccb8..bda842985 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -13,6 +13,7 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { public static final Map KIOSK_MAP; + public static final Map REVERSE_KIOSK_MAP; public static final String KIOSK_TRENDING = "Trending"; public static final String KIOSK_RECENT = "Recently added"; public static final String KIOSK_LOCAL = "Local"; @@ -23,6 +24,12 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); map.put(KIOSK_LOCAL, "%s/api/v1/videos?filter=local"); KIOSK_MAP = Collections.unmodifiableMap(map); + + Map reverseMap = new HashMap<>(); + for(Map.Entry entry : KIOSK_MAP.entrySet()){ + reverseMap.put(entry.getValue(), entry.getKey()); + } + REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap); } public String getUrl(String id, List contentFilters, String sortFilter) { @@ -33,13 +40,15 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { - if(url.contains("/videos/trending")) { + if (url.contains("/videos/trending")) { return KIOSK_TRENDING; - }else if(url.contains("/videos/recently-added")) { + } else if (url.contains("/videos/recently-added")) { return KIOSK_RECENT; - }else if(url.contains("/videos/local")){ + } else if (url.contains("/videos/local")) { return KIOSK_LOCAL; - }else { + } else if (REVERSE_KIOSK_MAP.containsKey(url)) { + return REVERSE_KIOSK_MAP.get(url); + } else { throw new ParsingException("no id found for this url"); } } From 203c884d51a913105e51975db97ab517167a8dce Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Wed, 26 Dec 2018 13:02:04 +0530 Subject: [PATCH 09/34] strip base url --- .../linkHandler/PeertubeTrendingLinkHandlerFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index bda842985..f169086ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -39,7 +39,8 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { - + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + url = url.replace(baseUrl, ""); if (url.contains("/videos/trending")) { return KIOSK_TRENDING; } else if (url.contains("/videos/recently-added")) { From bc82a53f2ca0b4ddf94d6b8810ef72c1e494bd0c Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Wed, 26 Dec 2018 13:07:06 +0530 Subject: [PATCH 10/34] replace base url --- .../linkHandler/PeertubeTrendingLinkHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index f169086ad..33f8a3731 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -40,7 +40,7 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { String baseUrl = ServiceList.PeerTube.getBaseUrl(); - url = url.replace(baseUrl, ""); + url = url.replace(baseUrl, "%s"); if (url.contains("/videos/trending")) { return KIOSK_TRENDING; } else if (url.contains("/videos/recently-added")) { From b2c692845980cf71acee9c25e11adbb0d449a029 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Wed, 26 Dec 2018 14:29:23 +0530 Subject: [PATCH 11/34] added logic for fetching related streams --- .../extractors/PeertubeStreamExtractor.java | 39 ++++++++++++++++--- .../PeertubeSearchQueryHandlerFactory.java | 2 +- .../PeertubeTrendingLinkHandlerFactory.java | 6 +++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index af2c1eb54..2b6afbc2f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -17,7 +19,7 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -191,12 +193,36 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - //TODO fetch related videos not trending - String apiUrl = new PeertubeTrendingLinkHandlerFactory().getUrl(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); - getStreamsFromApi(collector, apiUrl); + List tags = getTags(); + String apiUrl = null; + if(!tags.isEmpty()) { + apiUrl = getRelatedStreamsUrl(tags); + + }else { + apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; + } + if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } + + private List getTags(){ + try { + return (List) JsonUtils.getArray(json, "tags"); + } catch (Exception e) { + return Collections.emptyList(); + } + } + + private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { + String url = ServiceList.PeerTube.getBaseUrl() + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; + StringBuilder params = new StringBuilder(); + params.append("start=0&count=8&sort=-createdAt"); + for(String tag : tags) { + params.append("&tagsOneOf="); + params.append(URLEncoder.encode(tag, "UTF-8")); + } + return url + "?" + params.toString(); + } private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { DownloadResponse response = getDownloader().get(apiUrl); @@ -226,7 +252,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); - collector.commit(extractor); + //do not add the same stream in related streams + if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 536d1bf42..19a5ba26f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -12,7 +12,7 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory public static final String CHARSET_UTF_8 = "UTF-8"; public static final String VIDEOS = "videos"; - private static final String SEARCH_ENDPOINT = "/api/v1/search/videos"; + public static final String SEARCH_ENDPOINT = "/api/v1/search/videos"; public static PeertubeSearchQueryHandlerFactory getInstance() { return new PeertubeSearchQueryHandlerFactory(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index 33f8a3731..c481a1992 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -12,6 +12,8 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + private static final PeertubeTrendingLinkHandlerFactory instance = new PeertubeTrendingLinkHandlerFactory(); + public static final Map KIOSK_MAP; public static final Map REVERSE_KIOSK_MAP; public static final String KIOSK_TRENDING = "Trending"; @@ -32,6 +34,10 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap); } + public static PeertubeTrendingLinkHandlerFactory getInstance() { + return instance; + } + public String getUrl(String id, List contentFilters, String sortFilter) { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return String.format(KIOSK_MAP.get(id), baseUrl); From a83342de0ffb87e40a9bcd0dca5066903691f141 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 29 Dec 2018 21:58:43 +0530 Subject: [PATCH 12/34] update peertube instance --- .../extractor/services/peertube/PeertubeInstance.java | 2 +- .../extractor/services/peertube/PeertubeService.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 2bedc07cf..967c387f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -26,7 +26,7 @@ public class PeertubeInstance { setInstanceMetaData(response); } - private PeertubeInstance(String url , String name) { + public PeertubeInstance(String url , String name) { this.url = url; this.name = name; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index efd86c758..2fc571ef3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -140,6 +140,15 @@ public class PeertubeService extends StreamingService { this.getServiceInfo().setName("PeerTube"); } } + + public void setInstance(String url, String name) throws IOException { + this.instance = new PeertubeInstance(url, name); + if(!StringUtil.isBlank(instance.getName())) { + this.getServiceInfo().setName(instance.getName()); + }else { + this.getServiceInfo().setName("PeerTube"); + } + } @Override public KioskList getKioskList() throws ExtractionException { From b1a77fa48451118bf495744690d5e67f49105b3d Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 29 Dec 2018 22:23:35 +0530 Subject: [PATCH 13/34] update peertube instance --- .../newpipe/extractor/services/peertube/PeertubeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 2fc571ef3..21c17ac8e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -141,7 +141,7 @@ public class PeertubeService extends StreamingService { } } - public void setInstance(String url, String name) throws IOException { + public void setInstance(String url, String name) { this.instance = new PeertubeInstance(url, name); if(!StringUtil.isBlank(instance.getName())) { this.getServiceInfo().setName(instance.getName()); From ef0ffb2229b3a9883ad40a949abeeefa497708ce Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sun, 30 Dec 2018 15:00:57 +0530 Subject: [PATCH 14/34] added torrent url in streams --- .../peertube/extractors/PeertubeStreamExtractor.java | 3 ++- .../org/schabi/newpipe/extractor/stream/Stream.java | 10 ++++++++-- .../schabi/newpipe/extractor/stream/VideoStream.java | 10 ++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 2b6afbc2f..7817a2ec7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -148,10 +148,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { if(!(s instanceof JsonObject)) continue; JsonObject stream = (JsonObject) s; String url = JsonUtils.getString(stream, "fileUrl"); + String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); String resolution = JsonUtils.getString(stream, "resolution.label"); String extension = url.substring(url.lastIndexOf(".") + 1); MediaFormat format = MediaFormat.getFromSuffix(extension); - VideoStream videoStream = new VideoStream(url, format, resolution); + VideoStream videoStream = new VideoStream(url, torrentUrl, format, resolution); if (!Stream.containSimilarStream(videoStream, videoStreams)) { videoStreams.add(videoStream); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 80208be63..754df790b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -1,13 +1,14 @@ package org.schabi.newpipe.extractor.stream; -import org.schabi.newpipe.extractor.MediaFormat; - import java.io.Serializable; import java.util.List; +import org.schabi.newpipe.extractor.MediaFormat; + public abstract class Stream implements Serializable { private final MediaFormat mediaFormat; public final String url; + public final String torrentUrl; /** * @deprecated Use {@link #getFormat()} or {@link #getFormatId()} @@ -16,7 +17,12 @@ public abstract class Stream implements Serializable { public final int format; public Stream(String url, MediaFormat format) { + this(url, null, format); + } + + public Stream(String url, String torrentUrl, MediaFormat format) { this.url = url; + this.torrentUrl = torrentUrl; this.format = format.id; this.mediaFormat = format; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java index 1e877bee3..f4531d365 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java @@ -36,6 +36,16 @@ public class VideoStream extends Stream { this.resolution = resolution; this.isVideoOnly = isVideoOnly; } + + public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) { + this(url, torrentUrl, format, resolution, false); + } + + public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution, boolean isVideoOnly) { + super(url, torrentUrl, format); + this.resolution = resolution; + this.isVideoOnly = isVideoOnly; + } @Override public boolean equalStats(Stream cmp) { From fade7bd749f6a86b9982f20b62a251e884344995 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sun, 30 Dec 2018 15:23:24 +0530 Subject: [PATCH 15/34] getter for torrent url --- .../main/java/org/schabi/newpipe/extractor/stream/Stream.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 754df790b..865cde0cc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -55,6 +55,10 @@ public abstract class Stream implements Serializable { public String getUrl() { return url; } + + public String getTorrentUrl() { + return torrentUrl; + } public MediaFormat getFormat() { return mediaFormat; From e66d188381e3de326ec00f69a7b839712d01888a Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 8 Mar 2019 04:54:39 +0530 Subject: [PATCH 16/34] throw content not available --- .../services/peertube/PeertubeParsingHelper.java | 11 +++++++++++ .../peertube/extractors/PeertubeChannelExtractor.java | 2 ++ 2 files changed, 13 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index b50cede26..68bc66bdf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,8 +4,12 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import com.grack.nanojson.JsonObject; + public class PeertubeParsingHelper { private PeertubeParsingHelper() { @@ -20,4 +24,11 @@ public class PeertubeParsingHelper { throw new ParsingException(e.getMessage(), e); } } + + public static void validate(JsonObject json) throws ContentNotAvailableException { + String error = json.getString("error"); + if(!StringUtil.isBlank(error)) { + throw new ContentNotAvailableException(error); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 620010254..0c9123b82 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -119,6 +120,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); if(json != null) { + PeertubeParsingHelper.validate(json); Number number = JsonUtils.getNumber(json, "total"); if(number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); From 36139558c623f6336a62d8ecae1d7d7c3c3bfa97 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 9 Mar 2019 23:33:51 +0530 Subject: [PATCH 17/34] handle stream 404 error, fixed tests --- .../services/peertube/extractors/PeertubeStreamExtractor.java | 1 + .../services/peertube/PeertubeChannelExtractorTest.java | 4 ++++ .../services/peertube/PeertubeStreamExtractorDefaultTest.java | 4 ++++ .../peertube/search/PeertubeSearchExtractorDefaultTest.java | 4 +++- 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 7817a2ec7..60cffe8c5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -283,6 +283,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { throw new ExtractionException("Unable to extract peertube stream data", e); } if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + PeertubeParsingHelper.validate(json); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 573a51455..b7c39ca28 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -31,6 +31,8 @@ public class PeertubeChannelExtractorTest { @BeforeClass public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + // setting instance might break test when running in parallel + PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); extractor = (PeertubeChannelExtractor) PeerTube .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org"); extractor.fetchPage(); @@ -116,6 +118,8 @@ public class PeertubeChannelExtractorTest { @BeforeClass public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + // setting instance might break test when running in parallel + PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); extractor = (PeertubeChannelExtractor) PeerTube .getChannelExtractor("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch"); extractor.fetchPage(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index bdd9d4ebf..33ab98a30 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -9,6 +9,7 @@ import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import java.io.IOException; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; @@ -29,6 +30,8 @@ public class PeertubeStreamExtractorDefaultTest { @BeforeClass public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + // setting instance might break test when running in parallel + PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); extractor.fetchPage(); } @@ -96,6 +99,7 @@ public class PeertubeStreamExtractorDefaultTest { assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM); } + @Ignore @Test public void testGetRelatedVideos() throws ExtractionException, IOException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java index 439688257..82f30c6ea 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -23,6 +23,8 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + // setting instance might break test when running in parallel + PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet's own boy"); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -30,7 +32,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("", extractor.getNextPageUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet%27s+own+boy&start=12&count=12", extractor.getNextPageUrl()); } @Test From c220700c94d89e967236c2ba3a8790265fa3c096 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sun, 10 Mar 2019 00:30:01 +0530 Subject: [PATCH 18/34] fixed tests --- .../services/media_ccc/MediaCCCSearchExtractorEventsTest.java | 2 +- .../services/peertube/PeertubeTrendingExtractorTest.java | 2 ++ .../peertube/PeertubeTrendingLinkHandlerFactoryTest.java | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java index 11b1fbdfb..81e8ed980 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java @@ -44,7 +44,7 @@ public class MediaCCCSearchExtractorEventsTest { @Test public void testServiceId() throws Exception { - assertEquals(2, extractor.getServiceId()); + assertEquals(3, extractor.getServiceId()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index e11bb5ee0..67b3f0261 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -28,6 +28,8 @@ public class PeertubeTrendingExtractorTest { @BeforeClass public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + // setting instance might break test when running in parallel + PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); extractor = PeerTube .getKioskList() .getExtractorById("Trending", null); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java index 6febed8d0..6bf1fbb3e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import org.junit.BeforeClass; import org.junit.Test; @@ -20,6 +21,8 @@ public class PeertubeTrendingLinkHandlerFactoryTest { @BeforeClass public static void setUp() throws Exception { + // setting instance might break test when running in parallel + PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory(); NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); } From 2193119fabb752af75f7f35a44332a3971e76003 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 23 Mar 2019 19:09:40 +0530 Subject: [PATCH 19/34] youtube's comment censorship breaks test --- .../youtube/YoutubeCommentsExtractorTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index a4bb6a61e..127eef7ab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -27,18 +27,18 @@ public class YoutubeCommentsExtractorTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = (YoutubeCommentsExtractor) YouTube - .getCommentsExtractor("https://www.youtube.com/watch?v=rrgFN3AxGfs"); + .getCommentsExtractor("https://www.youtube.com/watch?v=ehn8mJ8vvsI"); } @Test public void testGetComments() throws IOException, ExtractionException { boolean result = false; InfoItemsPage comments = extractor.getInitialPage(); - result = findInComments(comments, "i should really be in the top comment.lol"); + result = findInComments(comments, "Tsuki blyat"); while (comments.hasNextPage() && !result) { comments = extractor.getPage(comments.getNextPageUrl()); - result = findInComments(comments, "i should really be in the top comment.lol"); + result = findInComments(comments, "Tsuki blyat"); } assertTrue(result); @@ -47,14 +47,14 @@ public class YoutubeCommentsExtractorTest { @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; - CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=rrgFN3AxGfs"); - assertTrue("what the fuck am i doing with my life.wmv".equals(commentsInfo.getName())); - result = findInComments(commentsInfo.getRelatedItems(), "i should really be in the top comment.lol"); + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=ehn8mJ8vvsI"); + assertTrue("the dark side of YouTube...".equals(commentsInfo.getName())); + result = findInComments(commentsInfo.getRelatedItems(), "Tsuki blyat"); String nextPage = commentsInfo.getNextPageUrl(); while (!StringUtil.isBlank(nextPage) && !result) { InfoItemsPage moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); - result = findInComments(moreItems.getItems(), "i should really be in the top comment.lol"); + result = findInComments(moreItems.getItems(), "Tsuki blyat"); nextPage = moreItems.getNextPageUrl(); } From bd22b55885b3207f44f9ce17cb60293230880fbe Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 23 Mar 2019 19:10:34 +0530 Subject: [PATCH 20/34] fixed youtube channel name test --- .../youtube/YoutubeChannelExtractorTest.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index efa8cbbea..2fb6f2d99 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,5 +1,14 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; + import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; @@ -11,11 +20,6 @@ import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.utils.Localization; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.*; - /** * Test for {@link ChannelExtractor} */ @@ -323,7 +327,7 @@ public class YoutubeChannelExtractorTest { @Test public void testName() throws Exception { - assertEquals("CaptainDisillusion", extractor.getName()); + assertEquals("Captain Disillusion", extractor.getName()); } @Test From 07a8129a737e6090e4c6209604e5d80e29566284 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 23 Mar 2019 19:11:19 +0530 Subject: [PATCH 21/34] fixed peertube search test --- .../services/peertube/search/PeertubeSearchQHTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java index 89680d609..5f5a91f39 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -3,9 +3,16 @@ package org.schabi.newpipe.extractor.services.peertube.search; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import org.junit.BeforeClass; import org.junit.Test; public class PeertubeSearchQHTest { + + @BeforeClass + public static void setUpClass() throws Exception { + // setting instance might break test when running in parallel + PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + } @Test public void testRegularValues() throws Exception { @@ -15,4 +22,4 @@ public class PeertubeSearchQHTest { assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); } -} +} \ No newline at end of file From aabc65b57d1f91eeaa96e46e52d874a3c9faee1a Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 23 Mar 2019 19:11:51 +0530 Subject: [PATCH 22/34] added java doc for stream --- .../newpipe/extractor/stream/Stream.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 865cde0cc..173534474 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -16,10 +16,23 @@ public abstract class Stream implements Serializable { @Deprecated public final int format; + /** + * Instantiates a new stream object. + * + * @param url the url + * @param format the format + */ public Stream(String url, MediaFormat format) { this(url, null, format); } + /** + * Instantiates a new stream object. + * + * @param url the url + * @param torrentUrl the url to torrent file, example https://webtorrent.io/torrents/big-buck-bunny.torrent + * @param format the format + */ public Stream(String url, String torrentUrl, MediaFormat format) { this.url = url; this.torrentUrl = torrentUrl; @@ -52,18 +65,38 @@ public abstract class Stream implements Serializable { return false; } + /** + * Gets the url. + * + * @return the url + */ public String getUrl() { return url; } + /** + * Gets the torrent url. + * + * @return the torrent url, example https://webtorrent.io/torrents/big-buck-bunny.torrent + */ public String getTorrentUrl() { return torrentUrl; } + /** + * Gets the format. + * + * @return the format + */ public MediaFormat getFormat() { return mediaFormat; } + /** + * Gets the format id. + * + * @return the format id + */ public int getFormatId() { return mediaFormat.id; } From e4e6d44770dfff29b750457e19ceb405c8e42c80 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 23 Mar 2019 19:12:26 +0530 Subject: [PATCH 23/34] reordered services --- .../main/java/org/schabi/newpipe/extractor/ServiceList.java | 6 +++--- .../media_ccc/MediaCCCSearchExtractorEventsTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index a916e9933..6be1cea40 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -37,8 +37,8 @@ public final class ServiceList { public static final YoutubeService YouTube; public static final SoundcloudService SoundCloud; - public static final PeertubeService PeerTube; public static final MediaCCCService MediaCCC; + public static final PeertubeService PeerTube; /** * When creating a new service, put this service in the end of this list, @@ -48,8 +48,8 @@ public final class ServiceList { Arrays.asList( YouTube = new YoutubeService(0), SoundCloud = new SoundcloudService(1), - PeerTube = new PeertubeService(2), - MediaCCC = new MediaCCCService(3) + MediaCCC = new MediaCCCService(2), + PeerTube = new PeertubeService(3) )); /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java index 81e8ed980..11b1fbdfb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java @@ -44,7 +44,7 @@ public class MediaCCCSearchExtractorEventsTest { @Test public void testServiceId() throws Exception { - assertEquals(3, extractor.getServiceId()); + assertEquals(2, extractor.getServiceId()); } @Test From f60c9739289619b579d77995c9010821b38c79f4 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 23 Mar 2019 19:32:06 +0530 Subject: [PATCH 24/34] more javadoc --- .../main/java/org/schabi/newpipe/extractor/stream/Stream.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 173534474..1d49e2535 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -5,6 +5,9 @@ import java.util.List; import org.schabi.newpipe.extractor.MediaFormat; +/** + * Creates a stream object from url, format and optional torrent url + */ public abstract class Stream implements Serializable { private final MediaFormat mediaFormat; public final String url; From 193442d01ce2fb5b90d2d154b85b5d3edd64a4d7 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 16 Nov 2019 02:42:39 +0530 Subject: [PATCH 25/34] add original url for share --- .../peertube/extractors/PeertubeChannelExtractor.java | 5 +++++ .../peertube/extractors/PeertubeStreamExtractor.java | 5 +++++ .../services/peertube/PeertubeChannelExtractorTest.java | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 0c9123b82..018c167aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -179,5 +179,10 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public String getName() throws ParsingException { return JsonUtils.getString(json, "displayName"); } + + @Override + public String getOriginalUrl() throws ParsingException { + return ServiceList.PeerTube.getBaseUrl() + "/accounts/" + getId(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 60cffe8c5..e0380c495 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -291,4 +291,9 @@ public class PeertubeStreamExtractor extends StreamExtractor { return JsonUtils.getString(json, "name"); } + @Override + public String getOriginalUrl() throws ParsingException { + return ServiceList.PeerTube.getBaseUrl() + "/videos/watch/" + getId(); + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index f62771d69..222dab310 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -64,7 +64,7 @@ public class PeertubeChannelExtractorTest { @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// From bb5ad49face622b301739ccba9161a86d5adb32f Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 16 Nov 2019 03:20:35 +0530 Subject: [PATCH 26/34] fix html comments --- .../extractors/PeertubeCommentsInfoItemExtractor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 1e6df7708..cfe3e6aa8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -5,6 +5,8 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Locale; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -59,7 +61,12 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac @Override public String getCommentText() throws ParsingException { String htmlText = JsonUtils.getString(item, "text"); - return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); + try { + Document doc = Jsoup.parse(htmlText); + return doc.body().text(); + }catch(Exception e) { + return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); + } } @Override From 318f600527a6328c9baf8adc1887116e1028e17a Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 16 Nov 2019 04:30:13 +0530 Subject: [PATCH 27/34] add subtitles support for peertube --- .../extractors/PeertubeStreamExtractor.java | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index e0380c495..2ffd76743 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -40,6 +40,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { private JsonObject json; + private List subtitles = new ArrayList<>(); public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) { super(service, linkHandler, localization); @@ -173,12 +174,18 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public List getSubtitlesDefault() throws IOException, ExtractionException { - return Collections.emptyList(); + return subtitles; } @Override public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { - return Collections.emptyList(); + List filteredSubs = new ArrayList<>(); + for(SubtitlesStream sub: subtitles) { + if(sub.getFormat() == format) { + filteredSubs.add(sub); + } + } + return filteredSubs; } @Override @@ -274,6 +281,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { }else { throw new ExtractionException("Unable to extract peertube channel data"); } + + loadSubtitles(); } private void setInitialData(String responseBody) throws ExtractionException { @@ -285,6 +294,28 @@ public class PeertubeStreamExtractor extends StreamExtractor { if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); PeertubeParsingHelper.validate(json); } + + private void loadSubtitles() { + if (subtitles.isEmpty()) { + try { + DownloadResponse response = getDownloader().get(getUrl() + "/captions"); + JsonObject captionsJson = JsonParser.object().from(response.getResponseBody()); + JsonArray captions = JsonUtils.getArray(captionsJson, "data"); + for(Object c: captions) { + if(c instanceof JsonObject) { + JsonObject caption = (JsonObject)c; + String url = ServiceList.PeerTube.getBaseUrl() + JsonUtils.getString(caption, "captionPath"); + String languageCode = JsonUtils.getString(caption, "language.id"); + String ext = url.substring(url.lastIndexOf(".") + 1); + MediaFormat fmt = MediaFormat.getFromSuffix(ext); + if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); + } + } + } catch (Exception e) { + // ignore all exceptions + } + } + } @Override public String getName() throws ParsingException { From 8a7aa6b9a9802755335d040d05fb2a980e3b17b9 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Thu, 21 Nov 2019 05:00:09 +0530 Subject: [PATCH 28/34] added baseUrl param in linkhandlers. Required for multi instance services --- .../linkhandler/LinkHandlerFactory.java | 21 +++++++++- .../linkhandler/ListLinkHandlerFactory.java | 39 ++++++++++++++++--- .../extractors/PeertubeChannelExtractor.java | 4 +- .../PeertubeCommentsInfoItemExtractor.java | 4 +- .../extractors/PeertubeSearchExtractor.java | 4 +- .../extractors/PeertubeStreamExtractor.java | 8 ++-- .../PeertubeStreamInfoItemExtractor.java | 8 ++-- .../extractors/PeertubeTrendingExtractor.java | 4 +- .../PeertubeChannelLinkHandlerFactory.java | 8 +++- .../PeertubeCommentsLinkHandlerFactory.java | 15 +++---- .../PeertubePlaylistLinkHandlerFactory.java | 7 +++- .../PeertubeSearchQueryHandlerFactory.java | 5 +++ .../PeertubeStreamLinkHandlerFactory.java | 5 +++ .../PeertubeTrendingLinkHandlerFactory.java | 8 +++- .../schabi/newpipe/extractor/utils/Utils.java | 15 ++++++- 15 files changed, 125 insertions(+), 30 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index 60a65f5ef..b9e60ce0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.linkhandler; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Utils; /* * Created by Christian Schabesberger on 26.07.16. @@ -33,26 +34,41 @@ public abstract class LinkHandlerFactory { public abstract String getUrl(String id) throws ParsingException; public abstract boolean onAcceptUrl(final String url) throws ParsingException; + public String getUrl(String id, String baseUrl) throws ParsingException{ + return getUrl(id); + } + /////////////////////////////////// // Logic /////////////////////////////////// public LinkHandler fromUrl(String url) throws ParsingException { + final String baseUrl = Utils.getBaseUrl(url); + return fromUrl(url, baseUrl); + } + + public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if(url == null) throw new IllegalArgumentException("url can not be null"); if(!acceptUrl(url)) { throw new ParsingException("Malformed unacceptable url: " + url); } final String id = getId(url); - return new LinkHandler(url, getUrl(id), id); + return new LinkHandler(url, getUrl(id,baseUrl), id); } - + public LinkHandler fromId(String id) throws ParsingException { if(id == null) throw new IllegalArgumentException("id can not be null"); final String url = getUrl(id); return new LinkHandler(url, url, id); } + public LinkHandler fromId(String id, String baseUrl) throws ParsingException { + if(id == null) throw new IllegalArgumentException("id can not be null"); + final String url = getUrl(id, baseUrl); + return new LinkHandler(url, url, id); + } + /** * When a VIEW_ACTION is caught this function will test if the url delivered within the calling * Intent was meant to be watched with this Service. @@ -65,4 +81,5 @@ public abstract class LinkHandlerFactory { throw fe; } } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index a9c4e51aa..83955265c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -1,10 +1,11 @@ package org.schabi.newpipe.extractor.linkhandler; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import java.util.ArrayList; import java.util.List; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Utils; + public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /////////////////////////////////// @@ -14,23 +15,37 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { public List getContentFilter(String url) throws ParsingException { return new ArrayList<>(0);} public String getSortFilter(String url) throws ParsingException {return ""; } public abstract String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException; + + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { + return getUrl(id, contentFilter, sortFilter); + } /////////////////////////////////// // Logic /////////////////////////////////// - - + @Override public ListLinkHandler fromUrl(String url) throws ParsingException { + String baseUrl = Utils.getBaseUrl(url); + return fromUrl(url, baseUrl); + } + + @Override + public ListLinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if(url == null) throw new IllegalArgumentException("url may not be null"); - return new ListLinkHandler(super.fromUrl(url), getContentFilter(url), getSortFilter(url)); + return new ListLinkHandler(super.fromUrl(url, baseUrl), getContentFilter(url), getSortFilter(url)); } @Override public ListLinkHandler fromId(String id) throws ParsingException { return new ListLinkHandler(super.fromId(id), new ArrayList(0), ""); } + + @Override + public ListLinkHandler fromId(String id, String baseUrl) throws ParsingException { + return new ListLinkHandler(super.fromId(id, baseUrl), new ArrayList(0), ""); + } public ListLinkHandler fromQuery(String id, List contentFilters, @@ -38,8 +53,15 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { final String url = getUrl(id, contentFilters, sortFilter); return new ListLinkHandler(url, url, id, contentFilters, sortFilter); } + + public ListLinkHandler fromQuery(String id, + List contentFilters, + String sortFilter, String baseUrl) throws ParsingException { + final String url = getUrl(id, contentFilters, sortFilter, baseUrl); + return new ListLinkHandler(url, url, id, contentFilters, sortFilter); + } - + /** * For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, * however it should not be overridden by the actual implementation. @@ -50,6 +72,11 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { return getUrl(id, new ArrayList(0), ""); } + @Override + public String getUrl(String id, String baseUrl) throws ParsingException { + return getUrl(id, new ArrayList(0), "", baseUrl); + } + /** * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index ae6591faf..70965f5b4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -89,10 +90,11 @@ public class PeertubeChannelExtractor extends ChannelExtractor { throw new ParsingException("unable to extract channel streams", e); } + String baseUrl = Utils.getBaseUrl(getUrl()); for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; - PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index f97836e44..13a8a6311 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonObject; @@ -97,7 +98,8 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac public String getAuthorEndpoint() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + String baseUrl = Utils.getBaseUrl(url); + return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index dc6cd43b1..ec8786deb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -58,10 +59,11 @@ public class PeertubeSearchExtractor extends SearchExtractor { throw new ParsingException("unable to extract search info", e); } + String baseUrl = Utils.getBaseUrl(getUrl()); for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; - PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 692210b59..bfd8cd93e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -19,7 +19,6 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Stream; @@ -30,6 +29,7 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -115,7 +115,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(json, "account.name"); String host = JsonUtils.getString(json, "account.host"); - return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + String baseUrl = Utils.getBaseUrl(getUrl()); + return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } @Override @@ -266,10 +267,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { throw new ParsingException("unable to extract related videos", e); } + String baseUrl = Utils.getBaseUrl(getUrl()); for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; - PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); //do not add the same stream in related streams if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index 49c19be07..3de68edaf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -15,15 +15,17 @@ import com.grack.nanojson.JsonObject; public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { protected final JsonObject item; + private final String baseUrl; - public PeertubeStreamInfoItemExtractor(JsonObject item) { + public PeertubeStreamInfoItemExtractor(JsonObject item, String baseUrl) { this.item = item; + this.baseUrl = baseUrl; } @Override public String getUrl() throws ParsingException { String uuid = JsonUtils.getString(item, "uuid"); - return PeertubeStreamLinkHandlerFactory.getInstance().fromId(uuid).getUrl(); + return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl(); } @Override @@ -52,7 +54,7 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index e481d89bc..df0fb2437 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -53,10 +54,11 @@ public class PeertubeTrendingExtractor extends KioskExtractor { throw new ParsingException("unable to extract kiosk info", e); } + String baseUrl = Utils.getBaseUrl(getUrl()); for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; - PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index 2d93b9f58..640f99e59 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -23,8 +23,14 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { } @Override - public String getUrl(String id, List contentFilters, String searchFilter) { + public String getUrl(String id, List contentFilters, String searchFilter) throws ParsingException { String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, contentFilters, searchFilter, baseUrl); + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) + throws ParsingException { return baseUrl + ACCOUNTS_ENDPOINT + id; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java index dcd0c3256..9faa3492b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java @@ -18,12 +18,6 @@ public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { return instance; } - @Override - public String getUrl(String id) { - String baseUrl = ServiceList.PeerTube.getBaseUrl(); - return baseUrl + String.format(COMMENTS_ENDPOINT, id); - } - @Override public String getId(String url) throws ParsingException, IllegalArgumentException { return Parser.matchGroup(ID_PATTERN, url, 2); @@ -36,6 +30,13 @@ public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { - return getUrl(id); + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, contentFilter, sortFilter, baseUrl); } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { + return baseUrl + String.format(COMMENTS_ENDPOINT, id); + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java index fa6413589..64e291a54 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java @@ -21,9 +21,14 @@ public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilters, String sortFilter) { String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, contentFilters, sortFilter, baseUrl); + } + + @Override + public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { return baseUrl + VIDEO_CHANNELS_ENDPOINT + id; } - + @Override public String getId(String url) throws ParsingException { return Parser.matchGroup1(ID_PATTERN, url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 19a5ba26f..695911670 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -21,6 +21,11 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(searchString, contentFilters, sortFilter, baseUrl); + } + + @Override + public String getUrl(String searchString, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { try { final String url = baseUrl + SEARCH_ENDPOINT + "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java index 880c4d937..3ee7744cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -22,6 +22,11 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getUrl(String id) { String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, baseUrl); + } + + @Override + public String getUrl(String id, String baseUrl) { return baseUrl + VIDEO_ENDPOINT + id; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index c481a1992..906af5830 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -37,9 +37,15 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { public static PeertubeTrendingLinkHandlerFactory getInstance() { return instance; } - + + @Override public String getUrl(String id, List contentFilters, String sortFilter) { String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, contentFilters, sortFilter, baseUrl); + } + + @Override + public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { return String.format(KIOSK_MAP.get(id), baseUrl); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 1f8da13b2..fd06d40f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -144,9 +144,9 @@ public class Utils { try { return new URL(url); } catch (MalformedURLException e) { - // if no protocol is given try prepending "http://" + // if no protocol is given try prepending "https://" if (e.getMessage().equals("no protocol: " + url)) { - return new URL(HTTP + url); + return new URL(HTTPS + url); } throw e; @@ -175,4 +175,15 @@ public class Utils { } return s; } + + public static String getBaseUrl(String url) throws ParsingException { + URL uri; + try { + uri = stringToURL(url); + } catch (MalformedURLException e) { + throw new ParsingException("Malformed url: " + url, e); + } + return uri.getProtocol() + "://" + uri.getAuthority(); + } + } \ No newline at end of file From 4e0adbefbcfbeb25cc0d1306c07b5f9dc7ed5c90 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Thu, 21 Nov 2019 05:35:22 +0530 Subject: [PATCH 29/34] [peertube] use baseUrl instead of global instance url --- .../extractors/PeertubeChannelExtractor.java | 10 +++++----- .../PeertubeCommentsInfoItemExtractor.java | 14 +++++++------- .../extractors/PeertubeStreamExtractor.java | 16 ++++++++-------- .../PeertubeStreamInfoItemExtractor.java | 4 +--- .../peertube/PeertubeCommentsExtractorTest.java | 2 +- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 70965f5b4..b179ae13f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import java.io.IOException; import org.jsoup.helper.StringUtil; -import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -35,9 +34,11 @@ public class PeertubeChannelExtractor extends ChannelExtractor { private long total; private JsonObject json; + private final String baseUrl; - public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { + public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { super(service, linkHandler); + this.baseUrl = Utils.getBaseUrl(getUrl()); } @Override @@ -48,7 +49,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { }catch(Exception e) { value = "/client/assets/images/default-avatar.png"; } - return ServiceList.PeerTube.getBaseUrl() + value; + return baseUrl + value; } @Override @@ -90,7 +91,6 @@ public class PeertubeChannelExtractor extends ChannelExtractor { throw new ParsingException("unable to extract channel streams", e); } - String baseUrl = Utils.getBaseUrl(getUrl()); for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; @@ -183,7 +183,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getOriginalUrl() throws ParsingException { - return ServiceList.PeerTube.getBaseUrl() + "/accounts/" + getId(); + return baseUrl + "/accounts/" + getId(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 13a8a6311..886dcda8f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; @@ -18,10 +17,12 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac private final JsonObject item; private final String url; + private final String baseUrl; - public PeertubeCommentsInfoItemExtractor(JsonObject item, String url) { + public PeertubeCommentsInfoItemExtractor(JsonObject item, String url) throws ParsingException { this.item = item; this.url = url; + this.baseUrl = Utils.getBaseUrl(url); } @Override @@ -37,7 +38,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac }catch(Exception e) { value = "/client/assets/images/default-avatar.png"; } - return ServiceList.PeerTube.getBaseUrl() + value; + return baseUrl + value; } @Override @@ -58,7 +59,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac @Override public int getLikeCount() throws ParsingException { - return 0; + return -1; } @Override @@ -86,7 +87,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac }catch(Exception e) { value = "/client/assets/images/default-avatar.png"; } - return ServiceList.PeerTube.getBaseUrl() + value; + return baseUrl + value; } @Override @@ -98,8 +99,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac public String getAuthorEndpoint() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - String baseUrl = Utils.getBaseUrl(url); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } -} \ No newline at end of file +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index bfd8cd93e..3ed260c5e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -41,9 +41,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { private JsonObject json; private List subtitles = new ArrayList<>(); + private final String baseUrl; - public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { super(service, linkHandler); + this.baseUrl = Utils.getBaseUrl(getUrl()); } @Override @@ -64,7 +66,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getThumbnailUrl() throws ParsingException { - return ServiceList.PeerTube.getBaseUrl() + JsonUtils.getString(json, "thumbnailPath"); + return baseUrl + JsonUtils.getString(json, "thumbnailPath"); } @Override @@ -115,7 +117,6 @@ public class PeertubeStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(json, "account.name"); String host = JsonUtils.getString(json, "account.host"); - String baseUrl = Utils.getBaseUrl(getUrl()); return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } @@ -132,7 +133,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { }catch(Exception e) { value = "/client/assets/images/default-avatar.png"; } - return ServiceList.PeerTube.getBaseUrl() + value; + return baseUrl + value; } @Override @@ -233,7 +234,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { - String url = ServiceList.PeerTube.getBaseUrl() + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; + String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); params.append("start=0&count=8&sort=-createdAt"); for(String tag : tags) { @@ -267,7 +268,6 @@ public class PeertubeStreamExtractor extends StreamExtractor { throw new ParsingException("unable to extract related videos", e); } - String baseUrl = Utils.getBaseUrl(getUrl()); for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; @@ -316,7 +316,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { for(Object c: captions) { if(c instanceof JsonObject) { JsonObject caption = (JsonObject)c; - String url = ServiceList.PeerTube.getBaseUrl() + JsonUtils.getString(caption, "captionPath"); + String url = baseUrl + JsonUtils.getString(caption, "captionPath"); String languageCode = JsonUtils.getString(caption, "language.id"); String ext = url.substring(url.lastIndexOf(".") + 1); MediaFormat fmt = MediaFormat.getFromSuffix(ext); @@ -336,7 +336,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getOriginalUrl() throws ParsingException { - return ServiceList.PeerTube.getBaseUrl() + "/videos/watch/" + getId(); + return baseUrl + "/videos/watch/" + getId(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index 3de68edaf..ca85270a9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -4,8 +4,6 @@ import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -31,7 +29,7 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor @Override public String getThumbnailUrl() throws ParsingException { String value = JsonUtils.getString(item, "thumbnailPath"); - return ServiceList.PeerTube.getBaseUrl() + value; + return baseUrl + value; } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index 319a8ae41..fef1c3ee5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -73,7 +73,7 @@ public class PeertubeCommentsExtractorTest { assertFalse(StringUtil.isBlank(c.getTextualPublishedTime())); assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); assertFalse(StringUtil.isBlank(c.getUrl())); - assertFalse(c.getLikeCount() == -1); + assertFalse(c.getLikeCount() != -1); } } From 279f175693fdbca0e545b365bfd4532c16e0208c Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Fri, 22 Nov 2019 22:59:14 +0530 Subject: [PATCH 30/34] validate peertube instance --- .../services/peertube/PeertubeInstance.java | 31 ++++++------------- .../services/peertube/PeertubeService.java | 24 ++------------ .../PeertubeChannelExtractorTest.java | 4 +-- .../PeertubeStreamExtractorDefaultTest.java | 2 +- .../PeertubeTrendingExtractorTest.java | 2 +- ...eertubeTrendingLinkHandlerFactoryTest.java | 2 +- .../PeertubeSearchExtractorDefaultTest.java | 3 +- .../peertube/search/PeertubeSearchQHTest.java | 3 +- 8 files changed, 21 insertions(+), 50 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 132262faf..992b8eb6d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -20,10 +20,9 @@ public class PeertubeInstance { private String name; public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); - public PeertubeInstance(String url) throws IOException { + public PeertubeInstance(String url) { this.url = url; - String response = validateInstance(url); - setInstanceMetaData(response); + this.name = "PeerTube"; } public PeertubeInstance(String url , String name) { @@ -35,37 +34,25 @@ public class PeertubeInstance { return url; } - private String validateInstance(String url) throws IOException { + public void fetchInstanceMetaData() throws Exception { Downloader downloader = NewPipe.getDownloader(); Response response = null; try { response = downloader.get(url + "/api/v1/config"); } catch (ReCaptchaException | IOException e) { - throw new IOException("unable to configure instance " + url, e); + throw new Exception("unable to configure instance " + url, e); } if(null == response || StringUtil.isBlank(response.responseBody())) { - throw new IOException("unable to configure instance " + url); + throw new Exception("unable to configure instance " + url); } - return response.responseBody(); - } - - private void setInstanceMetaData(String responseBody) { - JsonObject json; - try { - json = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { - return; - } - - if(null == json) return; - - try { + try { + JsonObject json = JsonParser.object().from(response.responseBody()); this.name = JsonUtils.getString(json, "instance.name"); - } catch (ParsingException e) { - return; + } catch (JsonParserException | ParsingException e) { + throw new Exception("unable to parse instance config", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 52fa5bf85..3c42cd396 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -7,7 +7,6 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap import java.io.IOException; import java.util.List; -import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -51,10 +50,6 @@ public class PeertubeService extends StreamingService { this.instance = instance; } - public PeertubeService(int id, String name, List capabilities) { - super(id, name, capabilities); - } - @Override public LinkHandlerFactory getStreamLHFactory() { return PeertubeStreamLinkHandlerFactory.getInstance(); @@ -127,24 +122,11 @@ public class PeertubeService extends StreamingService { return instance.getUrl(); } - public void setInstance(String url) throws IOException { - this.instance = new PeertubeInstance(url); - if(!StringUtil.isBlank(instance.getName())) { - this.getServiceInfo().setName(instance.getName()); - }else { - this.getServiceInfo().setName("PeerTube"); - } + public void setInstance(PeertubeInstance instance) throws IOException { + this.instance = instance; + this.getServiceInfo().setName(instance.getName()); } - public void setInstance(String url, String name) { - this.instance = new PeertubeInstance(url, name); - if(!StringUtil.isBlank(instance.getName())) { - this.getServiceInfo().setName(instance.getName()); - }else { - this.getServiceInfo().setName("PeerTube"); - } - } - @Override public KioskList getKioskList() throws ExtractionException { KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 348474957..3d8bdff33 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -31,7 +31,7 @@ public class PeertubeChannelExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); extractor.fetchPage(); @@ -118,7 +118,7 @@ public class PeertubeChannelExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); extractor.fetchPage(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index a6303b793..75a692d42 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -34,7 +34,7 @@ public class PeertubeStreamExtractorDefaultTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); extractor.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index f271d109f..c693e25be 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -28,7 +28,7 @@ public class PeertubeTrendingExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = PeerTube .getKioskList() .getExtractorById("Trending", null); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java index 0cc6fa013..0b89aa1aa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -21,7 +21,7 @@ public class PeertubeTrendingLinkHandlerFactoryTest { @BeforeClass public static void setUp() throws Exception { // setting instance might break test when running in parallel - PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory(); NewPipe.init(DownloaderTestImpl.getInstance()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java index 5608faf28..dd9c6deb3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -23,7 +24,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde"); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java index 5f5a91f39..23e3100ce 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -5,13 +5,14 @@ import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import org.junit.BeforeClass; import org.junit.Test; +import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; public class PeertubeSearchQHTest { @BeforeClass public static void setUpClass() throws Exception { // setting instance might break test when running in parallel - PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); } @Test From bc75c6603b9d6777916f89af81515a0a7e590c61 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sat, 23 Nov 2019 00:05:49 +0530 Subject: [PATCH 31/34] added getBaseUrl method to linkhandler --- .../org/schabi/newpipe/extractor/Extractor.java | 15 ++++++++++----- .../extractor/linkhandler/LinkHandler.java | 7 +++++++ .../services/peertube/PeertubeService.java | 2 +- .../extractors/PeertubeChannelExtractor.java | 3 +-- .../extractors/PeertubeCommentsExtractor.java | 2 +- .../PeertubeCommentsInfoItemExtractor.java | 7 +++---- .../extractors/PeertubeSearchExtractor.java | 3 +-- .../extractors/PeertubeStreamExtractor.java | 2 +- .../extractors/PeertubeTrendingExtractor.java | 3 +-- 9 files changed, 26 insertions(+), 18 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index e0fe4912c..e5c03d3f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -1,5 +1,10 @@ package org.schabi.newpipe.extractor; +import java.io.IOException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -8,11 +13,6 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.io.IOException; -import java.io.Serializable; - public abstract class Extractor{ /** * {@link StreamingService} currently related to this extractor.
@@ -93,6 +93,11 @@ public abstract class Extractor{ public String getUrl() throws ParsingException { return linkHandler.getUrl(); } + + @Nonnull + public String getBaseUrl() throws ParsingException { + return linkHandler.getBaseUrl(); + } @Nonnull public StreamingService getService() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java index c28bc5c83..525105134 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java @@ -2,6 +2,9 @@ package org.schabi.newpipe.extractor.linkhandler; import java.io.Serializable; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Utils; + public class LinkHandler implements Serializable { protected final String originalUrl; protected final String url; @@ -28,4 +31,8 @@ public class LinkHandler implements Serializable { public String getId() { return id; } + + public String getBaseUrl() throws ParsingException { + return Utils.getBaseUrl(url); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 3c42cd396..9d247abc2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -122,7 +122,7 @@ public class PeertubeService extends StreamingService { return instance.getUrl(); } - public void setInstance(PeertubeInstance instance) throws IOException { + public void setInstance(PeertubeInstance instance) { this.instance = instance; this.getServiceInfo().setName(instance.getName()); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index b179ae13f..8b214670c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -16,7 +16,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -38,7 +37,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { super(service, linkHandler); - this.baseUrl = Utils.getBaseUrl(getUrl()); + this.baseUrl = getBaseUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index f49c76736..b7aba4fac 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -56,7 +56,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; - PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, pageUrl); + PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this); collector.commit(extractor); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 886dcda8f..edb59e1fe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonObject; @@ -19,10 +18,10 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac private final String url; private final String baseUrl; - public PeertubeCommentsInfoItemExtractor(JsonObject item, String url) throws ParsingException { + public PeertubeCommentsInfoItemExtractor(JsonObject item, PeertubeCommentsExtractor extractor) throws ParsingException { this.item = item; - this.url = url; - this.baseUrl = Utils.getBaseUrl(url); + this.url = extractor.getUrl(); + this.baseUrl = extractor.getBaseUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index ec8786deb..ff021f732 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -17,7 +17,6 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -59,7 +58,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { throw new ParsingException("unable to extract search info", e); } - String baseUrl = Utils.getBaseUrl(getUrl()); + String baseUrl = getBaseUrl(); for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 3ed260c5e..d8d0de005 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -45,7 +45,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { super(service, linkHandler); - this.baseUrl = Utils.getBaseUrl(getUrl()); + this.baseUrl = getBaseUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index df0fb2437..0ff340f29 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -15,7 +15,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -54,7 +53,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { throw new ParsingException("unable to extract kiosk info", e); } - String baseUrl = Utils.getBaseUrl(getUrl()); + String baseUrl = getBaseUrl(); for(Object c: contents) { if(c instanceof JsonObject) { final JsonObject item = (JsonObject) c; From 281ccea39f9469dd78372a42d58a3b974de55a47 Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sun, 24 Nov 2019 02:35:41 +0530 Subject: [PATCH 32/34] [peertube] constant name for service --- .../java/org/schabi/newpipe/extractor/StreamingService.java | 6 +----- .../extractor/services/peertube/PeertubeService.java | 3 +-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index edc132542..ad22642db 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -48,7 +48,7 @@ public abstract class StreamingService { * This class holds meta information about the service implementation. */ public static class ServiceInfo { - private String name; + private final String name; private final List mediaCapabilities; @@ -66,10 +66,6 @@ public abstract class StreamingService { return name; } - public void setName(String name) { - this.name = name; - } - public List getMediaCapabilities() { return mediaCapabilities; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 9d247abc2..29bd1db71 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -46,7 +46,7 @@ public class PeertubeService extends StreamingService { } public PeertubeService(int id, PeertubeInstance instance) { - super(id, instance.getName(), asList(VIDEO, COMMENTS)); + super(id, "PeerTube", asList(VIDEO, COMMENTS)); this.instance = instance; } @@ -124,7 +124,6 @@ public class PeertubeService extends StreamingService { public void setInstance(PeertubeInstance instance) { this.instance = instance; - this.getServiceInfo().setName(instance.getName()); } @Override From 6a7680ce17e0ca9751c23c4d69347a75a13b0ecc Mon Sep 17 00:00:00 2001 From: yausername <13ritvik@gmail.com> Date: Sun, 24 Nov 2019 03:38:27 +0530 Subject: [PATCH 33/34] added instance getter --- .../extractor/services/peertube/PeertubeService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 29bd1db71..a3faac4ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -4,11 +4,7 @@ import static java.util.Arrays.asList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; -import java.io.IOException; -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -122,6 +118,10 @@ public class PeertubeService extends StreamingService { return instance.getUrl(); } + public PeertubeInstance getInstance() { + return this.instance; + } + public void setInstance(PeertubeInstance instance) { this.instance = instance; } From 51155412687057751897ba744bbe9734198b45b7 Mon Sep 17 00:00:00 2001 From: yausername Date: Tue, 3 Dec 2019 00:42:06 +0530 Subject: [PATCH 34/34] fix peertube trending, added most-liked kiosk --- .../extractor/services/peertube/PeertubeService.java | 1 + .../peertube/extractors/PeertubeTrendingExtractor.java | 2 +- .../PeertubeTrendingLinkHandlerFactory.java | 10 +++++++--- .../peertube/PeertubeTrendingExtractorTest.java | 2 +- .../PeertubeTrendingLinkHandlerFactoryTest.java | 9 +++++++-- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index a3faac4ef..47271732f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -145,6 +145,7 @@ public class PeertubeService extends StreamingService { final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory(); try { list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_MOST_LIKED); list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT); list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL); list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index 0ff340f29..1cb420460 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -20,7 +20,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -public class PeertubeTrendingExtractor extends KioskExtractor { +public class PeertubeTrendingExtractor extends KioskExtractor { private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index 906af5830..c0773c202 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -17,14 +17,16 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { public static final Map KIOSK_MAP; public static final Map REVERSE_KIOSK_MAP; public static final String KIOSK_TRENDING = "Trending"; + public static final String KIOSK_MOST_LIKED = "Most liked"; public static final String KIOSK_RECENT = "Recently added"; public static final String KIOSK_LOCAL = "Local"; static { Map map = new HashMap<>(); - map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-views"); + map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending"); + map.put(KIOSK_MOST_LIKED, "%s/api/v1/videos?sort=-likes"); map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); - map.put(KIOSK_LOCAL, "%s/api/v1/videos?filter=local"); + map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local"); KIOSK_MAP = Collections.unmodifiableMap(map); Map reverseMap = new HashMap<>(); @@ -55,6 +57,8 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { url = url.replace(baseUrl, "%s"); if (url.contains("/videos/trending")) { return KIOSK_TRENDING; + } else if (url.contains("/videos/most-liked")) { + return KIOSK_MOST_LIKED; } else if (url.contains("/videos/recently-added")) { return KIOSK_RECENT; } else if (url.contains("/videos/local")) { @@ -68,6 +72,6 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { - return url.contains("/videos?") || url.contains("/videos/trending") || url.contains("/videos/recently-added") || url.contains("/videos/local"); + return url.contains("/videos?") || url.contains("/videos/trending") || url.contains("/videos/most-liked") || url.contains("/videos/recently-added") || url.contains("/videos/local"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index c693e25be..bb4e1529b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -92,6 +92,6 @@ public class PeertubeTrendingExtractorTest { @Test public void testGetCleanUrl() throws Exception { - assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-views"); + assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java index 0b89aa1aa..d192e8be3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -29,15 +29,17 @@ public class PeertubeTrendingLinkHandlerFactoryTest { @Test public void getUrl() throws Exception { - assertEquals(LinkHandlerFactory.fromId("Trending").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-views"); + assertEquals(LinkHandlerFactory.fromId("Trending").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending"); + assertEquals(LinkHandlerFactory.fromId("Most liked").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-likes"); assertEquals(LinkHandlerFactory.fromId("Recently added").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt"); - assertEquals(LinkHandlerFactory.fromId("Local").getUrl(), "https://peertube.mastodon.host/api/v1/videos?filter=local"); + assertEquals(LinkHandlerFactory.fromId("Local").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt&filter=local"); } @Test public void getId() throws Exception { assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/trending").getId(), "Trending"); + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/most-liked").getId(), "Most liked"); assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/recently-added").getId(), "Recently added"); assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/local").getId(), "Local"); } @@ -47,6 +49,9 @@ public class PeertubeTrendingLinkHandlerFactoryTest { assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending?adsf=fjaj#fhe")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked?adsf=fjaj#fhe")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added?adsf=fjaj#fhe"));