From a579337c9a4d317f2c8f247201a57de11c9e196e Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sat, 21 Dec 2019 19:00:07 +0100 Subject: [PATCH 01/91] Bandcamp service with support for streams and searches --- extractor/build.gradle | 1 + .../schabi/newpipe/extractor/ServiceList.java | 5 +- .../services/bandcamp/BandcampService.java | 102 ++++++++ .../extractors/BandcampExtractorHelper.java | 83 +++++++ .../extractors/BandcampSearchExtractor.java | 123 ++++++++++ .../extractors/BandcampStreamExtractor.java | 224 ++++++++++++++++++ .../BandcampStreamInfoItemExtractor.java | 87 +++++++ .../BandcampChannelLinkHandlerFactory.java | 59 +++++ .../BandcampSearchQueryHandlerFactory.java | 30 +++ .../BandcampStreamLinkHandlerFactory.java | 47 ++++ ...BandcampChannelLinkHandlerFactoryTest.java | 40 ++++ .../bandcamp/BandcampSearchExtractorTest.java | 55 +++++ ...BandcampSearchQueryHandlerFactoryTest.java | 34 +++ .../bandcamp/BandcampStreamExtractorTest.java | 64 +++++ .../BandcampStreamLinkHandlerFactoryTest.java | 47 ++++ 15 files changed, 1000 insertions(+), 1 deletion(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java diff --git a/extractor/build.gradle b/extractor/build.gradle index 1b7fbf001..cc2f69dab 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -6,6 +6,7 @@ dependencies { implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' implementation 'org.nibor.autolink:autolink:0.8.0' + implementation 'org.json:json:20190722' testImplementation 'junit:junit:4.12' } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 6be1cea40..742d342df 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.schabi.newpipe.extractor.services.bandcamp.BandcampService; import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; @@ -39,6 +40,7 @@ public final class ServiceList { public static final SoundcloudService SoundCloud; public static final MediaCCCService MediaCCC; public static final PeertubeService PeerTube; + public static final BandcampService bandcamp; /** * When creating a new service, put this service in the end of this list, @@ -49,7 +51,8 @@ public final class ServiceList { YouTube = new YoutubeService(0), SoundCloud = new SoundcloudService(1), MediaCCC = new MediaCCCService(2), - PeerTube = new PeertubeService(3) + PeerTube = new PeertubeService(3), + bandcamp = new BandcampService(4) )); /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java new file mode 100644 index 000000000..fd891365d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -0,0 +1,102 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +import java.util.Collections; + +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; + +public class BandcampService extends StreamingService { + + public BandcampService(int id) { + super(id, "bandcamp", Collections.singletonList(AUDIO)); + } + + @Override + public String getBaseUrl() { + return "https://bandcamp.com"; + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return new BandcampStreamLinkHandlerFactory(); + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + //return new BandcampChannelLinkHandlerFactory(); TODO + return null; + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + return null; + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return new BandcampSearchQueryHandlerFactory(); + } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return null; + } + + @Override + public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { + return new BandcampSearchExtractor(this, queryHandler); + } + + @Override + public SuggestionExtractor getSuggestionExtractor() { + return null; + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + return null; + } + + @Override + public KioskList getKioskList() throws ExtractionException { + return null; + } + + @Override + public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { + return null; + } + + @Override + public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException { + return null; + } + + @Override + public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException { + return new BandcampStreamExtractor(this, linkHandler); + } + + @Override + public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java new file mode 100644 index 000000000..452c46934 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -0,0 +1,83 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONException; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.util.ArrayList; +import java.util.Arrays; + +public class BandcampExtractorHelper { + + /** + * Get JSON behind var $variable = out of web page + *
+ * Originally a part of bandcampDirect. + * + * @param html The HTML where the JSON we're looking for is stored inside a + * variable inside some JavaScript block + * @param variable Name of the variable + * @return The JsonObject stored in the variable with this name + */ + public static JSONObject getJSONFromJavaScriptVariables(String html, String variable) throws JSONException, ParsingException { + + String[] part = html.split("var " + variable + " = "); + + String firstHalfGone = part[1]; + + firstHalfGone = firstHalfGone.replaceAll("\" \\+ \"", ""); + + int position = -1; + int level = 0; + for (char character : firstHalfGone.toCharArray()) { + position++; + + switch (character) { + case '{': + level++; + continue; + case '}': + level--; + if (level == 0) { + return new JSONObject(firstHalfGone.substring(0, position + 1) + .replaceAll(" {4}//.+", "") // Remove comments in JSON + ); + } + } + } + + throw new ParsingException("Unexpected HTML: JSON never ends"); + } + + /** + * Concatenate all non-null and non-empty strings together while separating them using + * the comma parameter + */ + public static String smartConcatenate(String[] strings, String comma) { + StringBuilder result = new StringBuilder(); + + // Remove empty strings + ArrayList list = new ArrayList<>(Arrays.asList(strings)); + for (int i = list.size() - 1; i >= 0; i--) { + if (list.get(i) == null || list.get(i).isEmpty()) { + list.remove(i); + } + } + + // Append remaining strings to result + for (int i = 0; i < list.size(); i++) { + String string = list.get(i); + result.append(string); + + if (i != list.size() - 1) { + // This is not the last iteration yet + result.append(comma); + } + + } + + return String.valueOf(result); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java new file mode 100644 index 000000000..1b49ff70a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -0,0 +1,123 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class BandcampSearchExtractor extends SearchExtractor { + + public BandcampSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public String getSearchSuggestion() { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + // okay apparently this is where we DOWNLOAD the page and then COMMIT its ENTRIES to an INFOITEMPAGE + String html = getDownloader().get(pageUrl).responseBody(); + + InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + + + Document d = Jsoup.parse(html); + + Elements searchResultsElements = d.getElementsByClass("searchresult"); + + for (Element searchResult : + searchResultsElements) { + + Element resultInfo = searchResult.getElementsByClass("result-info").first(); + + String type = resultInfo + .getElementsByClass("itemtype").first().text(); + + String image = null; + Element img = searchResult.getElementsByClass("art").first() + .getElementsByTag("img").first(); + if (img != null) { + image = img.attr("src"); + } + + String heading = resultInfo.getElementsByClass("heading").text(); + + String subhead = resultInfo.getElementsByClass("subhead").text(); + + String url = resultInfo.getElementsByClass("itemurl").text(); + + switch (type) { + default: + continue; + case "FAN": + //collector.commit Channel (?) with heading, url, image + break; + + case "ARTIST": + String id = resultInfo.getElementsByClass("itemurl").first() + .getElementsByTag("a").first() + .attr("href") // the link contains the id + .split("search_item_id=") + [1] // the number is behind its name + .split("&") // there is another attribute behind the name + [0]; // get the number + + //searchResults.add(new Artist(heading, Long.parseLong(id), image, subhead)); + //collector.commit Channel with heading, id, image, subhead + break; + + case "ALBUM": + String artist = subhead.split(" by")[0]; + //searchResults.add(new Album(heading, artist, url, image)); + //collector.commit Playlist with heading, artist, url, image + break; + + case "TRACK": + String album = subhead.split("from ")[0].split(" by")[0]; + + String[] splitBy = subhead.split(" by"); + String artist1 = null; + if (splitBy.length > 1) { + artist1 = subhead.split(" by")[1]; + } + collector.commit(new BandcampStreamInfoItemExtractor(heading, url, image, artist1, album)); + break; + } + + } + + + return new InfoItemsPage<>(getInfoItemSearchCollector(), null); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return getPage(getUrl());//new InfoItemsPage<>(getInfoItemSearchCollector(), null); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java new file mode 100644 index 000000000..0bb100c09 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -0,0 +1,224 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.*; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.List; + +public class BandcampStreamExtractor extends StreamExtractor { + + private JSONObject albumJson; + private JSONObject current; + private Document document; + + public BandcampStreamExtractor(StreamingService service, LinkHandler linkHandler) { + super(service, linkHandler); + } + + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + String html = downloader.get(getLinkHandler().getUrl()).responseBody(); + document = Jsoup.parse(html); + albumJson = getAlbumInfoJson(html); + current = albumJson.getJSONObject("current"); + + if (albumJson.getJSONArray("trackinfo").length() > 1) { + // In this case, we are actually viewing an album page! + throw new ExtractionException("Page is actually an album, not a track"); + } + } + + /** + * Get the JSON that contains album's metadata from page + * + * @param html Website + * @return Album metadata JSON + * @throws ParsingException In case of a faulty website + */ + public static JSONObject getAlbumInfoJson(String html) throws ParsingException { + try { + return BandcampExtractorHelper.getJSONFromJavaScriptVariables(html, "TralbumData"); + } catch (JSONException e) { + throw new ParsingException("Faulty JSON", e); + } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return current.getString("title"); + } + + @Nonnull + @Override + public String getUploaderUrl() throws ParsingException { + String[] parts = getUrl().split("/"); + // https: (/) (/) * .bandcamp.com (/) and leave out the rest + return "https://" + parts[2] + "/"; + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return albumJson.getString("url").replace("http://", "https://"); + } + + @Nonnull + @Override + public String getUploaderName() throws ParsingException { + return albumJson.getString("artist"); + } + + @Nullable + @Override + public String getTextualUploadDate() throws ParsingException { + return current.getString("release_date"); + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Nonnull + @Override + public String getThumbnailUrl() throws ParsingException { + return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); + } + + @Nonnull + @Override + public String getUploaderAvatarUrl() { + return document.getElementsByClass("band-photo").first().attr("src"); + } + + @Nonnull + @Override + public String getDescription() { + return BandcampExtractorHelper.smartConcatenate( + new String[]{ + getStringOrNull(current, "about"), + getStringOrNull(current, "lyrics"), + getStringOrNull(current, "credits") + }, "\n\n" + ); + } + + /** + * Avoid exceptions like "JSONObject["about"] not a string." and instead just return null. + * This is for the case that the actual JSON has something like "about": null. + */ + private String getStringOrNull(JSONObject jsonObject, String value) { + try { + return jsonObject.getString(value); + } catch (JSONException e) { + return null; + } + } + + @Override + public int getAgeLimit() throws ParsingException { + return 0; + } + + @Override + public long getLength() throws ParsingException { + return 0; + } + + @Override + public long getTimeStamp() throws ParsingException { + return 0; + } + + @Override + public long getViewCount() throws ParsingException { + return -1; + } + + @Override + public long getLikeCount() throws ParsingException { + return -1; + } + + @Override + public long getDislikeCount() throws ParsingException { + return -1; + } + + @Nonnull + @Override + public String getDashMpdUrl() throws ParsingException { + return null; + } + + @Nonnull + @Override + public String getHlsUrl() throws ParsingException { + return null; + } + + @Override + public List getAudioStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public List getVideoStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public List getVideoOnlyStreams() throws IOException, ExtractionException { + return null; + } + + @Nonnull + @Override + public List getSubtitlesDefault() throws IOException, ExtractionException { + return null; + } + + @Nonnull + @Override + public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { + return null; + } + + @Override + public StreamType getStreamType() throws ParsingException { + return null; + } + + @Override + public StreamInfoItem getNextStream() throws IOException, ExtractionException { + return null; + } + + @Override + public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public String getErrorMessage() { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java new file mode 100644 index 000000000..d1d965797 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java @@ -0,0 +1,87 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; + +public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor { + + private String title; + private String url; + private String cover; + private String artist; + private String albumName; + + public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist, String albumName) { + this.title = title; + this.url = url; + this.cover = cover; + this.artist = artist; + this.albumName = albumName; + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.AUDIO_STREAM; + } + + @Override + public long getDuration() throws ParsingException { + return -1; + } + + @Override + public long getViewCount() throws ParsingException { + return -1; + } + + @Override + public String getUploaderName() throws ParsingException { + return artist; + } + + @Override + public String getUploaderUrl() throws ParsingException { + return null; + } + + @Nullable + @Override + public String getTextualUploadDate() throws ParsingException { + return null; // TODO + } + + @Nullable + @Override + public DateWrapper getUploadDate() throws ParsingException { + return null; + } + + @Override + public String getName() throws ParsingException { + return title; + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return cover; + } + + /** + * There are no ads just like that, duh + */ + @Override + public boolean isAd() throws ParsingException { + return false; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java new file mode 100644 index 000000000..24f19f761 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -0,0 +1,59 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; + +import java.io.IOException; +import java.util.List; + +/** + * Artist do have IDs that are useful + */ +public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { + + + @Override + public String getId(String url) throws ParsingException { + try { + String response = NewPipe.getDownloader().get(url).responseBody(); + return BandcampStreamExtractor.getAlbumInfoJson(response) + .getString("band_id"); + } catch (IOException | ReCaptchaException e) { + throw new ParsingException("Download failed", e); + } + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + return null; // TODO + } + + /** + * Matches * .bandcamp.com as well as custom domains + * where the profile is at * . * /releases + */ + @Override + public boolean onAcceptUrl(String url) throws ParsingException { + + // Ends with "bandcamp.com" or "bandcamp.com/"? + boolean endsWithBandcampCom = url.endsWith("bandcamp.com") + || url.endsWith("bandcamp.com/"); + + // Is a subdomain of bandcamp.com? + boolean isBandcampComSubdomain = url.matches("https?://.+\\.bandcamp\\.com"); + + // Is root of bandcamp.com subdomain? + boolean isBandcampComArtistPage = endsWithBandcampCom && isBandcampComSubdomain; + + boolean isCustomDomainReleases = url.matches("https?://.+\\..+/releases/?(?!.)"); + + return isBandcampComArtistPage || isCustomDomainReleases; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java new file mode 100644 index 000000000..3f23956fe --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java @@ -0,0 +1,30 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + + private static final String SEARCH_URL = "https://bandcamp.com/search?q="; + + public static final String CHARSET_UTF_8 = "UTF-8"; + + + @Override + public String getUrl(String query, List contentFilter, String sortFilter) throws ParsingException { + try { + + return SEARCH_URL + + URLEncoder.encode(query, CHARSET_UTF_8); + + } catch (UnsupportedEncodingException e) { + throw new ParsingException("query \"" + query + "\" could not be encoded", e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java new file mode 100644 index 000000000..e534f925f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -0,0 +1,47 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; + +/** + * Tracks do have IDs, but they are not really useful. That's why id = url. + * Instead, URLs are cleaned up so that they always look the same. + */ +public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { + + + /** + * @see BandcampStreamLinkHandlerFactory + */ + @Override + public String getId(String url) throws ParsingException { + return getUrl(url); + } + + /** + * Clean up url + * @see BandcampStreamLinkHandlerFactory + */ + @Override + public String getUrl(String url) { + if (url.endsWith("/")) + url = url.substring(0, url.length() - 1); + url = url.replace("http://", "https://").toLowerCase(); + return url; + } + + /** + * Sometimes, the root page of an artist is also an album or track + * page. In that case, it is assumed that one actually wants to open + * the profile and not the track it has set as the default one. + *

Urls are expected to be in this format to account for + * custom domains: + *
https:// * . * /track/ * + */ + @Override + public boolean onAcceptUrl(String url) { + return getUrl(url).matches("https?://.+\\..+/track/.+"); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java new file mode 100644 index 000000000..d8c7497d7 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -0,0 +1,40 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; + +/** + * Test for {@link BandcampChannelLinkHandlerFactory} + */ +public class BandcampChannelLinkHandlerFactoryTest { + private static BandcampChannelLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = new BandcampChannelLinkHandlerFactory(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testAcceptUrl() throws ParsingException { + // Tests expecting true + assertTrue(linkHandler.acceptUrl("http://interovgm.com/releases/")); + assertTrue(linkHandler.acceptUrl("https://interovgm.com/releases")); + assertTrue(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); + + // Tests expecting false + assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); + assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); + assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java new file mode 100644 index 000000000..11ce3d0d3 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -0,0 +1,55 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +/** + * Test for {@link BandcampSearchExtractor} + */ +public class BandcampSearchExtractorTest { + + private static BandcampSearchExtractor extractor; + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + + } + + /** + * Tests whether searching bandcamp for "best friend's basement" returns + * the accordingly named song by Zach Benson + */ + @Test + public void testBestFriendsBasement() throws ExtractionException, IOException { + extractor = (BandcampSearchExtractor) bandcamp + .getSearchExtractor("best friend's basement"); + + ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + InfoItem bestFriendsBasement = page.getItems().get(0); + + // The track by Zach Benson should be the first result, no? + assertEquals("Best Friend's Basement", bestFriendsBasement.getName()); + assertTrue(bestFriendsBasement.getThumbnailUrl().endsWith(".jpg")); + assertTrue(bestFriendsBasement.getThumbnailUrl().contains("f4.bcbits.com/img/")); + assertEquals(InfoItem.InfoType.STREAM, bestFriendsBasement.getInfoType()); + + + + + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java new file mode 100644 index 000000000..1a8fdac5b --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java @@ -0,0 +1,34 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +public class BandcampSearchQueryHandlerFactoryTest { + + static BandcampSearchQueryHandlerFactory searchQuery; + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + + searchQuery = (BandcampSearchQueryHandlerFactory) bandcamp + .getSearchQHFactory(); + } + + @Test + public void testEncoding() throws ParsingException { + // Note: this isn't exactly as bandcamp does it (it wouldn't encode '!'), but both works + assertEquals("https://bandcamp.com/search?q=hello%21%22%C2%A7%24%25%26%2F%28%29%3D", searchQuery.getUrl("hello!\"ยง$%&/()=")); + // Note: bandcamp uses %20 instead of '+', but both works + assertEquals("https://bandcamp.com/search?q=search+query+with+spaces", searchQuery.getUrl("search query with spaces")); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java new file mode 100644 index 000000000..f64e5b555 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -0,0 +1,64 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +public class BandcampStreamExtractorTest { + + private static BandcampStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (BandcampStreamExtractor) bandcamp + .getStreamExtractor("https://zachbenson.bandcamp.com/track/kitchen"); + extractor.fetchPage(); + } + + @Test(expected = ExtractionException.class) + public void testAlbum() throws ExtractionException { + bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/album/prom"); + } + + @Test + public void testServiceId() { + } + + @Test + public void testName() throws ParsingException { + assertEquals("kitchen", extractor.getName()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://zachbenson.bandcamp.com/track/kitchen", extractor.getUrl()); + } + + @Test + public void testArtistUrl() throws ParsingException { + assertEquals("https://zachbenson.bandcamp.com/", extractor.getUploaderUrl()); + } + + @Test + public void testDescription() { + assertEquals(831, extractor.getDescription().length()); + } + + @Test + public void testArtistProfilePicture() { + String url = extractor.getUploaderAvatarUrl(); + assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java new file mode 100644 index 000000000..0eb760776 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -0,0 +1,47 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; + +/** + * Test for {@link BandcampStreamLinkHandlerFactory} + */ +public class BandcampStreamLinkHandlerFactoryTest { + + private static BandcampStreamLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = new BandcampStreamLinkHandlerFactory(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testUrlCleanup() { + assertEquals("https://zachbenson.bandcamp.com/track/u-i-tonite", linkHandler.getUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); + } + + @Test + public void testAcceptUrl() throws ParsingException { + // Tests expecting false + assertFalse(linkHandler.acceptUrl("http://interovgm.com/releases/")); + assertFalse(linkHandler.acceptUrl("https://interovgm.com/releases")); + assertFalse(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); + assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); + assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); + + // Tests expecting true + assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); + assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); + assertTrue(linkHandler.acceptUrl("https://interovgm.com/track/title")); + } +} From 43dc3c3d4c44de432ed0adb1483f284bd1055449 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sat, 21 Dec 2019 19:20:06 +0100 Subject: [PATCH 02/91] Return empty kiosk list instead of null --- .../newpipe/extractor/services/bandcamp/BandcampService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index fd891365d..7cf03009f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -77,7 +77,7 @@ public class BandcampService extends StreamingService { @Override public KioskList getKioskList() throws ExtractionException { - return null; + return new KioskList(this); } @Override From 80d67e22b7185596968e907b942b1f0f5acbc502 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sat, 21 Dec 2019 19:31:46 +0100 Subject: [PATCH 03/91] Return mp3-128 audio stream [experimental] --- .../bandcamp/extractors/BandcampStreamExtractor.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 0bb100c09..8ab4f8ce7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.stream.*; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; import java.util.List; public class BandcampStreamExtractor extends StreamExtractor { @@ -177,7 +178,14 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public List getAudioStreams() throws IOException, ExtractionException { - return null; + List audioStreams = new ArrayList<>(); + + audioStreams.add(new AudioStream( + albumJson.getJSONArray("trackinfo").getJSONObject(0) + .getJSONArray("file").getJSONObject(0).getString("mp3-128"), + MediaFormat.MP3, 128 + )); + return audioStreams; } @Override @@ -204,7 +212,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public StreamType getStreamType() throws ParsingException { - return null; + return StreamType.AUDIO_STREAM; } @Override From c3d127ccd934c2cd6e8baedd9bce08e200aabe2e Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sat, 21 Dec 2019 21:47:06 +0100 Subject: [PATCH 04/91] Bandcamp audio stream test and fix --- .../bandcamp/extractors/BandcampStreamExtractor.java | 12 ++++++------ .../bandcamp/BandcampStreamExtractorTest.java | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 8ab4f8ce7..0c872c92f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -177,36 +177,36 @@ public class BandcampStreamExtractor extends StreamExtractor { } @Override - public List getAudioStreams() throws IOException, ExtractionException { + public List getAudioStreams() { List audioStreams = new ArrayList<>(); audioStreams.add(new AudioStream( albumJson.getJSONArray("trackinfo").getJSONObject(0) - .getJSONArray("file").getJSONObject(0).getString("mp3-128"), + .getJSONObject("file").getString("mp3-128"), MediaFormat.MP3, 128 )); return audioStreams; } @Override - public List getVideoStreams() throws IOException, ExtractionException { + public List getVideoStreams() { return null; } @Override - public List getVideoOnlyStreams() throws IOException, ExtractionException { + public List getVideoOnlyStreams() { return null; } @Nonnull @Override - public List getSubtitlesDefault() throws IOException, ExtractionException { + public List getSubtitlesDefault() { return null; } @Nonnull @Override - public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { + public List getSubtitles(MediaFormat format) { return null; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index f64e5b555..bcd925fcf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -61,4 +61,10 @@ public class BandcampStreamExtractorTest { assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); } + @Test + public void testAudioStream() { + assertTrue(extractor.getAudioStreams().get(0).getUrl().contains("bcbits.com/stream")); + assertEquals(1, extractor.getAudioStreams().size()); + } + } From 5281456899db32ffff9362202a72bad430d1f70c Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sat, 21 Dec 2019 23:14:23 +0100 Subject: [PATCH 05/91] Bandcamp channel link handler factory --- .../services/bandcamp/BandcampService.java | 3 +- .../extractors/BandcampExtractorHelper.java | 3 +- .../extractors/BandcampStreamExtractor.java | 8 ++-- .../BandcampChannelLinkHandlerFactory.java | 46 +++++++++++++------ ...BandcampChannelLinkHandlerFactoryTest.java | 16 ++++++- 5 files changed, 55 insertions(+), 21 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 7cf03009f..86e1f7dad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -41,8 +41,7 @@ public class BandcampService extends StreamingService { @Override public ListLinkHandlerFactory getChannelLHFactory() { - //return new BandcampChannelLinkHandlerFactory(); TODO - return null; + return new BandcampChannelLinkHandlerFactory(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 452c46934..98ad88923 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -42,7 +42,8 @@ public class BandcampExtractorHelper { level--; if (level == 0) { return new JSONObject(firstHalfGone.substring(0, position + 1) - .replaceAll(" {4}//.+", "") // Remove comments in JSON + .replaceAll(" {4}//.+", "") // Remove "for the curious" in JSON + .replaceAll("// xxx: note - don't internationalize this variable", "") // Remove this comment ); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 0c872c92f..d9253b600 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -56,7 +56,7 @@ public class BandcampStreamExtractor extends StreamExtractor { try { return BandcampExtractorHelper.getJSONFromJavaScriptVariables(html, "TralbumData"); } catch (JSONException e) { - throw new ParsingException("Faulty JSON", e); + throw new ParsingException("Faulty JSON; page likely does not contain album data", e); } } @@ -211,17 +211,17 @@ public class BandcampStreamExtractor extends StreamExtractor { } @Override - public StreamType getStreamType() throws ParsingException { + public StreamType getStreamType() { return StreamType.AUDIO_STREAM; } @Override - public StreamInfoItem getNextStream() throws IOException, ExtractionException { + public StreamInfoItem getNextStream() { return null; } @Override - public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { + public StreamInfoItemsCollector getRelatedStreams() { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index 24f19f761..e4521ffb0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -2,13 +2,15 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; +import org.json.JSONObject; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; import java.util.List; @@ -23,16 +25,41 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { public String getId(String url) throws ParsingException { try { String response = NewPipe.getDownloader().get(url).responseBody(); - return BandcampStreamExtractor.getAlbumInfoJson(response) - .getString("band_id"); + + // This variable contains band data! + JSONObject bandData = BandcampExtractorHelper.getJSONFromJavaScriptVariables(response, "BandData"); + + return String.valueOf(bandData.getLong("id")); + } catch (IOException | ReCaptchaException e) { throw new ParsingException("Download failed", e); } } + /** + * Fetch artist details from mobile endpoint, thereby receiving their URL. + * + * I once took a moment to note down how it works. + * + * @throws ParsingException + */ @Override public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { - return null; // TODO + try { + String data = NewPipe.getDownloader().post( + "https://bandcamp.com/api/mobile/22/band_details", + null, + ("{\"band_id\":\"" + id + "\"}").getBytes() + ).responseBody(); + + return new JSONObject(data) + .getString("bandcamp_url") + .replace("http://", "https://"); + + + } catch (IOException | ReCaptchaException e) { + throw new ParsingException("Download failed", e); + } } /** @@ -40,17 +67,10 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { * where the profile is at * . * /releases */ @Override - public boolean onAcceptUrl(String url) throws ParsingException { - - // Ends with "bandcamp.com" or "bandcamp.com/"? - boolean endsWithBandcampCom = url.endsWith("bandcamp.com") - || url.endsWith("bandcamp.com/"); + public boolean onAcceptUrl(String url) { // Is a subdomain of bandcamp.com? - boolean isBandcampComSubdomain = url.matches("https?://.+\\.bandcamp\\.com"); - - // Is root of bandcamp.com subdomain? - boolean isBandcampComArtistPage = endsWithBandcampCom && isBandcampComSubdomain; + boolean isBandcampComArtistPage = url.matches("https?://.+\\.bandcamp\\.com/?"); boolean isCustomDomainReleases = url.matches("https?://.+\\..+/releases/?(?!.)"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index d8c7497d7..7e46a096b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -30,11 +30,25 @@ public class BandcampChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("http://interovgm.com/releases/")); assertTrue(linkHandler.acceptUrl("https://interovgm.com/releases")); assertTrue(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); + assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); // Tests expecting false assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); - assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); + } + + @Test + public void testGetId() throws ParsingException { + assertEquals("1196681540", linkHandler.getId("https://macbenson.bandcamp.com/")); + assertEquals("1581461772", linkHandler.getId("https://interovgm.com/releases")); + assertEquals("3321800855", linkHandler.getId("https://infiniteammo.bandcamp.com/")); + } + + @Test + public void testGetUrl() throws ParsingException { + assertEquals("https://macbenson.bandcamp.com", linkHandler.getUrl("1196681540")); + assertEquals("https://interovgm.com", linkHandler.getUrl("1581461772")); + assertEquals("https://infiniteammo.bandcamp.com", linkHandler.getUrl("3321800855")); } } From a42c77425dfc5165a75546a1b9822789d3623de2 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 00:42:26 +0100 Subject: [PATCH 06/91] Bandcamp channel extractor (ignoring everything but tracks) --- .../services/bandcamp/BandcampService.java | 3 +- .../extractors/BandcampChannelExtractor.java | 155 ++++++++++++++++++ .../extractors/BandcampExtractorHelper.java | 25 ++- .../extractors/BandcampSearchExtractor.java | 4 +- .../extractors/BandcampStreamExtractor.java | 2 + .../BandcampStreamInfoItemExtractor.java | 4 +- .../BandcampChannelLinkHandlerFactory.java | 28 +--- .../BandcampStreamLinkHandlerFactory.java | 4 +- .../BandcampChannelExtractorTest.java | 42 +++++ ...BandcampChannelLinkHandlerFactoryTest.java | 10 ++ .../bandcamp/BandcampStreamExtractorTest.java | 5 + 11 files changed, 252 insertions(+), 30 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 86e1f7dad..4752e1f09 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; @@ -81,7 +82,7 @@ public class BandcampService extends StreamingService { @Override public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return null; + return new BandcampChannelExtractor(this, linkHandler); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java new file mode 100644 index 000000000..bb39ca1d0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -0,0 +1,155 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class BandcampChannelExtractor extends ChannelExtractor { + + private JSONObject channelInfo; + + public BandcampChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { + super(service, linkHandler); + + channelInfo = getArtistDetails(getId()); + } + + /** + * Fetch artist details from mobile endpoint. + * + * I once took a moment to note down how it works. + */ + public static JSONObject getArtistDetails(String id) throws ParsingException { + try { + return + new JSONObject( + NewPipe.getDownloader().post( + "https://bandcamp.com/api/mobile/22/band_details", + null, + ("{\"band_id\":\"" + id + "\"}").getBytes() + ).responseBody() + ); + } catch (IOException | ReCaptchaException e) { + throw new ParsingException("Could not download band details", e); + } + } + + /** + * @param id The image ID + * @param album Whether this is the cover of an album + * @return Url of image with this ID in size 10 which is 1200x1200 (we could also choose size 0 + * but we don't want something as large as 3460x3460 here, do we?) + */ + public static String getImageUrl(long id, boolean album) { + return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; + } + + @Override + public String getAvatarUrl() { + return getImageUrl(channelInfo.getLong("bio_image_id"), false); + } + + /** + * Why does the mobile endpoint not contain the header?? Or at least not the same one? + * Anyway we're back to querying websites + */ + @Override + public String getBannerUrl() throws ParsingException { + try { + String html = getDownloader().get(channelInfo.getString("bandcamp_url")) + .responseBody(); + + return new Document(html).getElementById("customHeader") + .getElementsByTag("img") + .attr("src"); + + } catch (IOException | ReCaptchaException e) { + throw new ParsingException("Could not download artist web site", e); + } + } + + /** + * I had to learn bandcamp stopped providing RSS feeds when appending /feed to any URL + * because too few people used it. Bummer! + */ + @Override + public String getFeedUrl() { + return null; + } + + @Override + public long getSubscriberCount() { + return -1; + } + + @Override + public String getDescription() { + return channelInfo.getString("bio"); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ParsingException { + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + JSONArray discography = channelInfo.getJSONArray("discography"); + + for (int i = 0; i < discography.length(); i++) { + // I define discograph as an item that can appear in a discography + JSONObject discograph = discography.getJSONObject(i); + + if (!discograph.getString("item_type").equals("track")) continue; + + collector.commit(new BandcampStreamInfoItemExtractor( + discograph.getString("title"), + BandcampExtractorHelper.getStreamUrlFromIds( + discograph.getLong("band_id"), + discograph.getLong("item_id"), + discograph.getString("item_type") + ), + getImageUrl( + discograph.getLong("art_id"), true + ), + discograph.getString("band_name") + )); + } + + return new InfoItemsPage<>(collector, null); + } + + @Override + public String getNextPageUrl() { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + } + + @Nonnull + @Override + public String getName() { + return channelInfo.getString("name"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 98ad88923..b1668641c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -4,8 +4,11 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.json.JSONException; import org.json.JSONObject; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -21,7 +24,7 @@ public class BandcampExtractorHelper { * @param variable Name of the variable * @return The JsonObject stored in the variable with this name */ - public static JSONObject getJSONFromJavaScriptVariables(String html, String variable) throws JSONException, ParsingException { + public static JSONObject getJSONFromJavaScriptVariables(String html, String variable) throws JSONException, ArrayIndexOutOfBoundsException, ParsingException { String[] part = html.split("var " + variable + " = "); @@ -52,6 +55,26 @@ public class BandcampExtractorHelper { throw new ParsingException("Unexpected HTML: JSON never ends"); } + /** + * Translate all these parameters together to the URL of the corresponding album or track + * using the mobile api + */ + public static String getStreamUrlFromIds(long bandId, long itemId, String itemType) throws ParsingException { + + try { + String html = NewPipe.getDownloader().get( + "https://bandcamp.com/api/mobile/22/tralbum_details?band_id=" + bandId + + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.substring(0, 1)) + .responseBody(); + + return new JSONObject(html).getString("bandcamp_url").replace("http://", "https://"); + + } catch (JSONException | ReCaptchaException | IOException e) { + throw new ParsingException("Ids could not be translated to URL", e); + } + + } + /** * Concatenate all non-null and non-empty strings together while separating them using * the comma parameter diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index 1b49ff70a..34f52af1e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -88,14 +88,12 @@ public class BandcampSearchExtractor extends SearchExtractor { break; case "TRACK": - String album = subhead.split("from ")[0].split(" by")[0]; - String[] splitBy = subhead.split(" by"); String artist1 = null; if (splitBy.length > 1) { artist1 = subhead.split(" by")[1]; } - collector.commit(new BandcampStreamInfoItemExtractor(heading, url, image, artist1, album)); + collector.commit(new BandcampStreamInfoItemExtractor(heading, url, image, artist1)); break; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index d9253b600..a8fa30004 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -57,6 +57,8 @@ public class BandcampStreamExtractor extends StreamExtractor { return BandcampExtractorHelper.getJSONFromJavaScriptVariables(html, "TralbumData"); } catch (JSONException e) { throw new ParsingException("Faulty JSON; page likely does not contain album data", e); + } catch (ArrayIndexOutOfBoundsException e) { + throw new ParsingException("JSON does not exist", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java index d1d965797..6cb96411e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java @@ -15,14 +15,12 @@ public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor private String url; private String cover; private String artist; - private String albumName; - public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist, String albumName) { + public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist) { this.title = title; this.url = url; this.cover = cover; this.artist = artist; - this.albumName = albumName; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index e4521ffb0..0d57eac01 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -2,15 +2,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; +import org.json.JSONException; import org.json.JSONObject; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; -import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; import java.util.List; @@ -31,35 +30,24 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { return String.valueOf(bandData.getLong("id")); - } catch (IOException | ReCaptchaException e) { + } catch (IOException | ReCaptchaException | ArrayIndexOutOfBoundsException e) { throw new ParsingException("Download failed", e); } } /** - * Fetch artist details from mobile endpoint, thereby receiving their URL. - * - * I once took a moment to note down how it works. - * - * @throws ParsingException + * Uses the mobile endpoint as a "translator" from id to url */ @Override public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { try { - String data = NewPipe.getDownloader().post( - "https://bandcamp.com/api/mobile/22/band_details", - null, - ("{\"band_id\":\"" + id + "\"}").getBytes() - ).responseBody(); - - return new JSONObject(data) + return BandcampChannelExtractor.getArtistDetails(id) .getString("bandcamp_url") .replace("http://", "https://"); - - - } catch (IOException | ReCaptchaException e) { - throw new ParsingException("Download failed", e); + } catch (JSONException e) { + throw new ParsingException("JSON does not contain URL (invalid id?) or is otherwise invalid", e); } + } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java index e534f925f..0536594e1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -6,8 +6,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; /** - * Tracks do have IDs, but they are not really useful. That's why id = url. - * Instead, URLs are cleaned up so that they always look the same. + * Tracks don't have standalone ids, they are always in combination with the band id. + * That's why id = url. Instead, URLs are cleaned up so that they always look the same. */ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java new file mode 100644 index 000000000..b39ae4f2f --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +public class BandcampChannelExtractorTest { + + private static BandcampChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (BandcampChannelExtractor) bandcamp + .getChannelExtractor("https://zachbenson.bandcamp.com/"); + } + + @Test + public void testImageUrl() { + assertEquals("https://f4.bcbits.com/img/a2405652335_10.jpg", BandcampChannelExtractor.getImageUrl(2405652335L, true)); + assertEquals("https://f4.bcbits.com/img/17433693_10.jpg", BandcampChannelExtractor.getImageUrl(17433693L, false)); + } + + @Test + public void testTranslateIdsToUrl() throws ParsingException { + assertEquals("https://zachbenson.bandcamp.com/album/covers", BandcampExtractorHelper.getStreamUrlFromIds(2862267535L, 2063639444L, "album")); + // TODO write more test cases + } + + @Test + public void testLength() throws ParsingException { + assertTrue(extractor.getInitialPage().getItems().size() > 2); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index 7e46a096b..191f096c0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -51,4 +51,14 @@ public class BandcampChannelLinkHandlerFactoryTest { assertEquals("https://infiniteammo.bandcamp.com", linkHandler.getUrl("3321800855")); } + @Test(expected = ParsingException.class) + public void testGetUrlWithInvalidId() throws ParsingException { + linkHandler.getUrl("0"); + } + + @Test(expected = ParsingException.class) + public void testGetIdWithInvalidUrl() throws ParsingException { + linkHandler.getId("https://bandcamp.com"); + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index bcd925fcf..49b7c6f15 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -67,4 +67,9 @@ public class BandcampStreamExtractorTest { assertEquals(1, extractor.getAudioStreams().size()); } + @Test(expected = ParsingException.class) + public void testInvalidUrl() throws ExtractionException { + bandcamp.getStreamExtractor("https://bandcamp.com"); + } + } From 7730eb2ea13011b275296d24469623709c08f1f8 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 00:46:31 +0100 Subject: [PATCH 07/91] Bandcamp channel extractor: enforce https when getting banner url --- .../services/bandcamp/extractors/BandcampChannelExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index bb39ca1d0..1c6de58bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -71,7 +71,7 @@ public class BandcampChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { try { - String html = getDownloader().get(channelInfo.getString("bandcamp_url")) + String html = getDownloader().get(channelInfo.getString("bandcamp_url").replace("http://", "https://")) .responseBody(); return new Document(html).getElementById("customHeader") From d5cdc20be16e966fe3f3b5c60f26499bca046be3 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 01:05:34 +0100 Subject: [PATCH 08/91] Bandcmap channel extractor: fix getting banner url --- .../bandcamp/extractors/BandcampChannelExtractor.java | 6 ++++-- .../services/bandcamp/BandcampChannelExtractorTest.java | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 1c6de58bb..21289801d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.json.JSONArray; import org.json.JSONObject; -import org.jsoup.nodes.Document; +import org.jsoup.Jsoup; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -74,8 +74,10 @@ public class BandcampChannelExtractor extends ChannelExtractor { String html = getDownloader().get(channelInfo.getString("bandcamp_url").replace("http://", "https://")) .responseBody(); - return new Document(html).getElementById("customHeader") + return Jsoup.parse(html) + .getElementById("customHeader") .getElementsByTag("img") + .first() .attr("src"); } catch (IOException | ReCaptchaException e) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index b39ae4f2f..99a06d209 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -39,4 +39,10 @@ public class BandcampChannelExtractorTest { public void testLength() throws ParsingException { assertTrue(extractor.getInitialPage().getItems().size() > 2); } + + @Test + public void testGetBannerUrl() throws ParsingException { + // Why is this picture in png format when all other pictures are jpg? + assertTrue(extractor.getBannerUrl().endsWith(".png")); + } } From 91c0ec7ceabf643f67234c23d4a70b76f9e6ed82 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 01:14:35 +0100 Subject: [PATCH 09/91] Bandcamp stream extractor: catch NullPointerException --- .../bandcamp/extractors/BandcampStreamExtractor.java | 6 +++++- .../services/bandcamp/BandcampStreamExtractorTest.java | 5 +++++ .../bandcamp/BandcampStreamLinkHandlerFactoryTest.java | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index a8fa30004..be4c03729 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -109,7 +109,11 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploaderAvatarUrl() { - return document.getElementsByClass("band-photo").first().attr("src"); + try { + return document.getElementsByClass("band-photo").first().attr("src"); + } catch (NullPointerException e) { + return ""; + } } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 49b7c6f15..2f7a03bff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -61,6 +61,11 @@ public class BandcampStreamExtractorTest { assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); } + @Test + public void testNoArtistProfilePicture() throws ExtractionException { + assertEquals("", bandcamp.getStreamExtractor("https://powertothequeerkids.bandcamp.com/track/human-nature").getUploaderAvatarUrl()); + } + @Test public void testAudioStream() { assertTrue(extractor.getAudioStreams().get(0).getUrl().contains("bcbits.com/stream")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java index 0eb760776..ba4c26727 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -38,6 +38,7 @@ public class BandcampStreamLinkHandlerFactoryTest { assertFalse(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); + assertFalse(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids")); // Tests expecting true assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); From 794ca5eeaea99e3152278e562de5bca5851611c0 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 01:38:48 +0100 Subject: [PATCH 10/91] Bandcamp channel extractor: handel nonexistent images better --- .../bandcamp/extractors/BandcampChannelExtractor.java | 11 ++++++++++- .../bandcamp/extractors/BandcampExtractorHelper.java | 2 +- .../bandcamp/BandcampChannelExtractorTest.java | 11 +++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 21289801d..3f39603e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.jsoup.Jsoup; import org.schabi.newpipe.extractor.NewPipe; @@ -61,7 +62,12 @@ public class BandcampChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() { - return getImageUrl(channelInfo.getLong("bio_image_id"), false); + try { + return getImageUrl(channelInfo.getLong("bio_image_id"), false); + } catch (JSONException e) { + // In this case, the id is null and no image is available + return ""; + } } /** @@ -82,6 +88,9 @@ public class BandcampChannelExtractor extends ChannelExtractor { } catch (IOException | ReCaptchaException e) { throw new ParsingException("Could not download artist web site", e); + } catch (NullPointerException e) { + // No banner available + return ""; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index b1668641c..b86504c62 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -85,7 +85,7 @@ public class BandcampExtractorHelper { // Remove empty strings ArrayList list = new ArrayList<>(Arrays.asList(strings)); for (int i = list.size() - 1; i >= 0; i--) { - if (list.get(i) == null || list.get(i).isEmpty()) { + if (list.get(i) == null || list.get(i).isEmpty() || list.get(i).equals("null")) { list.remove(i); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 99a06d209..d924b4fb2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -4,6 +4,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; @@ -45,4 +46,14 @@ public class BandcampChannelExtractorTest { // Why is this picture in png format when all other pictures are jpg? assertTrue(extractor.getBannerUrl().endsWith(".png")); } + + @Test + public void testGetNoAvatar() throws ExtractionException { + assertEquals("", bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/").getAvatarUrl()); + } + + @Test + public void testGetNoBanner() throws ExtractionException { + assertEquals("", bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/").getBannerUrl()); + } } From d05b14ae48e3c82e8cbe671d5a7cd7133f4c9fd4 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 01:55:04 +0100 Subject: [PATCH 11/91] Add channels (artists) to search results --- .../BandcampChannelInfoItemExtractor.java | 46 +++++++++++++++++++ .../extractors/BandcampSearchExtractor.java | 11 +---- .../bandcamp/BandcampSearchExtractorTest.java | 21 +++++++-- 3 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java new file mode 100644 index 000000000..b5d851eab --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtractor { + + private String name, url, image, location; + + public BandcampChannelInfoItemExtractor(String name, String url, String image, String location) { + this.name = name; + this.url = url; + this.image = image; + this.location = location; + } + + @Override + public String getName() throws ParsingException { + return name; + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return image; + } + + @Override + public String getDescription() { + return location; + } + + @Override + public long getSubscriberCount() { + return -1; + } + + @Override + public long getStreamCount() { + return -1; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index 34f52af1e..fc89002fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -69,16 +69,7 @@ public class BandcampSearchExtractor extends SearchExtractor { break; case "ARTIST": - String id = resultInfo.getElementsByClass("itemurl").first() - .getElementsByTag("a").first() - .attr("href") // the link contains the id - .split("search_item_id=") - [1] // the number is behind its name - .split("&") // there is another attribute behind the name - [0]; // get the number - - //searchResults.add(new Artist(heading, Long.parseLong(id), image, subhead)); - //collector.commit Channel with heading, id, image, subhead + collector.commit(new BandcampChannelInfoItemExtractor(heading, url, image, subhead)); break; case "ALBUM": diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 11ce3d0d3..5780e1612 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -5,10 +5,13 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelInfoItemExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; import java.io.IOException; @@ -36,8 +39,7 @@ public class BandcampSearchExtractorTest { */ @Test public void testBestFriendsBasement() throws ExtractionException, IOException { - extractor = (BandcampSearchExtractor) bandcamp - .getSearchExtractor("best friend's basement"); + SearchExtractor extractor = bandcamp.getSearchExtractor("best friend's basement"); ListExtractor.InfoItemsPage page = extractor.getInitialPage(); InfoItem bestFriendsBasement = page.getItems().get(0); @@ -47,9 +49,22 @@ public class BandcampSearchExtractorTest { assertTrue(bestFriendsBasement.getThumbnailUrl().endsWith(".jpg")); assertTrue(bestFriendsBasement.getThumbnailUrl().contains("f4.bcbits.com/img/")); assertEquals(InfoItem.InfoType.STREAM, bestFriendsBasement.getInfoType()); + } + /** + * Tests whether searching bandcamp for "C418" returns the artist's profile + */ + @Test + public void testC418() throws ExtractionException, IOException { + SearchExtractor extractor = bandcamp.getSearchExtractor("C418"); + InfoItem c418 = extractor.getInitialPage() + .getItems().get(0); - + // C418's artist profile should be the first result, no? + assertEquals("C418", c418.getName()); + assertTrue(c418.getThumbnailUrl().endsWith(".jpg")); + assertTrue(c418.getThumbnailUrl().contains("f4.bcbits.com/img/")); + assertEquals("https://c418.bandcamp.com", c418.getUrl()); } } From ba700bfb3e05752ea2e55ed57ce6973d0258a0e7 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 02:55:54 +0100 Subject: [PATCH 12/91] Add bandcamp playlists (albums) --- .../services/bandcamp/BandcampService.java | 6 +- .../BandcampChannelInfoItemExtractor.java | 2 + .../extractors/BandcampPlaylistExtractor.java | 133 ++++++++++++++++++ .../BandcampPlaylistInfoItemExtractor.java | 40 ++++++ .../extractors/BandcampSearchExtractor.java | 3 +- .../extractors/BandcampStreamExtractor.java | 10 +- .../BandcampStreamInfoItemExtractor.java | 28 ++-- .../BandcampPlaylistLinkHandlerFactory.java | 31 ++++ .../BandcampChannelExtractorTest.java | 2 + .../BandcampPlaylistExtractorTest.java | 29 ++++ ...andcampPlaylistLinkHandlerFactoryTest.java | 45 ++++++ .../bandcamp/BandcampSearchExtractorTest.java | 21 ++- 12 files changed, 330 insertions(+), 20 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 4752e1f09..211f7310f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -11,9 +11,11 @@ import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampPlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -47,7 +49,7 @@ public class BandcampService extends StreamingService { @Override public ListLinkHandlerFactory getPlaylistLHFactory() { - return null; + return new BandcampPlaylistLinkHandlerFactory(); } @Override @@ -87,7 +89,7 @@ public class BandcampService extends StreamingService { @Override public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return null; + return new BandcampPlaylistExtractor(this, linkHandler); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java index b5d851eab..33d999117 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java @@ -1,3 +1,5 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java new file mode 100644 index 000000000..ac5e8998d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -0,0 +1,133 @@ +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getJSONFromJavaScriptVariables; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; + +public class BandcampPlaylistExtractor extends PlaylistExtractor { + + private Document document; + private JSONObject albumJson; + private JSONArray trackInfo; + private String name; + + public BandcampPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + String html = downloader.get(getLinkHandler().getUrl()).responseBody(); + document = Jsoup.parse(html); + albumJson = getAlbumInfoJson(html); + trackInfo = albumJson.getJSONArray("trackinfo"); + + try { + name = getJSONFromJavaScriptVariables(html, "EmbedData").getString("album_title"); + } catch (JSONException e) { + throw new ParsingException("Faulty JSON; page likely does not contain album data", e); + } catch (ArrayIndexOutOfBoundsException e) { + throw new ParsingException("JSON does not exist", e); + } + + + + if (trackInfo.length() <= 1) { + // In this case, we are actually viewing a track page! + throw new ExtractionException("Page is actually a track, not an album"); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); + } catch (NullPointerException e) { + return ""; + } + } + + @Override + public String getBannerUrl() { + return ""; + } + + @Override + public String getUploaderUrl() throws ParsingException { + String[] parts = getUrl().split("/"); + // https: (/) (/) * .bandcamp.com (/) and leave out the rest + return "https://" + parts[2] + "/"; + } + + @Override + public String getUploaderName() { + return albumJson.getString("artist"); + } + + @Override + public String getUploaderAvatarUrl() { + try { + return document.getElementsByClass("band-photo").first().attr("src"); + } catch (NullPointerException e) { + return ""; + } + } + + @Override + public long getStreamCount() { + return trackInfo.length(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException { + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + for (int i = 0; i < trackInfo.length(); i++) { + JSONObject track = trackInfo.getJSONObject(i); + + collector.commit(new BandcampStreamInfoItemExtractor( + track.getString("title"), + getUploaderUrl() + track.getString("title_link"), + "", + "", + track.getLong("duration") + )); + } + + return new InfoItemsPage<>(collector, null); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return name; + } + + @Override + public String getNextPageUrl() { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java new file mode 100644 index 000000000..11aea8ea4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -0,0 +1,40 @@ +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; + +public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + + private String title, artist, url, cover; + + public BandcampPlaylistInfoItemExtractor(String title, String artist, String url, String cover) { + this.title = title; + this.artist = artist; + this.url = url; + this.cover = cover; + } + + @Override + public String getUploaderName() { + return artist; + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public String getName() { + return title; + } + + @Override + public String getUrl() { + return url; + } + + @Override + public String getThumbnailUrl() { + return cover; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index fc89002fc..38891c004 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -74,8 +74,7 @@ public class BandcampSearchExtractor extends SearchExtractor { case "ALBUM": String artist = subhead.split(" by")[0]; - //searchResults.add(new Album(heading, artist, url, image)); - //collector.commit Playlist with heading, artist, url, image + collector.commit(new BandcampPlaylistInfoItemExtractor(heading, artist, url, image)); break; case "TRACK": diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index be4c03729..72c3a551c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -50,7 +50,7 @@ public class BandcampStreamExtractor extends StreamExtractor { * * @param html Website * @return Album metadata JSON - * @throws ParsingException In case of a faulty website + * @throws ParsingException In case of a faulty website */ public static JSONObject getAlbumInfoJson(String html) throws ParsingException { try { @@ -103,7 +103,11 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public String getThumbnailUrl() throws ParsingException { - return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); + try { + return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); + } catch (NullPointerException e) { + return ""; + } } @Nonnull @@ -188,7 +192,7 @@ public class BandcampStreamExtractor extends StreamExtractor { audioStreams.add(new AudioStream( albumJson.getJSONArray("trackinfo").getJSONObject(0) - .getJSONObject("file").getString("mp3-128"), + .getJSONObject("file").getString("mp3-128"), MediaFormat.MP3, 128 )); return audioStreams; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java index 6cb96411e..4f2a5724f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java @@ -15,48 +15,54 @@ public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor private String url; private String cover; private String artist; + private long duration; public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist) { + this(title, url, cover, artist, -1); + } + + public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist, long duration) { this.title = title; this.url = url; this.cover = cover; this.artist = artist; + this.duration = duration; } @Override - public StreamType getStreamType() throws ParsingException { + public StreamType getStreamType() { return StreamType.AUDIO_STREAM; } @Override - public long getDuration() throws ParsingException { + public long getDuration() { + return duration; + } + + @Override + public long getViewCount() { return -1; } @Override - public long getViewCount() throws ParsingException { - return -1; - } - - @Override - public String getUploaderName() throws ParsingException { + public String getUploaderName() { return artist; } @Override - public String getUploaderUrl() throws ParsingException { + public String getUploaderUrl() { return null; } @Nullable @Override - public String getTextualUploadDate() throws ParsingException { + public String getTextualUploadDate() { return null; // TODO } @Nullable @Override - public DateWrapper getUploadDate() throws ParsingException { + public DateWrapper getUploadDate() { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java new file mode 100644 index 000000000..e7aac43e2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java @@ -0,0 +1,31 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import java.util.List; + +/** + * Just as with streams, the album ids are essentially useless for us. + */ +public class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFactory { + @Override + public String getId(String url) throws ParsingException { + return getUrl(url); + } + + @Override + public String getUrl(String url, List contentFilter, String sortFilter) throws ParsingException { + if (url.endsWith("/")) + url = url.substring(0, url.length() - 1); + url = url.replace("http://", "https://").toLowerCase(); + return url; + } + + @Override + public boolean onAcceptUrl(String url) throws ParsingException { + return getUrl(url).matches("https?://.+\\..+/album/.+"); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index d924b4fb2..850529774 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -1,3 +1,5 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java new file mode 100644 index 000000000..e7482dfb5 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -0,0 +1,29 @@ +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +public class BandcampPlaylistExtractorTest { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testCount() throws ExtractionException, IOException { + PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); + extractor.fetchPage(); + + assertEquals(5, extractor.getStreamCount()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java new file mode 100644 index 000000000..1da3069c9 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java @@ -0,0 +1,45 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampPlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test for {@link BandcampPlaylistLinkHandlerFactory} + */ +public class BandcampPlaylistLinkHandlerFactoryTest { + + private static BandcampPlaylistLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = new BandcampPlaylistLinkHandlerFactory(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testAcceptUrl() throws ParsingException { + // Tests expecting false + assertFalse(linkHandler.acceptUrl("http://interovgm.com/releases/")); + assertFalse(linkHandler.acceptUrl("https://interovgm.com/releases")); + assertFalse(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); + assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); + assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); + assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); + assertFalse(linkHandler.acceptUrl("https://interovgm.com/track/title")); + + // Tests expecting true + assertTrue(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids")); + assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/album/prom")); + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 5780e1612..98bdc843f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -38,7 +38,7 @@ public class BandcampSearchExtractorTest { * the accordingly named song by Zach Benson */ @Test - public void testBestFriendsBasement() throws ExtractionException, IOException { + public void testStreamSearch() throws ExtractionException, IOException { SearchExtractor extractor = bandcamp.getSearchExtractor("best friend's basement"); ListExtractor.InfoItemsPage page = extractor.getInitialPage(); @@ -55,7 +55,7 @@ public class BandcampSearchExtractorTest { * Tests whether searching bandcamp for "C418" returns the artist's profile */ @Test - public void testC418() throws ExtractionException, IOException { + public void testChannelSearch() throws ExtractionException, IOException { SearchExtractor extractor = bandcamp.getSearchExtractor("C418"); InfoItem c418 = extractor.getInitialPage() .getItems().get(0); @@ -67,4 +67,21 @@ public class BandcampSearchExtractorTest { assertEquals("https://c418.bandcamp.com", c418.getUrl()); } + + /** + * Tests whether searching bandcamp for "minecraft volume alpha" returns the corresponding album + */ + @Test + public void testAlbumSearch() throws ExtractionException, IOException { + SearchExtractor extractor = bandcamp.getSearchExtractor("minecraft volume alpha"); + InfoItem minecraft = extractor.getInitialPage() + .getItems().get(0); + + // C418's artist profile should be the first result, no? + assertEquals("Minecraft - Volume Alpha", minecraft.getName()); + assertTrue(minecraft.getThumbnailUrl().endsWith(".jpg")); + assertTrue(minecraft.getThumbnailUrl().contains("f4.bcbits.com/img/")); + assertEquals("https://c418.bandcamp.com/album/minecraft-volume-alpha", minecraft.getUrl()); + + } } From 655df356a2b7c407211e3f1d05974ce5f5dd3d2d Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 03:10:54 +0100 Subject: [PATCH 13/91] Various improvements concerning bandcamp playlists --- .../extractors/BandcampPlaylistExtractor.java | 4 +-- .../BandcampPlaylistInfoItemExtractor.java | 20 ++++++++++++--- .../extractors/BandcampSearchExtractor.java | 2 +- .../BandcampStreamInfoItemExtractor.java | 25 ++++++++++++++++++- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index ac5e8998d..78c5f0213 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -107,8 +107,8 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { track.getString("title"), getUploaderUrl() + track.getString("title_link"), "", - "", - track.getLong("duration") + track.getLong("duration"), + getService() )); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index 11aea8ea4..5257691e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -1,16 +1,24 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import java.io.IOException; + public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private String title, artist, url, cover; + private StreamingService service; - public BandcampPlaylistInfoItemExtractor(String title, String artist, String url, String cover) { + public BandcampPlaylistInfoItemExtractor(String title, String artist, String url, String cover, StreamingService service) { this.title = title; this.artist = artist; this.url = url; this.cover = cover; + this.service = service; } @Override @@ -19,8 +27,14 @@ public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtrac } @Override - public long getStreamCount() { - return -1; + public long getStreamCount() throws ParsingException { + try { + PlaylistExtractor extractor = service.getPlaylistExtractor(url); + extractor.fetchPage(); + return extractor.getStreamCount(); + } catch (ExtractionException | IOException e) { + throw new ParsingException("Could not find out how many tracks there are", e); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index 38891c004..f31a50651 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -74,7 +74,7 @@ public class BandcampSearchExtractor extends SearchExtractor { case "ALBUM": String artist = subhead.split(" by")[0]; - collector.commit(new BandcampPlaylistInfoItemExtractor(heading, artist, url, image)); + collector.commit(new BandcampPlaylistInfoItemExtractor(heading, artist, url, image, getService())); break; case "TRACK": diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java index 4f2a5724f..dd51f7692 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java @@ -2,12 +2,16 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; +import java.io.IOException; public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -16,6 +20,12 @@ public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor private String cover; private String artist; private long duration; + private StreamingService service; + + public BandcampStreamInfoItemExtractor(String title, String url, String artist, long duration, StreamingService service) { + this(title, url, null, artist, duration); + this.service = service; + } public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist) { this(title, url, cover, artist, -1); @@ -76,9 +86,22 @@ public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor return url; } + /** + * There is no guarantee that every track of an album has the same cover art, so it needs to be fetched + * per-track if in playlist view + */ @Override public String getThumbnailUrl() throws ParsingException { - return cover; + if (cover != null) return cover; + else { + try { + StreamExtractor extractor = service.getStreamExtractor(getUrl()); + extractor.fetchPage(); + return extractor.getThumbnailUrl(); + } catch (ExtractionException | IOException e) { + throw new ParsingException("could not download cover art location", e); + } + } } /** From e12ddaef7f7ab8c447df2310490e5a37d23bd84a Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 12:53:15 +0100 Subject: [PATCH 14/91] Bandcamp playlist and search extractors: better and more tests --- .../extractors/BandcampChannelExtractor.java | 1 + .../extractors/BandcampPlaylistExtractor.java | 3 +- .../BandcampPlaylistExtractorTest.java | 36 ++++++++++++++++++- .../bandcamp/BandcampSearchExtractorTest.java | 7 +++- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 3f39603e6..ddb8f3365 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -6,6 +6,7 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.jsoup.Jsoup; +import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 78c5f0213..4544644ea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -7,6 +7,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -50,7 +51,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { if (trackInfo.length() <= 1) { // In this case, we are actually viewing a track page! - throw new ExtractionException("Page is actually a track, not an album"); + throw new ContentNotAvailableException("Album needs to be purchased or is actually a track"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index e7482dfb5..c6572974a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -1,17 +1,26 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; +import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +/** + * Tests for {@link BandcampPlaylistExtractor} + */ public class BandcampPlaylistExtractorTest { @BeforeClass @@ -19,6 +28,9 @@ public class BandcampPlaylistExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); } + /** + * Test whether playlists contain the correct amount of items + */ @Test public void testCount() throws ExtractionException, IOException { PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); @@ -26,4 +38,26 @@ public class BandcampPlaylistExtractorTest { assertEquals(5, extractor.getStreamCount()); } + + /** + * Tests whether different stream thumbnails (track covers) get loaded correctly + */ + @Test + public void testDifferentTrackCovers() throws ExtractionException, IOException { + PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://zachbensonarchive.bandcamp.com/album/results-of-boredom"); + extractor.fetchPage(); + + List l = extractor.getInitialPage().getItems(); + assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); + assertNotEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); + } + + /** + * Test playlists with locked content + */ + @Test(expected = ContentNotAvailableException.class) + public void testLockedContent() throws ExtractionException, IOException { + PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://billwurtz.bandcamp.com/album/high-enough"); + extractor.fetchPage(); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 98bdc843f..45624fe76 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -10,8 +10,10 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; import java.io.IOException; @@ -77,11 +79,14 @@ public class BandcampSearchExtractorTest { InfoItem minecraft = extractor.getInitialPage() .getItems().get(0); - // C418's artist profile should be the first result, no? + // Minecraft volume alpha should be the first result, no? assertEquals("Minecraft - Volume Alpha", minecraft.getName()); assertTrue(minecraft.getThumbnailUrl().endsWith(".jpg")); assertTrue(minecraft.getThumbnailUrl().contains("f4.bcbits.com/img/")); assertEquals("https://c418.bandcamp.com/album/minecraft-volume-alpha", minecraft.getUrl()); + // Verify that playlists get counted correctly + assertEquals(24, ((PlaylistInfoItem) minecraft).getStreamCount()); + } } From ce2a88e56ff2e7c819429b8fff55f20775a5706d Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 13:28:02 +0100 Subject: [PATCH 15/91] Add bandcamp search suggestion extractor --- .../services/bandcamp/BandcampService.java | 7 +-- .../BandcampSuggestionExtractor.java | 51 +++++++++++++++++++ .../BandcampSuggestionExtractorTest.java | 41 +++++++++++++++ 3 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 211f7310f..103ff6ec4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -10,10 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.*; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampPlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; @@ -69,7 +66,7 @@ public class BandcampService extends StreamingService { @Override public SuggestionExtractor getSuggestionExtractor() { - return null; + return new BandcampSuggestionExtractor(this); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java new file mode 100644 index 000000000..6cf5d70b2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -0,0 +1,51 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory.CHARSET_UTF_8; + +public class BandcampSuggestionExtractor extends SuggestionExtractor { + + private static final String AUTOCOMPLETE_URL = "https://bandcamp.com/api/fuzzysearch/1/autocomplete?q="; + public BandcampSuggestionExtractor(StreamingService service) { + super(service); + } + + @Override + public List suggestionList(String query) throws IOException, ExtractionException { + Downloader downloader = NewPipe.getDownloader(); + + JSONObject fuzzyResults = new JSONObject( + downloader.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, CHARSET_UTF_8)).responseBody() + ); + + JSONArray jsonArray = fuzzyResults.getJSONObject("auto") + .getJSONArray("results"); + + ArrayList suggestions = new ArrayList<>(); + + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject fuzzyResult = jsonArray.getJSONObject(i); + String res = fuzzyResult.getString("name"); + + if (!suggestions.contains(res)) suggestions.add(res); + } + + + + return suggestions; + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java new file mode 100644 index 000000000..77b4f26b5 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java @@ -0,0 +1,41 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSuggestionExtractor; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +/** + * Tests for {@link BandcampSuggestionExtractor} + */ +public class BandcampSuggestionExtractorTest { + + private static BandcampSuggestionExtractor extractor; + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (BandcampSuggestionExtractor) bandcamp.getSuggestionExtractor(); + } + + @Test + public void testSearchExample() throws IOException, ExtractionException { + List c418 = extractor.suggestionList("c418"); + + // When looking for c418, one should find C418 + assertTrue(c418.contains("C418")); + + // There should be five results, but we can't be sure of that forever + assertTrue(c418.size() > 2); + } +} From 9b16baffb7438af1f3b79f365cdc01a2a32dadbc Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 13:51:17 +0100 Subject: [PATCH 16/91] Bandcamp search: multiple pages --- .../extractors/BandcampSearchExtractor.java | 34 ++++++++++++++++--- .../BandcampSearchQueryHandlerFactory.java | 3 +- .../bandcamp/BandcampSearchExtractorTest.java | 13 +++++++ ...BandcampSearchQueryHandlerFactoryTest.java | 4 +-- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index f31a50651..31b1b4b84 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -89,19 +89,45 @@ public class BandcampSearchExtractor extends SearchExtractor { } + // Count pages + Elements pageLists = d.getElementsByClass("pagelist"); + if (pageLists.size() == 0) + return new InfoItemsPage<>(collector, null); + + Elements pages = pageLists.first().getElementsByTag("li"); + + // Find current page + int currentPage = -1; + for (int i = 0; i < pages.size(); i++) { + Element page = pages.get(i); + if (page.getElementsByTag("span").size() > 0) { + currentPage = i + 1; + break; + } + } + + // Search results appear to be capped at six pages + assert pages.size() < 10; + + String nextUrl = null; + if (currentPage < pages.size()) { + nextUrl = pageUrl.substring(0, pageUrl.length() - 1) + (currentPage + 1); + } + + return new InfoItemsPage<>(collector, nextUrl); - return new InfoItemsPage<>(getInfoItemSearchCollector(), null); } @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - return getPage(getUrl());//new InfoItemsPage<>(getInfoItemSearchCollector(), null); + return getPage(getUrl()); } @Override - public String getNextPageUrl() throws IOException, ExtractionException { - return null; + public String getNextPageUrl() throws ExtractionException { + String url = getUrl(); + return url.substring(0, url.length() - 1).concat("2"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java index 3f23956fe..0fad75d5f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java @@ -21,7 +21,8 @@ public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory try { return SEARCH_URL + - URLEncoder.encode(query, CHARSET_UTF_8); + URLEncoder.encode(query, CHARSET_UTF_8) + + "&page=1"; } catch (UnsupportedEncodingException e) { throw new ParsingException("query \"" + query + "\" could not be encoded", e); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 45624fe76..8cffcf73e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -89,4 +89,17 @@ public class BandcampSearchExtractorTest { assertEquals(24, ((PlaylistInfoItem) minecraft).getStreamCount()); } + + /** + * Tests searches with multiple pages + */ + @Test + public void testMultiplePages() throws ExtractionException, IOException { + // A query practically guaranteed to have the maximum amount of pages + SearchExtractor extractor = bandcamp.getSearchExtractor("e"); + + assertEquals("https://bandcamp.com/search?q=e&page=2", extractor.getInitialPage().getNextPageUrl()); + + assertEquals("https://bandcamp.com/search?q=e&page=3", extractor.getPage(extractor.getNextPageUrl()).getNextPageUrl()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java index 1a8fdac5b..4158e655b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java @@ -27,8 +27,8 @@ public class BandcampSearchQueryHandlerFactoryTest { @Test public void testEncoding() throws ParsingException { // Note: this isn't exactly as bandcamp does it (it wouldn't encode '!'), but both works - assertEquals("https://bandcamp.com/search?q=hello%21%22%C2%A7%24%25%26%2F%28%29%3D", searchQuery.getUrl("hello!\"ยง$%&/()=")); + assertEquals("https://bandcamp.com/search?q=hello%21%22%C2%A7%24%25%26%2F%28%29%3D&page=1", searchQuery.getUrl("hello!\"ยง$%&/()=")); // Note: bandcamp uses %20 instead of '+', but both works - assertEquals("https://bandcamp.com/search?q=search+query+with+spaces", searchQuery.getUrl("search query with spaces")); + assertEquals("https://bandcamp.com/search?q=search+query+with+spaces&page=1", searchQuery.getUrl("search query with spaces")); } } From 13e4908b832e042141c3d8577b3cec4e69ee27d0 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 14:07:56 +0100 Subject: [PATCH 17/91] Bandcamp search extractor: read track count from result page --- .../BandcampPlaylistInfoItemExtractor.java | 16 +++++----------- .../extractors/BandcampSearchExtractor.java | 4 +++- .../bandcamp/BandcampSearchExtractorTest.java | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index 5257691e6..61551bac5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -11,14 +11,14 @@ import java.io.IOException; public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private String title, artist, url, cover; - private StreamingService service; + private int trackCount; - public BandcampPlaylistInfoItemExtractor(String title, String artist, String url, String cover, StreamingService service) { + public BandcampPlaylistInfoItemExtractor(String title, String artist, String url, String cover, int trackCount) { this.title = title; this.artist = artist; this.url = url; this.cover = cover; - this.service = service; + this.trackCount = trackCount; } @Override @@ -27,14 +27,8 @@ public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtrac } @Override - public long getStreamCount() throws ParsingException { - try { - PlaylistExtractor extractor = service.getPlaylistExtractor(url); - extractor.fetchPage(); - return extractor.getStreamCount(); - } catch (ExtractionException | IOException e) { - throw new ParsingException("Could not find out how many tracks there are", e); - } + public long getStreamCount() { + return trackCount; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index 31b1b4b84..6831cdf62 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -74,7 +74,9 @@ public class BandcampSearchExtractor extends SearchExtractor { case "ALBUM": String artist = subhead.split(" by")[0]; - collector.commit(new BandcampPlaylistInfoItemExtractor(heading, artist, url, image, getService())); + String length = resultInfo.getElementsByClass("length").text(); + int tracks = Integer.parseInt(length.split(" track")[0]); + collector.commit(new BandcampPlaylistInfoItemExtractor(heading, artist, url, image, tracks)); break; case "TRACK": diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 8cffcf73e..1e6f9b579 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -85,7 +85,7 @@ public class BandcampSearchExtractorTest { assertTrue(minecraft.getThumbnailUrl().contains("f4.bcbits.com/img/")); assertEquals("https://c418.bandcamp.com/album/minecraft-volume-alpha", minecraft.getUrl()); - // Verify that playlists get counted correctly + // Verify that playlist tracks counts get extracted correctly assertEquals(24, ((PlaylistInfoItem) minecraft).getStreamCount()); } From db02850d4f899e2226e2fcc99dd39bbcb70b0091 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 14:12:24 +0100 Subject: [PATCH 18/91] Bandcamp playlist extractor: support playlists with just one stream --- .../extractors/BandcampPlaylistExtractor.java | 6 +++--- .../bandcamp/BandcampPlaylistExtractorTest.java | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 4544644ea..e24e603de 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -49,9 +49,9 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { - if (trackInfo.length() <= 1) { - // In this case, we are actually viewing a track page! - throw new ContentNotAvailableException("Album needs to be purchased or is actually a track"); + if (trackInfo.length() < 0) { + // Albums without trackInfo need to be purchased before they can be played + throw new ContentNotAvailableException("Album needs to be purchased"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index c6572974a..7400b1821 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -60,4 +60,16 @@ public class BandcampPlaylistExtractorTest { PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://billwurtz.bandcamp.com/album/high-enough"); extractor.fetchPage(); } + + /** + * Test playlist with just one track + */ + @Test + public void testSingleStreamPlaylist() throws ExtractionException, IOException { + PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://zachjohnson1.bandcamp.com/album/endless"); + extractor.fetchPage(); + + assertEquals(1, extractor.getStreamCount()); + + } } From 13ef11e9c8a1c60aaf8e68b63ad2fb88723569f9 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 15:15:12 +0100 Subject: [PATCH 19/91] Bandcamp featured kiosk --- .../services/bandcamp/BandcampService.java | 34 +++++-- .../extractors/BandcampFeaturedExtractor.java | 91 +++++++++++++++++++ .../BandcampFeaturedLinkHandlerFactory.java | 34 +++++++ .../BandcampFeaturedExtractorTest.java | 39 ++++++++ 4 files changed, 190 insertions(+), 8 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 103ff6ec4..658101bd3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -6,22 +6,23 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.*; -import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampPlaylistLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.*; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import java.io.IOException; import java.util.Collections; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED; +import static org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory.FEATURED_API_URL; public class BandcampService extends StreamingService { @@ -76,7 +77,24 @@ public class BandcampService extends StreamingService { @Override public KioskList getKioskList() throws ExtractionException { - return new KioskList(this); + + KioskList kioskList = new KioskList(this); + try { + kioskList + .addKioskEntry(new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException { + return new BandcampFeaturedExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), kioskId); + } + }, new BandcampFeaturedLinkHandlerFactory(), KIOSK_FEATURED); + + kioskList.setDefaultKiosk(KIOSK_FEATURED); + + } catch (Exception e) { + throw new ExtractionException(e); + } + + return kioskList; } @Override @@ -85,17 +103,17 @@ public class BandcampService extends StreamingService { } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException { + public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { return new BandcampPlaylistExtractor(this, linkHandler); } @Override - public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException { + public StreamExtractor getStreamExtractor(LinkHandler linkHandler) { return new BandcampStreamExtractor(this, linkHandler); } @Override - public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException { + public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) { return null; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java new file mode 100644 index 000000000..5e8f21d49 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -0,0 +1,91 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.Collector; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory; + +import javax.annotation.Nonnull; +import java.io.IOException; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory.FEATURED_API_URL; + +public class BandcampFeaturedExtractor extends KioskExtractor { + + public static final String KIOSK_FEATURED = "Featured"; + + public BandcampFeaturedExtractor(StreamingService streamingService, ListLinkHandler listLinkHandler, String kioskId) { + super(streamingService, listLinkHandler, kioskId); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return null; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + + InfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); + + JSONObject json = new JSONObject( + getDownloader().post( + FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes() + ).responseBody() + ); + + JSONArray featuredStories = json.getJSONObject("feed_content") + .getJSONObject("stories") + .getJSONArray("featured"); + + for (int i = 0; i < featuredStories.length(); i++) { + JSONObject featuredStory = featuredStories.getJSONObject(i); + + if (featuredStory.isNull("album_title")) { + // Is not an album, ignore + continue; + } + + c.commit(new BandcampPlaylistInfoItemExtractor( + featuredStory.getString("album_title"), + featuredStory.getString("band_name"), + featuredStory.getString("item_url"), + featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "", + featuredStory.getInt("num_streamable_tracks") + )); + } + + return new InfoItemsPage(c, null); + } + + @Override + public String getNextPageUrl() { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java new file mode 100644 index 000000000..3fdcb33ba --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java @@ -0,0 +1,34 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED; + +public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory { + + /** + * This kiosk doesn't have a corresponding website + */ + public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data"; + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) { + if (id.equals(KIOSK_FEATURED)) return FEATURED_API_URL; + else return null; + } + + @Override + public String getId(String url) { + return KIOSK_FEATURED; + } + + @Override + public boolean onAcceptUrl(String url) { + return url.equals(FEATURED_API_URL); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java new file mode 100644 index 000000000..258098d33 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -0,0 +1,39 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +/** + * Tests for {@link BandcampFeaturedExtractor} + */ +public class BandcampFeaturedExtractorTest { + + private static BandcampFeaturedExtractor extractor; + + @BeforeClass + public static void setUp() throws ExtractionException, IOException { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (BandcampFeaturedExtractor) bandcamp + .getKioskList().getDefaultKioskExtractor(); + } + + @Test + public void testKioskItems() throws ExtractionException, IOException { + List list = extractor.getInitialPage().getItems(); + assertTrue(list.size() > 1); + } + +} From 808c8aa0872ef0bee4d8d042dc5d772babfde154 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 17:30:22 +0100 Subject: [PATCH 20/91] Add bandcamp weekly kiosk Unfortunately a little messy --- .../services/bandcamp/BandcampService.java | 31 +++-- .../extractors/BandcampFeaturedExtractor.java | 4 +- .../extractors/BandcampRadioExtractor.java | 74 +++++++++++ .../BandcampRadioInfoItemExtractor.java | 88 ++++++++++++ .../BandcampRadioStreamExtractor.java | 125 ++++++++++++++++++ .../BandcampFeaturedLinkHandlerFactory.java | 20 +-- .../BandcampStreamLinkHandlerFactory.java | 21 ++- .../BandcampFeaturedExtractorTest.java | 2 +- ...andcampFeaturedLinkHandlerFactoryTest.java | 52 ++++++++ .../bandcamp/BandcampRadioExtractorTest.java | 40 ++++++ .../BandcampRadioStreamExtractorTest.java | 37 ++++++ .../BandcampStreamLinkHandlerFactoryTest.java | 11 ++ 12 files changed, 477 insertions(+), 28 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 658101bd3..1aa5b3c77 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -21,8 +21,10 @@ import java.io.IOException; import java.util.Collections; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED; -import static org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory.FEATURED_API_URL; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL; public class BandcampService extends StreamingService { @@ -79,14 +81,22 @@ public class BandcampService extends StreamingService { public KioskList getKioskList() throws ExtractionException { KioskList kioskList = new KioskList(this); + try { - kioskList - .addKioskEntry(new KioskList.KioskExtractorFactory() { - @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException { - return new BandcampFeaturedExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), kioskId); - } - }, new BandcampFeaturedLinkHandlerFactory(), KIOSK_FEATURED); + kioskList.addKioskEntry(new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException { + return new BandcampFeaturedExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), kioskId); + } + }, new BandcampFeaturedLinkHandlerFactory(), KIOSK_FEATURED); + + + kioskList.addKioskEntry(new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException { + return new BandcampRadioExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL), kioskId); + } + }, new BandcampFeaturedLinkHandlerFactory(), KIOSK_RADIO); kioskList.setDefaultKiosk(KIOSK_FEATURED); @@ -109,7 +119,10 @@ public class BandcampService extends StreamingService { @Override public StreamExtractor getStreamExtractor(LinkHandler linkHandler) { - return new BandcampStreamExtractor(this, linkHandler); + if (linkHandler.getUrl().matches("https?://bandcamp\\.com/\\?show=\\d+")) + return new BandcampRadioStreamExtractor(this, linkHandler); + else + return new BandcampStreamExtractor(this, linkHandler); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 5e8f21d49..f8e67c6bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -22,11 +22,11 @@ import javax.annotation.Nonnull; import java.io.IOException; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; -import static org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory.FEATURED_API_URL; public class BandcampFeaturedExtractor extends KioskExtractor { public static final String KIOSK_FEATURED = "Featured"; + public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data"; public BandcampFeaturedExtractor(StreamingService streamingService, ListLinkHandler listLinkHandler, String kioskId) { super(streamingService, listLinkHandler, kioskId); @@ -40,7 +40,7 @@ public class BandcampFeaturedExtractor extends KioskExtractor { @Nonnull @Override public String getName() throws ParsingException { - return null; + return KIOSK_FEATURED; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java new file mode 100644 index 000000000..1e2f8aa7f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java @@ -0,0 +1,74 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class BandcampRadioExtractor extends KioskExtractor { + + public static final String KIOSK_RADIO = "Radio"; + public static final String RADIO_API_URL = "https://bandcamp.com/api/bcweekly/1/list"; + + public BandcampRadioExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) { + super(streamingService, linkHandler, kioskId); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return KIOSK_RADIO; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + InfoItemsCollector c = new StreamInfoItemsCollector(getServiceId()); + + JSONObject json = new JSONObject( + getDownloader().get( + RADIO_API_URL + ).responseBody() + ); + + JSONArray radioShows = json.getJSONArray("results"); + + for (int i = 0; i < radioShows.length(); i++) { + JSONObject radioShow = radioShows.getJSONObject(i); + + c.commit( + new BandcampRadioInfoItemExtractor(radioShow) + ); + } + + return new InfoItemsPage(c, null); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java new file mode 100644 index 000000000..7e7017140 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -0,0 +1,88 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONObject; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; +import java.io.IOException; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor.query; + +public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { + + private JSONObject show; + + public BandcampRadioInfoItemExtractor(JSONObject radioShow) { + show = radioShow; + } + + @Override + public long getDuration() { + /* Duration is only present in the more detailed information that has to be queried seperately. + * Because the servers would probably not like over 300 queries every time someone opens the kiosk, + * we're just providing 0 here. + */ + //return query(show.getInt("id")).getLong("audio_duration"); + return 0; + } + + @Nullable + @Override + public String getTextualUploadDate() { + return show.getString("date"); + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public String getName() throws ParsingException { + return show.getString("date"); + } + + @Override + public String getUrl() { + return "https://bandcamp.com/?show=" + show.getInt("id"); + } + + @Override + public String getThumbnailUrl() { + return getImageUrl(show.getLong("image_id"), false); + } + + @Override + public StreamType getStreamType() { + return StreamType.AUDIO_STREAM; + } + + @Override + public long getViewCount() { + return -1; + } + + @Override + public String getUploaderName() { + return ""; + } + + @Override + public String getUploaderUrl() { + return ""; + } + + @Override + public boolean isAd() { + return false; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java new file mode 100644 index 000000000..2449f572f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -0,0 +1,125 @@ +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.AudioStream; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; + +public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { + + private JSONObject showInfo; + private LinkHandler linkHandler; + + public BandcampRadioStreamExtractor(StreamingService service, LinkHandler linkHandler) { + super(service, linkHandler); + this.linkHandler = linkHandler; + } + + static JSONObject query(int id) throws ParsingException { + try { + return new JSONObject( + NewPipe.getDownloader().get("https://bandcamp.com/api/bcweekly/1/get?id=" + id).responseBody() + ); + } catch (IOException | ReCaptchaException e) { + throw new ParsingException("could not get show data", e); + } + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + showInfo = query(Integer.parseInt(getId())); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return showInfo.getString("subtitle"); // "audio_title" is a boring title + } + + @Nonnull + @Override + public String getUploaderUrl() { + return Jsoup.parse(showInfo.getString("image_caption")) + .getElementsByTag("a").first().attr("href"); + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return linkHandler.getUrl(); + } + + @Nonnull + @Override + public String getUploaderName() { + return Jsoup.parse(showInfo.getString("image_caption")) + .getElementsByTag("a").first().text(); + } + + @Nullable + @Override + public String getTextualUploadDate() { + return showInfo.getString("published_date"); + } + + @Nonnull + @Override + public String getThumbnailUrl() throws ParsingException { + return getImageUrl(showInfo.getLong("show_image_id"), false); + } + + @Nonnull + @Override + public String getUploaderAvatarUrl() { + return "https://bandcamp.com/img/buttons/bandcamp-button-circle-whitecolor-512.png"; + } + + @Nonnull + @Override + public String getDescription() { + return showInfo.getString("desc"); + } + + @Override + public long getLength() { + return showInfo.getLong("audio_duration"); + } + + @Override + public List getAudioStreams() { + ArrayList list = new ArrayList<>(); + JSONObject streams = showInfo.getJSONObject("audio_stream"); + + if (streams.has("opus-lo")) { + list.add(new AudioStream( + streams.getString("opus-lo"), + MediaFormat.OPUS, 100 + )); + } + if (streams.has("mp3-128")) { + list.add(new AudioStream( + streams.getString("mp3-128"), + MediaFormat.MP3, 128 + )); + } + + return list; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java index 3fdcb33ba..0c5cb5fe8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java @@ -2,33 +2,35 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import java.util.List; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL; public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory { - /** - * This kiosk doesn't have a corresponding website - */ - public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data"; - @Override public String getUrl(String id, List contentFilter, String sortFilter) { - if (id.equals(KIOSK_FEATURED)) return FEATURED_API_URL; + if (id.equals(KIOSK_FEATURED)) return FEATURED_API_URL; // doesn't have a website + else if (id.equals(KIOSK_RADIO)) return RADIO_API_URL; // doesn't have its own website else return null; } @Override public String getId(String url) { - return KIOSK_FEATURED; + if (url.matches("https?://bandcamp\\.com/\\?show=\\d+") || url.equals(RADIO_API_URL)) + return KIOSK_RADIO; + else if (url.equals(FEATURED_API_URL)) + return KIOSK_FEATURED; + else return null; } @Override public boolean onAcceptUrl(String url) { - return url.equals(FEATURED_API_URL); + return url.equals(FEATURED_API_URL) || (url.equals(RADIO_API_URL) || url.matches("https?://bandcamp\\.com/\\?show=\\d+")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java index 0536594e1..80a297df8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; /** * Tracks don't have standalone ids, they are always in combination with the band id. * That's why id = url. Instead, URLs are cleaned up so that they always look the same. + *

+ * Radio (bandcamp weekly) shows do have ids. */ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { @@ -17,7 +19,10 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { */ @Override public String getId(String url) throws ParsingException { - return getUrl(url); + if (url.matches("https?://bandcamp\\.com/\\?show=\\d+")) { + return url.split("bandcamp.com/\\?show=")[1]; + } else + return getUrl(url); } /** @@ -25,11 +30,13 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { * @see BandcampStreamLinkHandlerFactory */ @Override - public String getUrl(String url) { - if (url.endsWith("/")) - url = url.substring(0, url.length() - 1); - url = url.replace("http://", "https://").toLowerCase(); - return url; + public String getUrl(String input) { + if (input.matches("\\d+")) + return "https://bandcamp.com/?show=" + input; + if (input.endsWith("/")) + input = input.substring(0, input.length() - 1); + input = input.replace("http://", "https://").toLowerCase(); + return input; } /** @@ -42,6 +49,6 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { */ @Override public boolean onAcceptUrl(String url) { - return getUrl(url).matches("https?://.+\\..+/track/.+"); + return getUrl(url).matches("https?://.+\\..+/track/.+") || getUrl(url).matches("https?://bandcamp\\.com/\\?show=\\d+"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index 258098d33..128841422 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -31,7 +31,7 @@ public class BandcampFeaturedExtractorTest { } @Test - public void testKioskItems() throws ExtractionException, IOException { + public void testFeaturedCount() throws ExtractionException, IOException { List list = extractor.getInitialPage().getItems(); assertTrue(list.size() > 1); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java new file mode 100644 index 000000000..3fd70369d --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java @@ -0,0 +1,52 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory; + +import static org.junit.Assert.*; + +/** + * Tests for {@link BandcampFeaturedLinkHandlerFactory} + */ +public class BandcampFeaturedLinkHandlerFactoryTest { + + private static BandcampFeaturedLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = new BandcampFeaturedLinkHandlerFactory(); + } + + + @Test + public void testAcceptUrl() throws ParsingException { + // Tests expecting true + assertTrue(linkHandler.acceptUrl("https://bandcamp.com/?show=1")); + assertTrue(linkHandler.acceptUrl("http://bandcamp.com/?show=2")); + assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/mobile/24/bootstrap_data")); + assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/bcweekly/1/list")); + + // Tests expecting false + assertFalse(linkHandler.acceptUrl("https://bandcamp.com/?show=")); + assertFalse(linkHandler.acceptUrl("https://bandcamp.com/?show=a")); + assertFalse(linkHandler.acceptUrl("https://bandcamp.com/")); + } + + @Test + public void testGetUrl() throws ParsingException { + assertEquals("https://bandcamp.com/api/mobile/24/bootstrap_data", linkHandler.getUrl("Featured")); + assertEquals("https://bandcamp.com/api/bcweekly/1/list", linkHandler.getUrl("Radio")); + } + + @Test + public void testGetId() { + assertEquals("Featured", linkHandler.getId("https://bandcamp.com/api/mobile/24/bootstrap_data")); + assertEquals("Radio", linkHandler.getId("http://bandcamp.com/?show=1")); + assertEquals("Radio", linkHandler.getId("https://bandcamp.com/api/bcweekly/1/list")); + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java new file mode 100644 index 000000000..4ec3f66a9 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java @@ -0,0 +1,40 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +/** + * Tests for {@link BandcampRadioExtractor} + */ +public class BandcampRadioExtractorTest { + + private static BandcampRadioExtractor extractor; + + @BeforeClass + public static void setUp() throws ExtractionException, IOException { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (BandcampRadioExtractor) bandcamp + .getKioskList() + .getExtractorById("Radio", null); + } + + @Test + public void testRadioCount() throws ExtractionException, IOException { + List list = bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); + System.out.println(list.size()); + assertTrue(list.size() > 300); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java new file mode 100644 index 000000000..9c9350bc3 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.Extractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; + +import java.io.IOException; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +public class BandcampRadioStreamExtractorTest { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testGettingCorrectStreamExtractor() throws ExtractionException { + assertTrue(bandcamp.getStreamExtractor("https://bandcamp.com/?show=3") instanceof BandcampRadioStreamExtractor); + assertFalse(bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/track/deflated") instanceof BandcampRadioStreamExtractor); + } + + @Test + public void testExtracting() throws ExtractionException, IOException { + BandcampRadioStreamExtractor e = (BandcampRadioStreamExtractor) bandcamp.getStreamExtractor("https://bandcamp.com/?show=230"); + e.fetchPage(); + assertEquals("Sound Movements", e.getName()); + assertEquals("Andrew Jervis", e.getUploaderName()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java index ba4c26727..9cbd03295 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -30,6 +30,16 @@ public class BandcampStreamLinkHandlerFactoryTest { assertEquals("https://zachbenson.bandcamp.com/track/u-i-tonite", linkHandler.getUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); } + @Test + public void testGetRadioUrl() { + assertEquals("https://bandcamp.com/?show=1", linkHandler.getUrl("1")); + } + + @Test + public void testGetRadioId() throws ParsingException { + assertEquals("2", linkHandler.getId("https://bandcamp.com/?show=2")); + } + @Test public void testAcceptUrl() throws ParsingException { // Tests expecting false @@ -44,5 +54,6 @@ public class BandcampStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); assertTrue(linkHandler.acceptUrl("https://interovgm.com/track/title")); + assertTrue(linkHandler.acceptUrl("http://bandcamP.com/?show=38")); } } From f0d36dfa7eb1b35b99c6c593d4e6e247868e09fe Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 22 Dec 2019 17:36:50 +0100 Subject: [PATCH 21/91] Bandcamp radio: proper name, date shortened --- .../extractors/BandcampRadioInfoItemExtractor.java | 8 ++------ .../bandcamp/extractors/BandcampRadioStreamExtractor.java | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index 7e7017140..2aeb79af3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -3,18 +3,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import org.json.JSONObject; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; -import java.io.IOException; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor.query; public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @@ -37,7 +33,7 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public String getTextualUploadDate() { - return show.getString("date"); + return show.getString("date").replace(" 00:00:00 GMT", ""); } @Nullable @@ -48,7 +44,7 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getName() throws ParsingException { - return show.getString("date"); + return show.getString("subtitle"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 2449f572f..82bd0d287 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Jsoup; import org.schabi.newpipe.extractor.MediaFormat; @@ -11,7 +10,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.AudioStream; import javax.annotation.Nonnull; @@ -76,7 +74,7 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nullable @Override public String getTextualUploadDate() { - return showInfo.getString("published_date"); + return showInfo.getString("published_date").replace(" 00:00:00 GMT", ""); } @Nonnull From 46e1f3922c8869db7bef79573c5f86dde3ab9b4f Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Fri, 3 Jan 2020 12:25:59 +0100 Subject: [PATCH 22/91] Refuse to load bandcamp playlists without content --- .../services/bandcamp/extractors/BandcampPlaylistExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index e24e603de..1886d4a92 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -49,7 +49,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { - if (trackInfo.length() < 0) { + if (trackInfo.length() <= 0) { // Albums without trackInfo need to be purchased before they can be played throw new ContentNotAvailableException("Album needs to be purchased"); } From ba967f1a1520f7fadded3b6bfd67c4da0f188c2b Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Fri, 3 Jan 2020 13:13:42 +0100 Subject: [PATCH 23/91] Workaround enourmous load times for long bandcamp playlists Additionally, get cover art from json instead of html --- .../extractors/BandcampPlaylistExtractor.java | 42 +++++++++++++------ .../extractors/BandcampStreamExtractor.java | 9 ++-- .../BandcampPlaylistExtractorTest.java | 16 +++++++ .../bandcamp/BandcampRadioExtractorTest.java | 1 - 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 1886d4a92..64f26b2e9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -18,11 +18,19 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getJSONFromJavaScriptVariables; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; public class BandcampPlaylistExtractor extends PlaylistExtractor { + /** + * An arbitrarily chosen number above which cover arts won't be fetched individually for each track; + * instead, it will be assumed that every track has the same cover art as the album, which is not + * always the case. + */ + private static final int MAXIMUM_INDIVIDUAL_COVER_ARTS = 10; + private Document document; private JSONObject albumJson; private JSONArray trackInfo; @@ -57,11 +65,8 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { - try { - return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); - } catch (NullPointerException e) { - return ""; - } + if (albumJson.isNull("art_id")) return ""; + else return getImageUrl(albumJson.getLong("art_id"), true); } @Override @@ -104,13 +109,26 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { for (int i = 0; i < trackInfo.length(); i++) { JSONObject track = trackInfo.getJSONObject(i); - collector.commit(new BandcampStreamInfoItemExtractor( - track.getString("title"), - getUploaderUrl() + track.getString("title_link"), - "", - track.getLong("duration"), - getService() - )); + if (trackInfo.length() < MAXIMUM_INDIVIDUAL_COVER_ARTS) { + // Load cover art of every track individually + collector.commit(new BandcampStreamInfoItemExtractor( + track.getString("title"), + getUploaderUrl() + track.getString("title_link"), + "", + track.getLong("duration"), + getService() + )); + } else { + // Pretend every track has the same cover art as the album + collector.commit(new BandcampStreamInfoItemExtractor( + track.getString("title"), + getUploaderUrl() + track.getString("title_link"), + getThumbnailUrl(), + "", + track.getLong("duration") + )); + } + } return new InfoItemsPage<>(collector, null); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 72c3a551c..c76cb474b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; + public class BandcampStreamExtractor extends StreamExtractor { private JSONObject albumJson; @@ -103,11 +105,8 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public String getThumbnailUrl() throws ParsingException { - try { - return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); - } catch (NullPointerException e) { - return ""; - } + if (albumJson.isNull("art_id")) return ""; + else return getImageUrl(albumJson.getLong("art_id"), true); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index 7400b1821..7d930ca8f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -52,6 +52,22 @@ public class BandcampPlaylistExtractorTest { assertNotEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); } + /** + * Tests that no attempt to load every track's cover individually is made + */ + @Test(timeout = 10000L) + public void testDifferentTrackCoversDuration() throws ExtractionException, IOException { + PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything"); + extractor.fetchPage(); + + /* All tracks in this album have the same cover art, but I don't know any albums with more than 10 tracks + * that has at least one track with a cover art different from the rest. + */ + List l = extractor.getInitialPage().getItems(); + assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); + assertEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); + } + /** * Test playlists with locked content */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java index 4ec3f66a9..03ea7e3b1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java @@ -34,7 +34,6 @@ public class BandcampRadioExtractorTest { @Test public void testRadioCount() throws ExtractionException, IOException { List list = bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); - System.out.println(list.size()); assertTrue(list.size() > 300); } } From 7b5702bdc2a84a37f158b41bbfe7f708507ea10d Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sat, 11 Jan 2020 16:32:45 +0100 Subject: [PATCH 24/91] Add bandcamp to list of supported services in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3cf4e4c5b..9bbdb826a 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ The following sites are currently supported: - YouTube - SoundCloud - MediaCCC +- bandcamp ## License From a1523eb293acbf074b51972b272b901f7bc9144f Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Tue, 17 Mar 2020 20:40:25 +0100 Subject: [PATCH 25/91] Update Bandcamp service to latest interface changes --- .../BandcampRadioStreamExtractor.java | 23 ++++- .../extractors/BandcampStreamExtractor.java | 87 +++++++++++++++++-- .../bandcamp/BandcampStreamExtractorTest.java | 20 ++++- 3 files changed, 122 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 82bd0d287..3c1148c7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -91,8 +92,8 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nonnull @Override - public String getDescription() { - return showInfo.getString("desc"); + public Description getDescription() { + return new Description(showInfo.getString("desc"), Description.PLAIN_TEXT); } @Override @@ -120,4 +121,22 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { return list; } + + @Nonnull + @Override + public String getLicence() { + return ""; + } + + @Nonnull + @Override + public String getCategory() { + return ""; + } + + @Nonnull + @Override + public List getTags() { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index c76cb474b..2300809d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -6,6 +6,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -20,6 +22,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; @@ -121,14 +124,15 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDescription() { - return BandcampExtractorHelper.smartConcatenate( + public Description getDescription() { + String s = BandcampExtractorHelper.smartConcatenate( new String[]{ getStringOrNull(current, "about"), getStringOrNull(current, "lyrics"), getStringOrNull(current, "credits") }, "\n\n" ); + return new Description(s, Description.PLAIN_TEXT); } /** @@ -176,7 +180,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public String getDashMpdUrl() throws ParsingException { - return null; + return ""; } @Nonnull @@ -210,13 +214,13 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public List getSubtitlesDefault() { - return null; + return new ArrayList<>(); } @Nonnull @Override public List getSubtitles(MediaFormat format) { - return null; + return new ArrayList<>(); } @Override @@ -238,4 +242,77 @@ public class BandcampStreamExtractor extends StreamExtractor { public String getErrorMessage() { return null; } + + @Nonnull + @Override + public String getHost() { + return ""; + } + + @Nonnull + @Override + public String getPrivacy() { + return ""; + } + + @Nonnull + @Override + public String getCategory() { + // Get first tag from html, which is the artist's Genre + return document.getElementsByAttributeValue("itemprop", "keywords").first().text(); + } + + @Nonnull + @Override + public String getLicence() { + + int license = current.getInt("license_type"); + + // Tests resulted in this mapping of ints to licence: https://cloud.disroot.org/s/ZTWBxbQ9fKRmRWJ/preview + + switch (license) { + case 1: + return "All rights reserved ยฉ"; + case 2: + return "CC BY-NC-ND 3.0"; + case 3: + return "CC BY-NC-SA 3.0"; + case 4: + return "CC BY-NC 3.0"; + case 5: + return "CC BY-ND 3.0"; + case 8: + return "CC BY-SA 3.0"; + case 6: + return "CC BY 3.0"; + default: + return "Unknown license (internal ID " + license + ")"; + } + } + + @Nullable + @Override + public Locale getLanguageInfo() { + return null; + } + + @Nonnull + @Override + public List getTags() { + Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords"); + + ArrayList tags = new ArrayList<>(); + + for (Element e : tagElements) { + tags.add(e.text()); + } + + return tags; + } + + @Nonnull + @Override + public String getSupportInfo() { + return ""; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 2f7a03bff..5574c3408 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -9,6 +9,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamExtractor; + +import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -52,7 +55,7 @@ public class BandcampStreamExtractorTest { @Test public void testDescription() { - assertEquals(831, extractor.getDescription().length()); + assertEquals(831, extractor.getDescription().getContent().length()); } @Test @@ -77,4 +80,19 @@ public class BandcampStreamExtractorTest { bandcamp.getStreamExtractor("https://bandcamp.com"); } + @Test + public void testCategory() throws ExtractionException, IOException { + StreamExtractor se = bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); + se.fetchPage(); + assertEquals("acoustic", se.getCategory()); + } + + @Test + public void testLicense() throws ExtractionException, IOException { + StreamExtractor se = bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); + se.fetchPage(); + assertEquals("CC BY 3.0", se.getLicence()); + } + + } From 9c239371f4cb8b781827a06eaa827e0b328f89b4 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 19 Apr 2020 14:29:39 +0200 Subject: [PATCH 26/91] Change nanojson dependency to @wb9688's fork --- build.gradle | 1 + extractor/build.gradle | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a62598060..6f349545f 100644 --- a/build.gradle +++ b/build.gradle @@ -10,6 +10,7 @@ allprojects { repositories { jcenter() + maven { url "https://jitpack.io" } } } diff --git a/extractor/build.gradle b/extractor/build.gradle index cc2f69dab..062861a0e 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -1,7 +1,9 @@ + + dependencies { implementation project(':timeago-parser') - implementation 'com.grack:nanojson:1.1' + implementation 'com.github.wb9688:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' implementation 'org.jsoup:jsoup:1.9.2' implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' From b100b9873ff1e6177ad53510ad4b39dd92a78ff7 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 19 Mar 2020 11:18:29 +0100 Subject: [PATCH 27/91] Bandcmap: Use nanojson fork instead of org.json --- extractor/build.gradle | 5 +- .../extractors/BandcampChannelExtractor.java | 31 +++++----- .../extractors/BandcampExtractorHelper.java | 15 ++--- .../extractors/BandcampFeaturedExtractor.java | 57 +++++++++++-------- .../extractors/BandcampPlaylistExtractor.java | 24 ++++---- .../extractors/BandcampRadioExtractor.java | 38 ++++++++----- .../BandcampRadioInfoItemExtractor.java | 6 +- .../BandcampRadioStreamExtractor.java | 14 +++-- .../extractors/BandcampStreamExtractor.java | 38 +++++-------- .../BandcampSuggestionExtractor.java | 36 +++++++----- .../BandcampChannelLinkHandlerFactory.java | 10 ++-- 11 files changed, 142 insertions(+), 132 deletions(-) diff --git a/extractor/build.gradle b/extractor/build.gradle index 062861a0e..7372288a2 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -1,5 +1,3 @@ - - dependencies { implementation project(':timeago-parser') @@ -8,7 +6,6 @@ dependencies { implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' implementation 'org.nibor.autolink:autolink:0.8.0' - implementation 'org.json:json:20190722' testImplementation 'junit:junit:4.12' -} \ No newline at end of file +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index ddb8f3365..9c1572cd4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -2,11 +2,11 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; -import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -23,7 +23,7 @@ import java.io.IOException; public class BandcampChannelExtractor extends ChannelExtractor { - private JSONObject channelInfo; + private JsonObject channelInfo; public BandcampChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { super(service, linkHandler); @@ -36,17 +36,17 @@ public class BandcampChannelExtractor extends ChannelExtractor { * * I once took a moment to note down how it works. */ - public static JSONObject getArtistDetails(String id) throws ParsingException { + public static JsonObject getArtistDetails(String id) throws ParsingException { try { return - new JSONObject( + JsonParser.object().from( NewPipe.getDownloader().post( "https://bandcamp.com/api/mobile/22/band_details", null, ("{\"band_id\":\"" + id + "\"}").getBytes() ).responseBody() ); - } catch (IOException | ReCaptchaException e) { + } catch (IOException | ReCaptchaException | JsonParserException e) { throw new ParsingException("Could not download band details", e); } } @@ -63,12 +63,9 @@ public class BandcampChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() { - try { - return getImageUrl(channelInfo.getLong("bio_image_id"), false); - } catch (JSONException e) { - // In this case, the id is null and no image is available - return ""; - } + if (channelInfo.getLong("bio_image_id") == 0) return ""; + + return getImageUrl(channelInfo.getLong("bio_image_id"), false); } /** @@ -120,11 +117,11 @@ public class BandcampChannelExtractor extends ChannelExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JSONArray discography = channelInfo.getJSONArray("discography"); + JsonArray discography = channelInfo.getArray("discography"); - for (int i = 0; i < discography.length(); i++) { + for (int i = 0; i < discography.size(); i++) { // I define discograph as an item that can appear in a discography - JSONObject discograph = discography.getJSONObject(i); + JsonObject discograph = discography.getObject(i); if (!discograph.getString("item_type").equals("track")) continue; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index b86504c62..39c7d1fdd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -2,8 +2,9 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONException; -import org.json.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -24,7 +25,7 @@ public class BandcampExtractorHelper { * @param variable Name of the variable * @return The JsonObject stored in the variable with this name */ - public static JSONObject getJSONFromJavaScriptVariables(String html, String variable) throws JSONException, ArrayIndexOutOfBoundsException, ParsingException { + public static JsonObject getJSONFromJavaScriptVariables(String html, String variable) throws JsonParserException, ArrayIndexOutOfBoundsException, ParsingException { String[] part = html.split("var " + variable + " = "); @@ -44,7 +45,7 @@ public class BandcampExtractorHelper { case '}': level--; if (level == 0) { - return new JSONObject(firstHalfGone.substring(0, position + 1) + return JsonParser.object().from(firstHalfGone.substring(0, position + 1) .replaceAll(" {4}//.+", "") // Remove "for the curious" in JSON .replaceAll("// xxx: note - don't internationalize this variable", "") // Remove this comment ); @@ -62,14 +63,14 @@ public class BandcampExtractorHelper { public static String getStreamUrlFromIds(long bandId, long itemId, String itemType) throws ParsingException { try { - String html = NewPipe.getDownloader().get( + String jsonString = NewPipe.getDownloader().get( "https://bandcamp.com/api/mobile/22/tralbum_details?band_id=" + bandId + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.substring(0, 1)) .responseBody(); - return new JSONObject(html).getString("bandcamp_url").replace("http://", "https://"); + return JsonParser.object().from(jsonString).getString("bandcamp_url").replace("http://", "https://"); - } catch (JSONException | ReCaptchaException | IOException e) { + } catch (JsonParserException | ReCaptchaException | IOException e) { throw new ParsingException("Ids could not be translated to URL", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index f8e67c6bd..37bfd1aea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -2,8 +2,10 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONArray; -import org.json.JSONObject; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Collector; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; @@ -49,34 +51,41 @@ public class BandcampFeaturedExtractor extends KioskExtractor { InfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); - JSONObject json = new JSONObject( - getDownloader().post( - FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes() - ).responseBody() - ); + try { - JSONArray featuredStories = json.getJSONObject("feed_content") - .getJSONObject("stories") - .getJSONArray("featured"); - for (int i = 0; i < featuredStories.length(); i++) { - JSONObject featuredStory = featuredStories.getJSONObject(i); + JsonObject json = JsonParser.object().from( + getDownloader().post( + FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes() + ).responseBody() + ); - if (featuredStory.isNull("album_title")) { - // Is not an album, ignore - continue; + JsonArray featuredStories = json.getObject("feed_content") + .getObject("stories") + .getArray("featured"); + + for (int i = 0; i < featuredStories.size(); i++) { + JsonObject featuredStory = featuredStories.getObject(i); + + if (featuredStory.isNull("album_title")) { + // Is not an album, ignore + continue; + } + + c.commit(new BandcampPlaylistInfoItemExtractor( + featuredStory.getString("album_title"), + featuredStory.getString("band_name"), + featuredStory.getString("item_url"), + featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "", + featuredStory.getInt("num_streamable_tracks") + )); } - c.commit(new BandcampPlaylistInfoItemExtractor( - featuredStory.getString("album_title"), - featuredStory.getString("band_name"), - featuredStory.getString("item_url"), - featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "", - featuredStory.getInt("num_streamable_tracks") - )); + return new InfoItemsPage(c, null); + } catch (JsonParserException e) { + e.printStackTrace(); + throw new ParsingException("JSON error", e); } - - return new InfoItemsPage(c, null); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 64f26b2e9..ddb78d3fa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -1,8 +1,8 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.StreamingService; @@ -32,8 +32,8 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { private static final int MAXIMUM_INDIVIDUAL_COVER_ARTS = 10; private Document document; - private JSONObject albumJson; - private JSONArray trackInfo; + private JsonObject albumJson; + private JsonArray trackInfo; private String name; public BandcampPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -45,11 +45,11 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { String html = downloader.get(getLinkHandler().getUrl()).responseBody(); document = Jsoup.parse(html); albumJson = getAlbumInfoJson(html); - trackInfo = albumJson.getJSONArray("trackinfo"); + trackInfo = albumJson.getArray("trackinfo"); try { name = getJSONFromJavaScriptVariables(html, "EmbedData").getString("album_title"); - } catch (JSONException e) { + } catch (JsonParserException e) { throw new ParsingException("Faulty JSON; page likely does not contain album data", e); } catch (ArrayIndexOutOfBoundsException e) { throw new ParsingException("JSON does not exist", e); @@ -57,7 +57,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { - if (trackInfo.length() <= 0) { + if (trackInfo.size() <= 0) { // Albums without trackInfo need to be purchased before they can be played throw new ContentNotAvailableException("Album needs to be purchased"); } @@ -97,7 +97,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { @Override public long getStreamCount() { - return trackInfo.length(); + return trackInfo.size(); } @Nonnull @@ -106,10 +106,10 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - for (int i = 0; i < trackInfo.length(); i++) { - JSONObject track = trackInfo.getJSONObject(i); + for (int i = 0; i < trackInfo.size(); i++) { + JsonObject track = trackInfo.getObject(i); - if (trackInfo.length() < MAXIMUM_INDIVIDUAL_COVER_ARTS) { + if (trackInfo.size() < MAXIMUM_INDIVIDUAL_COVER_ARTS) { // Load cover art of every track individually collector.commit(new BandcampStreamInfoItemExtractor( track.getString("title"), diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java index 1e2f8aa7f..e7ff3417a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java @@ -2,8 +2,10 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONArray; -import org.json.JSONObject; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.StreamingService; @@ -12,7 +14,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; @@ -43,20 +45,26 @@ public class BandcampRadioExtractor extends KioskExtractor { public InfoItemsPage getInitialPage() throws IOException, ExtractionException { InfoItemsCollector c = new StreamInfoItemsCollector(getServiceId()); - JSONObject json = new JSONObject( - getDownloader().get( - RADIO_API_URL - ).responseBody() - ); + try { - JSONArray radioShows = json.getJSONArray("results"); - - for (int i = 0; i < radioShows.length(); i++) { - JSONObject radioShow = radioShows.getJSONObject(i); - - c.commit( - new BandcampRadioInfoItemExtractor(radioShow) + JsonObject json = JsonParser.object().from( + getDownloader().get( + RADIO_API_URL + ).responseBody() ); + + JsonArray radioShows = json.getArray("results"); + + for (int i = 0; i < radioShows.size(); i++) { + JsonObject radioShow = radioShows.getObject(i); + + c.commit( + new BandcampRadioInfoItemExtractor(radioShow) + ); + } + + } catch (JsonParserException e) { + e.printStackTrace(); } return new InfoItemsPage(c, null); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index 2aeb79af3..adae3e557 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONObject; +import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; @@ -14,9 +14,9 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { - private JSONObject show; + private JsonObject show; - public BandcampRadioInfoItemExtractor(JSONObject radioShow) { + public BandcampRadioInfoItemExtractor(JsonObject radioShow) { show = radioShow; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 3c1148c7c..98518b442 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; @@ -23,7 +25,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { - private JSONObject showInfo; + private JsonObject showInfo; private LinkHandler linkHandler; public BandcampRadioStreamExtractor(StreamingService service, LinkHandler linkHandler) { @@ -31,12 +33,12 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { this.linkHandler = linkHandler; } - static JSONObject query(int id) throws ParsingException { + static JsonObject query(int id) throws ParsingException { try { - return new JSONObject( + return JsonParser.object().from( NewPipe.getDownloader().get("https://bandcamp.com/api/bcweekly/1/get?id=" + id).responseBody() ); - } catch (IOException | ReCaptchaException e) { + } catch (IOException | ReCaptchaException | JsonParserException e) { throw new ParsingException("could not get show data", e); } } @@ -104,7 +106,7 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Override public List getAudioStreams() { ArrayList list = new ArrayList<>(); - JSONObject streams = showInfo.getJSONObject("audio_stream"); + JsonObject streams = showInfo.getObject("audio_stream"); if (streams.has("opus-lo")) { list.add(new AudioStream( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 2300809d7..3785f8a13 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -2,8 +2,8 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONException; -import org.json.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -28,8 +28,8 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp public class BandcampStreamExtractor extends StreamExtractor { - private JSONObject albumJson; - private JSONObject current; + private JsonObject albumJson; + private JsonObject current; private Document document; public BandcampStreamExtractor(StreamingService service, LinkHandler linkHandler) { @@ -42,9 +42,9 @@ public class BandcampStreamExtractor extends StreamExtractor { String html = downloader.get(getLinkHandler().getUrl()).responseBody(); document = Jsoup.parse(html); albumJson = getAlbumInfoJson(html); - current = albumJson.getJSONObject("current"); + current = albumJson.getObject("current"); - if (albumJson.getJSONArray("trackinfo").length() > 1) { + if (albumJson.getArray("trackinfo").size() > 1) { // In this case, we are actually viewing an album page! throw new ExtractionException("Page is actually an album, not a track"); } @@ -57,10 +57,10 @@ public class BandcampStreamExtractor extends StreamExtractor { * @return Album metadata JSON * @throws ParsingException In case of a faulty website */ - public static JSONObject getAlbumInfoJson(String html) throws ParsingException { + public static JsonObject getAlbumInfoJson(String html) throws ParsingException { try { return BandcampExtractorHelper.getJSONFromJavaScriptVariables(html, "TralbumData"); - } catch (JSONException e) { + } catch (JsonParserException e) { throw new ParsingException("Faulty JSON; page likely does not contain album data", e); } catch (ArrayIndexOutOfBoundsException e) { throw new ParsingException("JSON does not exist", e); @@ -127,26 +127,14 @@ public class BandcampStreamExtractor extends StreamExtractor { public Description getDescription() { String s = BandcampExtractorHelper.smartConcatenate( new String[]{ - getStringOrNull(current, "about"), - getStringOrNull(current, "lyrics"), - getStringOrNull(current, "credits") + current.getString("about"), + current.getString("lyrics"), + current.getString("credits") }, "\n\n" ); return new Description(s, Description.PLAIN_TEXT); } - /** - * Avoid exceptions like "JSONObject["about"] not a string." and instead just return null. - * This is for the case that the actual JSON has something like "about": null. - */ - private String getStringOrNull(JSONObject jsonObject, String value) { - try { - return jsonObject.getString(value); - } catch (JSONException e) { - return null; - } - } - @Override public int getAgeLimit() throws ParsingException { return 0; @@ -194,8 +182,8 @@ public class BandcampStreamExtractor extends StreamExtractor { List audioStreams = new ArrayList<>(); audioStreams.add(new AudioStream( - albumJson.getJSONArray("trackinfo").getJSONObject(0) - .getJSONObject("file").getString("mp3-128"), + albumJson.getArray("trackinfo").getObject(0) + .getObject("file").getString("mp3-128"), MediaFormat.MP3, 128 )); return audioStreams; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index 6cf5d70b2..4a819ed40 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -2,8 +2,10 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.json.JSONArray; -import org.json.JSONObject; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -28,24 +30,30 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { public List suggestionList(String query) throws IOException, ExtractionException { Downloader downloader = NewPipe.getDownloader(); - JSONObject fuzzyResults = new JSONObject( - downloader.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, CHARSET_UTF_8)).responseBody() - ); + try { + JsonObject fuzzyResults = JsonParser.object().from( + downloader.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, CHARSET_UTF_8)).responseBody() + ); - JSONArray jsonArray = fuzzyResults.getJSONObject("auto") - .getJSONArray("results"); + JsonArray jsonArray = fuzzyResults.getObject("auto") + .getArray("results"); - ArrayList suggestions = new ArrayList<>(); + ArrayList suggestions = new ArrayList<>(); - for (int i = 0; i < jsonArray.length(); i++) { - JSONObject fuzzyResult = jsonArray.getJSONObject(i); - String res = fuzzyResult.getString("name"); + for (int i = 0; i < jsonArray.size(); i++) { + JsonObject fuzzyResult = jsonArray.getObject(i); + String res = fuzzyResult.getString("name"); - if (!suggestions.contains(res)) suggestions.add(res); + if (!suggestions.contains(res)) suggestions.add(res); + } + + return suggestions; + } catch (JsonParserException e) { + e.printStackTrace(); + + return new ArrayList<>(); } - - return suggestions; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index 0d57eac01..94b45771e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -2,8 +2,8 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; -import org.json.JSONException; -import org.json.JSONObject; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -26,11 +26,11 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { String response = NewPipe.getDownloader().get(url).responseBody(); // This variable contains band data! - JSONObject bandData = BandcampExtractorHelper.getJSONFromJavaScriptVariables(response, "BandData"); + JsonObject bandData = BandcampExtractorHelper.getJSONFromJavaScriptVariables(response, "BandData"); return String.valueOf(bandData.getLong("id")); - } catch (IOException | ReCaptchaException | ArrayIndexOutOfBoundsException e) { + } catch (IOException | ReCaptchaException | ArrayIndexOutOfBoundsException | JsonParserException e) { throw new ParsingException("Download failed", e); } } @@ -44,7 +44,7 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { return BandcampChannelExtractor.getArtistDetails(id) .getString("bandcamp_url") .replace("http://", "https://"); - } catch (JSONException e) { + } catch (NullPointerException e) { throw new ParsingException("JSON does not contain URL (invalid id?) or is otherwise invalid", e); } From c133190c53ab9df70d3a809fd94c6e59dbba73fc Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 21:55:35 +0200 Subject: [PATCH 28/91] Bandcamp capitalization --- README.md | 2 +- .../org/schabi/newpipe/extractor/ServiceList.java | 4 ++-- .../bandcamp/BandcampChannelExtractorTest.java | 8 ++++---- .../bandcamp/BandcampFeaturedExtractorTest.java | 4 ++-- .../bandcamp/BandcampPlaylistExtractorTest.java | 12 ++++++------ .../bandcamp/BandcampRadioExtractorTest.java | 6 +++--- .../bandcamp/BandcampRadioStreamExtractorTest.java | 8 ++++---- .../bandcamp/BandcampSearchExtractorTest.java | 10 +++++----- .../BandcampSearchQueryHandlerFactoryTest.java | 4 ++-- .../bandcamp/BandcampStreamExtractorTest.java | 14 +++++++------- .../bandcamp/BandcampSuggestionExtractorTest.java | 4 ++-- 11 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 2efee965a..368886dfb 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ The following sites are currently supported: - SoundCloud - MediaCCC - PeerTube (no P2P) -- bandcamp +- Bandcamp ## License diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index d91083bfd..5bfc5692f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -40,7 +40,7 @@ public final class ServiceList { public static final SoundcloudService SoundCloud; public static final MediaCCCService MediaCCC; public static final PeertubeService PeerTube; - public static final BandcampService bandcamp; + public static final BandcampService Bandcamp; /** * When creating a new service, put this service in the end of this list, @@ -52,7 +52,7 @@ public final class ServiceList { SoundCloud = new SoundcloudService(1), MediaCCC = new MediaCCCService(2), PeerTube = new PeertubeService(3), - bandcamp = new BandcampService(4) + Bandcamp = new BandcampService(4) )); /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 850529774..6aea55939 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -13,7 +13,7 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtract import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; public class BandcampChannelExtractorTest { @@ -22,7 +22,7 @@ public class BandcampChannelExtractorTest { @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (BandcampChannelExtractor) bandcamp + extractor = (BandcampChannelExtractor) Bandcamp .getChannelExtractor("https://zachbenson.bandcamp.com/"); } @@ -51,11 +51,11 @@ public class BandcampChannelExtractorTest { @Test public void testGetNoAvatar() throws ExtractionException { - assertEquals("", bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/").getAvatarUrl()); + assertEquals("", Bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/").getAvatarUrl()); } @Test public void testGetNoBanner() throws ExtractionException { - assertEquals("", bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/").getBannerUrl()); + assertEquals("", Bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/").getBannerUrl()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index 128841422..e02e45932 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -14,7 +14,7 @@ import java.io.IOException; import java.util.List; import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** * Tests for {@link BandcampFeaturedExtractor} @@ -26,7 +26,7 @@ public class BandcampFeaturedExtractorTest { @BeforeClass public static void setUp() throws ExtractionException, IOException { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (BandcampFeaturedExtractor) bandcamp + extractor = (BandcampFeaturedExtractor) Bandcamp .getKioskList().getDefaultKioskExtractor(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index 7d930ca8f..fc2f434cd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -16,7 +16,7 @@ import java.io.IOException; import java.util.List; import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** * Tests for {@link BandcampPlaylistExtractor} @@ -33,7 +33,7 @@ public class BandcampPlaylistExtractorTest { */ @Test public void testCount() throws ExtractionException, IOException { - PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); + PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); extractor.fetchPage(); assertEquals(5, extractor.getStreamCount()); @@ -44,7 +44,7 @@ public class BandcampPlaylistExtractorTest { */ @Test public void testDifferentTrackCovers() throws ExtractionException, IOException { - PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://zachbensonarchive.bandcamp.com/album/results-of-boredom"); + PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachbensonarchive.bandcamp.com/album/results-of-boredom"); extractor.fetchPage(); List l = extractor.getInitialPage().getItems(); @@ -57,7 +57,7 @@ public class BandcampPlaylistExtractorTest { */ @Test(timeout = 10000L) public void testDifferentTrackCoversDuration() throws ExtractionException, IOException { - PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything"); + PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything"); extractor.fetchPage(); /* All tracks in this album have the same cover art, but I don't know any albums with more than 10 tracks @@ -73,7 +73,7 @@ public class BandcampPlaylistExtractorTest { */ @Test(expected = ContentNotAvailableException.class) public void testLockedContent() throws ExtractionException, IOException { - PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://billwurtz.bandcamp.com/album/high-enough"); + PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://billwurtz.bandcamp.com/album/high-enough"); extractor.fetchPage(); } @@ -82,7 +82,7 @@ public class BandcampPlaylistExtractorTest { */ @Test public void testSingleStreamPlaylist() throws ExtractionException, IOException { - PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://zachjohnson1.bandcamp.com/album/endless"); + PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachjohnson1.bandcamp.com/album/endless"); extractor.fetchPage(); assertEquals(1, extractor.getStreamCount()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java index 03ea7e3b1..c1f493c92 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java @@ -14,7 +14,7 @@ import java.io.IOException; import java.util.List; import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** * Tests for {@link BandcampRadioExtractor} @@ -26,14 +26,14 @@ public class BandcampRadioExtractorTest { @BeforeClass public static void setUp() throws ExtractionException, IOException { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (BandcampRadioExtractor) bandcamp + extractor = (BandcampRadioExtractor) Bandcamp .getKioskList() .getExtractorById("Radio", null); } @Test public void testRadioCount() throws ExtractionException, IOException { - List list = bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); + List list = Bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); assertTrue(list.size() > 300); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 9c9350bc3..760a833f7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -12,7 +12,7 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamE import java.io.IOException; import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; public class BandcampRadioStreamExtractorTest { @@ -23,13 +23,13 @@ public class BandcampRadioStreamExtractorTest { @Test public void testGettingCorrectStreamExtractor() throws ExtractionException { - assertTrue(bandcamp.getStreamExtractor("https://bandcamp.com/?show=3") instanceof BandcampRadioStreamExtractor); - assertFalse(bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/track/deflated") instanceof BandcampRadioStreamExtractor); + assertTrue(Bandcamp.getStreamExtractor("https://bandcamp.com/?show=3") instanceof BandcampRadioStreamExtractor); + assertFalse(Bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/track/deflated") instanceof BandcampRadioStreamExtractor); } @Test public void testExtracting() throws ExtractionException, IOException { - BandcampRadioStreamExtractor e = (BandcampRadioStreamExtractor) bandcamp.getStreamExtractor("https://bandcamp.com/?show=230"); + BandcampRadioStreamExtractor e = (BandcampRadioStreamExtractor) Bandcamp.getStreamExtractor("https://bandcamp.com/?show=230"); e.fetchPage(); assertEquals("Sound Movements", e.getName()); assertEquals("Andrew Jervis", e.getUploaderName()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 1e6f9b579..ba8c147b5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -20,7 +20,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** * Test for {@link BandcampSearchExtractor} @@ -41,7 +41,7 @@ public class BandcampSearchExtractorTest { */ @Test public void testStreamSearch() throws ExtractionException, IOException { - SearchExtractor extractor = bandcamp.getSearchExtractor("best friend's basement"); + SearchExtractor extractor = Bandcamp.getSearchExtractor("best friend's basement"); ListExtractor.InfoItemsPage page = extractor.getInitialPage(); InfoItem bestFriendsBasement = page.getItems().get(0); @@ -58,7 +58,7 @@ public class BandcampSearchExtractorTest { */ @Test public void testChannelSearch() throws ExtractionException, IOException { - SearchExtractor extractor = bandcamp.getSearchExtractor("C418"); + SearchExtractor extractor = Bandcamp.getSearchExtractor("C418"); InfoItem c418 = extractor.getInitialPage() .getItems().get(0); @@ -75,7 +75,7 @@ public class BandcampSearchExtractorTest { */ @Test public void testAlbumSearch() throws ExtractionException, IOException { - SearchExtractor extractor = bandcamp.getSearchExtractor("minecraft volume alpha"); + SearchExtractor extractor = Bandcamp.getSearchExtractor("minecraft volume alpha"); InfoItem minecraft = extractor.getInitialPage() .getItems().get(0); @@ -96,7 +96,7 @@ public class BandcampSearchExtractorTest { @Test public void testMultiplePages() throws ExtractionException, IOException { // A query practically guaranteed to have the maximum amount of pages - SearchExtractor extractor = bandcamp.getSearchExtractor("e"); + SearchExtractor extractor = Bandcamp.getSearchExtractor("e"); assertEquals("https://bandcamp.com/search?q=e&page=2", extractor.getInitialPage().getNextPageUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java index 4158e655b..4271bc97c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; import static org.junit.Assert.assertEquals; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; public class BandcampSearchQueryHandlerFactoryTest { @@ -20,7 +20,7 @@ public class BandcampSearchQueryHandlerFactoryTest { public static void setUp() { NewPipe.init(DownloaderTestImpl.getInstance()); - searchQuery = (BandcampSearchQueryHandlerFactory) bandcamp + searchQuery = (BandcampSearchQueryHandlerFactory) Bandcamp .getSearchQHFactory(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 5574c3408..1e5bd7eb9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -15,7 +15,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; public class BandcampStreamExtractorTest { @@ -24,14 +24,14 @@ public class BandcampStreamExtractorTest { @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (BandcampStreamExtractor) bandcamp + extractor = (BandcampStreamExtractor) Bandcamp .getStreamExtractor("https://zachbenson.bandcamp.com/track/kitchen"); extractor.fetchPage(); } @Test(expected = ExtractionException.class) public void testAlbum() throws ExtractionException { - bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/album/prom"); + Bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/album/prom"); } @Test @@ -66,7 +66,7 @@ public class BandcampStreamExtractorTest { @Test public void testNoArtistProfilePicture() throws ExtractionException { - assertEquals("", bandcamp.getStreamExtractor("https://powertothequeerkids.bandcamp.com/track/human-nature").getUploaderAvatarUrl()); + assertEquals("", Bandcamp.getStreamExtractor("https://powertothequeerkids.bandcamp.com/track/human-nature").getUploaderAvatarUrl()); } @Test @@ -77,19 +77,19 @@ public class BandcampStreamExtractorTest { @Test(expected = ParsingException.class) public void testInvalidUrl() throws ExtractionException { - bandcamp.getStreamExtractor("https://bandcamp.com"); + Bandcamp.getStreamExtractor("https://bandcamp.com"); } @Test public void testCategory() throws ExtractionException, IOException { - StreamExtractor se = bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); + StreamExtractor se = Bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); se.fetchPage(); assertEquals("acoustic", se.getCategory()); } @Test public void testLicense() throws ExtractionException, IOException { - StreamExtractor se = bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); + StreamExtractor se = Bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); se.fetchPage(); assertEquals("CC BY 3.0", se.getLicence()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java index 77b4f26b5..68cdae518 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java @@ -13,7 +13,7 @@ import java.io.IOException; import java.util.List; import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.bandcamp; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** * Tests for {@link BandcampSuggestionExtractor} @@ -25,7 +25,7 @@ public class BandcampSuggestionExtractorTest { @BeforeClass public static void setUp() { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (BandcampSuggestionExtractor) bandcamp.getSuggestionExtractor(); + extractor = (BandcampSuggestionExtractor) Bandcamp.getSuggestionExtractor(); } @Test From 67de0285e155f41c6810305b8dfe135da20086df Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 22:03:29 +0200 Subject: [PATCH 29/91] Bandcamp: Generate query JSON in a proper way --- .../extractors/BandcampChannelExtractor.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 9c1572cd4..fd068e60e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -2,10 +2,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.*; import org.jsoup.Jsoup; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -43,7 +40,12 @@ public class BandcampChannelExtractor extends ChannelExtractor { NewPipe.getDownloader().post( "https://bandcamp.com/api/mobile/22/band_details", null, - ("{\"band_id\":\"" + id + "\"}").getBytes() + JsonWriter.string() + .object() + .value("band_id", id) + .end() + .done() + .getBytes() ).responseBody() ); } catch (IOException | ReCaptchaException | JsonParserException e) { From 433d72c26a9b96527ab02bc7827bb8534f53ae0b Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 22:04:10 +0200 Subject: [PATCH 30/91] Depend on nanojson fork in timeago-parser --- timeago-parser/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timeago-parser/build.gradle b/timeago-parser/build.gradle index bd99b48bb..3175269d0 100644 --- a/timeago-parser/build.gradle +++ b/timeago-parser/build.gradle @@ -1,6 +1,6 @@ dependencies { testImplementation 'junit:junit:4.12' - implementation 'com.grack:nanojson:1.1' + implementation 'com.github.wb9688:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' } \ No newline at end of file From 00c03330596b00b2c6246cc29b5dccd84df1d7d5 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 22:06:48 +0200 Subject: [PATCH 31/91] Bandcamp: Fetch channelInfo in onFetchPage --- .../extractors/BandcampChannelExtractor.java | 5 +-- .../extractors/BandcampFeaturedExtractor.java | 8 +--- .../BandcampPlaylistInfoItemExtractor.java | 40 ++++++++++++++----- .../BandcampChannelExtractorTest.java | 10 ++++- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index fd068e60e..66d972ce0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -22,10 +22,8 @@ public class BandcampChannelExtractor extends ChannelExtractor { private JsonObject channelInfo; - public BandcampChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { + public BandcampChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); - - channelInfo = getArtistDetails(getId()); } /** @@ -156,6 +154,7 @@ public class BandcampChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + channelInfo = getArtistDetails(getId()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 37bfd1aea..986bdc3b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -72,13 +72,7 @@ public class BandcampFeaturedExtractor extends KioskExtractor { continue; } - c.commit(new BandcampPlaylistInfoItemExtractor( - featuredStory.getString("album_title"), - featuredStory.getString("band_name"), - featuredStory.getString("item_url"), - featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "", - featuredStory.getInt("num_streamable_tracks") - )); + c.commit(new BandcampPlaylistInfoItemExtractor(featuredStory)); } return new InfoItemsPage(c, null); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index 61551bac5..e146766c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -1,24 +1,44 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import java.io.IOException; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; + public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private String title, artist, url, cover; private int trackCount; - public BandcampPlaylistInfoItemExtractor(String title, String artist, String url, String cover, int trackCount) { - this.title = title; - this.artist = artist; - this.url = url; - this.cover = cover; - this.trackCount = trackCount; + public BandcampPlaylistInfoItemExtractor(Element searchResult) { + + Element resultInfo = searchResult.getElementsByClass("result-info").first(); + + Element img = searchResult.getElementsByClass("art").first() + .getElementsByTag("img").first(); + if (img != null) { + cover = img.attr("src"); + } + + title = resultInfo.getElementsByClass("heading").text(); + url = resultInfo.getElementsByClass("itemurl").text(); + + artist = resultInfo.getElementsByClass("subhead").text() + .split(" by")[0]; + + String length = resultInfo.getElementsByClass("length").text(); + trackCount = Integer.parseInt(length.split(" track")[0]); + + } + + public BandcampPlaylistInfoItemExtractor(JsonObject featuredStory) { + title = featuredStory.getString("album_title"); + artist = featuredStory.getString("band_name"); + url = featuredStory.getString("item_url"); + cover = featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : ""; + trackCount = featuredStory.getInt("num_streamable_tracks"); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 6aea55939..0d861eab3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -6,6 +6,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; @@ -18,12 +19,17 @@ import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; public class BandcampChannelExtractorTest { private static BandcampChannelExtractor extractor; + private static ChannelExtractor noAvatarExtractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (BandcampChannelExtractor) Bandcamp .getChannelExtractor("https://zachbenson.bandcamp.com/"); + extractor.fetchPage(); + + noAvatarExtractor = Bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/"); + noAvatarExtractor.fetchPage(); } @Test @@ -51,11 +57,11 @@ public class BandcampChannelExtractorTest { @Test public void testGetNoAvatar() throws ExtractionException { - assertEquals("", Bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/").getAvatarUrl()); + assertEquals("", noAvatarExtractor.getAvatarUrl()); } @Test public void testGetNoBanner() throws ExtractionException { - assertEquals("", Bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/").getBannerUrl()); + assertEquals("", noAvatarExtractor.getBannerUrl()); } } From 82099592c72fff0f5aac77e0cd8d5b67269ff70c Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 22:30:00 +0200 Subject: [PATCH 32/91] Bandcamp: Move code from SearchExtractor to InfoItemExtractors --- .../BandcampChannelInfoItemExtractor.java | 21 +++++++++--- .../BandcampPlaylistInfoItemExtractor.java | 3 +- .../extractors/BandcampSearchExtractor.java | 33 +++---------------- .../BandcampStreamInfoItemExtractor.java | 20 +++++++++++ 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java index 33d999117..2ff853407 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -9,11 +10,21 @@ public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtracto private String name, url, image, location; - public BandcampChannelInfoItemExtractor(String name, String url, String image, String location) { - this.name = name; - this.url = url; - this.image = image; - this.location = location; + public BandcampChannelInfoItemExtractor(Element searchResult) { + + Element resultInfo = searchResult.getElementsByClass("result-info").first(); + + Element img = searchResult.getElementsByClass("art").first() + .getElementsByTag("img").first(); + if (img != null) { + image = img.attr("src"); + } + + name = resultInfo.getElementsByClass("heading").text(); + + location = resultInfo.getElementsByClass("subhead").text(); + + url = resultInfo.getElementsByClass("itemurl").text(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index e146766c3..75d841723 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -1,10 +1,9 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import com.grack.nanojson.JsonObject; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; -import java.io.IOException; - import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index cd27a01d9..3f6a35b1a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -43,49 +43,26 @@ public class BandcampSearchExtractor extends SearchExtractor { for (Element searchResult : searchResultsElements) { - Element resultInfo = searchResult.getElementsByClass("result-info").first(); - - String type = resultInfo + String type = searchResult.getElementsByClass("result-info").first() .getElementsByClass("itemtype").first().text(); - String image = null; - Element img = searchResult.getElementsByClass("art").first() - .getElementsByTag("img").first(); - if (img != null) { - image = img.attr("src"); - } - - String heading = resultInfo.getElementsByClass("heading").text(); - - String subhead = resultInfo.getElementsByClass("subhead").text(); - - String url = resultInfo.getElementsByClass("itemurl").text(); - switch (type) { default: continue; case "FAN": - //collector.commit Channel (?) with heading, url, image + // don't display fan results break; case "ARTIST": - collector.commit(new BandcampChannelInfoItemExtractor(heading, url, image, subhead)); + collector.commit(new BandcampChannelInfoItemExtractor(searchResult)); break; case "ALBUM": - String artist = subhead.split(" by")[0]; - String length = resultInfo.getElementsByClass("length").text(); - int tracks = Integer.parseInt(length.split(" track")[0]); - collector.commit(new BandcampPlaylistInfoItemExtractor(heading, artist, url, image, tracks)); + collector.commit(new BandcampPlaylistInfoItemExtractor(searchResult)); break; case "TRACK": - String[] splitBy = subhead.split(" by"); - String artist1 = null; - if (splitBy.length > 1) { - artist1 = subhead.split(" by")[1]; - } - collector.commit(new BandcampStreamInfoItemExtractor(heading, url, image, artist1)); + collector.commit(new BandcampStreamInfoItemExtractor(searchResult)); break; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java index dd51f7692..e29ac241b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -31,6 +32,25 @@ public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor this(title, url, cover, artist, -1); } + public BandcampStreamInfoItemExtractor(Element searchResult) { + Element resultInfo = searchResult.getElementsByClass("result-info").first(); + + Element img = searchResult.getElementsByClass("art").first() + .getElementsByTag("img").first(); + if (img != null) { + cover = img.attr("src"); + } + + title = resultInfo.getElementsByClass("heading").text(); + url = resultInfo.getElementsByClass("itemurl").text(); + + String subhead = resultInfo.getElementsByClass("subhead").text(); + String[] splitBy = subhead.split(" by"); + if (splitBy.length > 1) { + artist = subhead.split(" by")[1]; + } + } + public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist, long duration) { this.title = title; this.url = url; From 965bce00cf61fb06513b6bbbc79e352dde828956 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 22:32:55 +0200 Subject: [PATCH 33/91] Bandcamp stream extractor: return NO_AGE_LIMIT --- .../services/bandcamp/extractors/BandcampStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 3785f8a13..65ef05455 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -137,7 +137,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() throws ParsingException { - return 0; + return NO_AGE_LIMIT; } @Override From 9201e0d3ea40dbbe0ae5e9e3d713365125e823c6 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 22:46:54 +0200 Subject: [PATCH 34/91] Don't declare exceptions that can't actually be thrown --- .../extractors/BandcampFeaturedExtractor.java | 4 ---- .../extractors/BandcampRadioExtractor.java | 2 +- .../extractors/BandcampStreamExtractor.java | 20 +++++++++---------- .../BandcampStreamInfoItemExtractor.java | 2 +- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 986bdc3b2..0456b4c2e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -6,7 +6,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.Collector; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.StreamingService; @@ -15,10 +14,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; -import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; -import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory; import javax.annotation.Nonnull; import java.io.IOException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java index e7ff3417a..55a16edbe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java @@ -76,7 +76,7 @@ public class BandcampRadioExtractor extends KioskExtractor { } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(String pageUrl) { return null; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 65ef05455..e38315af6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -89,13 +89,13 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getUploaderName() throws ParsingException { + public String getUploaderName() { return albumJson.getString("artist"); } @Nullable @Override - public String getTextualUploadDate() throws ParsingException { + public String getTextualUploadDate() { return current.getString("release_date"); } @@ -136,44 +136,44 @@ public class BandcampStreamExtractor extends StreamExtractor { } @Override - public int getAgeLimit() throws ParsingException { + public int getAgeLimit() { return NO_AGE_LIMIT; } @Override - public long getLength() throws ParsingException { + public long getLength() { return 0; } @Override - public long getTimeStamp() throws ParsingException { + public long getTimeStamp() { return 0; } @Override - public long getViewCount() throws ParsingException { + public long getViewCount() { return -1; } @Override - public long getLikeCount() throws ParsingException { + public long getLikeCount() { return -1; } @Override - public long getDislikeCount() throws ParsingException { + public long getDislikeCount() { return -1; } @Nonnull @Override - public String getDashMpdUrl() throws ParsingException { + public String getDashMpdUrl() { return ""; } @Nonnull @Override - public String getHlsUrl() throws ParsingException { + public String getHlsUrl() { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java index e29ac241b..4c2cd2319 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java @@ -128,7 +128,7 @@ public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor * There are no ads just like that, duh */ @Override - public boolean isAd() throws ParsingException { + public boolean isAd() { return false; } } From 10ae3db1184036e79c311583a3a0f2a6af5df6ad Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 23:03:12 +0200 Subject: [PATCH 35/91] Bandcamp javadoc: Replace br tags with p tags --- .../extractors/BandcampExtractorHelper.java | 6 +++--- .../BandcampStreamLinkHandlerFactory.java | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 39c7d1fdd..a9f4ec746 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -16,9 +16,9 @@ import java.util.Arrays; public class BandcampExtractorHelper { /** - * Get JSON behind var $variable = out of web page - *
- * Originally a part of bandcampDirect. + *

Get JSON behind var $variable = out of web page

+ * + *

Originally a part of bandcampDirect.

* * @param html The HTML where the JSON we're looking for is stored inside a * variable inside some JavaScript block diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java index 80a297df8..2d7495b97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -6,10 +6,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; /** - * Tracks don't have standalone ids, they are always in combination with the band id. - * That's why id = url. Instead, URLs are cleaned up so that they always look the same. - *

- * Radio (bandcamp weekly) shows do have ids. + *

Tracks don't have standalone ids, they are always in combination with the band id. + * That's why id = url. Instead, URLs are cleaned up so that they always look the same.

+ * + *

Radio (bandcamp weekly) shows do have ids.

*/ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { @@ -43,9 +43,9 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { * Sometimes, the root page of an artist is also an album or track * page. In that case, it is assumed that one actually wants to open * the profile and not the track it has set as the default one. - *

Urls are expected to be in this format to account for - * custom domains: - *
https:// * . * /track/ * + *

Urls are expected to be in this format to account for + * custom domains:

+ * https:// * . * /track/ * */ @Override public boolean onAcceptUrl(String url) { From 8c70dab8c5fe978d74b381bdc92915222bd928ea Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 23:03:46 +0200 Subject: [PATCH 36/91] Bandcamp search query handler factory: inline variable --- .../bandcamp/extractors/BandcampSuggestionExtractor.java | 4 +--- .../linkHandler/BandcampSearchQueryHandlerFactory.java | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index 4a819ed40..ac75d859a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -17,8 +17,6 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; -import static org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory.CHARSET_UTF_8; - public class BandcampSuggestionExtractor extends SuggestionExtractor { private static final String AUTOCOMPLETE_URL = "https://bandcamp.com/api/fuzzysearch/1/autocomplete?q="; @@ -32,7 +30,7 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { try { JsonObject fuzzyResults = JsonParser.object().from( - downloader.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, CHARSET_UTF_8)).responseBody() + downloader.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, "UTF-8")).responseBody() ); JsonArray jsonArray = fuzzyResults.getObject("auto") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java index 0fad75d5f..4a90c6c0e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java @@ -11,17 +11,13 @@ import java.util.List; public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - private static final String SEARCH_URL = "https://bandcamp.com/search?q="; - - public static final String CHARSET_UTF_8 = "UTF-8"; - @Override public String getUrl(String query, List contentFilter, String sortFilter) throws ParsingException { try { - return SEARCH_URL + - URLEncoder.encode(query, CHARSET_UTF_8) + return "https://bandcamp.com/search?q=" + + URLEncoder.encode(query, "UTF-8") + "&page=1"; } catch (UnsupportedEncodingException e) { From b5e251c82fdd61ba1187d5615364065f55212a75 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 23:06:41 +0200 Subject: [PATCH 37/91] =?UTF-8?q?Bandcamp=20channel=20extractor:=20package?= =?UTF-8?q?-private=20access=20for=20getImageUrl(=E2=80=A6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bandcamp/extractors/BandcampChannelExtractor.java | 2 +- .../services/bandcamp/BandcampChannelExtractorTest.java | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 66d972ce0..8b3cabfdb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -57,7 +57,7 @@ public class BandcampChannelExtractor extends ChannelExtractor { * @return Url of image with this ID in size 10 which is 1200x1200 (we could also choose size 0 * but we don't want something as large as 3460x3460 here, do we?) */ - public static String getImageUrl(long id, boolean album) { + static String getImageUrl(long id, boolean album) { return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 0d861eab3..fe329d398 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -32,12 +32,6 @@ public class BandcampChannelExtractorTest { noAvatarExtractor.fetchPage(); } - @Test - public void testImageUrl() { - assertEquals("https://f4.bcbits.com/img/a2405652335_10.jpg", BandcampChannelExtractor.getImageUrl(2405652335L, true)); - assertEquals("https://f4.bcbits.com/img/17433693_10.jpg", BandcampChannelExtractor.getImageUrl(17433693L, false)); - } - @Test public void testTranslateIdsToUrl() throws ParsingException { assertEquals("https://zachbenson.bandcamp.com/album/covers", BandcampExtractorHelper.getStreamUrlFromIds(2862267535L, 2063639444L, "album")); From 5009d9f53b762af578b5b0cd4b932869e62c6c7a Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 23:25:00 +0200 Subject: [PATCH 38/91] Bandcamp tests: remove certain tests --- .../services/bandcamp/BandcampStreamExtractorTest.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 1e5bd7eb9..ccb891ce0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -29,11 +29,6 @@ public class BandcampStreamExtractorTest { extractor.fetchPage(); } - @Test(expected = ExtractionException.class) - public void testAlbum() throws ExtractionException { - Bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/album/prom"); - } - @Test public void testServiceId() { } @@ -75,11 +70,6 @@ public class BandcampStreamExtractorTest { assertEquals(1, extractor.getAudioStreams().size()); } - @Test(expected = ParsingException.class) - public void testInvalidUrl() throws ExtractionException { - Bandcamp.getStreamExtractor("https://bandcamp.com"); - } - @Test public void testCategory() throws ExtractionException, IOException { StreamExtractor se = Bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); From f71bd29d58be131bb3783607a3395251fdb4cd29 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 20 Apr 2020 23:52:12 +0200 Subject: [PATCH 39/91] Bandcamp playlist extractor: add more tests --- .../BandcampPlaylistExtractorTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index fc2f434cd..8af486a1d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -8,8 +8,11 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; @@ -88,4 +91,61 @@ public class BandcampPlaylistExtractorTest { assertEquals(1, extractor.getStreamCount()); } + + public static class ComingOfAge { + + private static PlaylistExtractor extractor; + + @BeforeClass + public static void setUp() throws ExtractionException, IOException { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = Bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); + extractor.fetchPage(); + } + + @Test + public void getThumbnailUrl() throws ParsingException { + assertTrue(extractor.getThumbnailUrl().contains("f4.bcbits.com/img")); + } + + @Test + public void getBannerUrl() throws ParsingException { + assertEquals("", extractor.getBannerUrl()); + } + + @Test + public void getUploaderUrl() throws ParsingException { + assertTrue(extractor.getUploaderUrl().contains("macbenson.bandcamp.com")); + } + + @Test + public void getUploaderName() throws ParsingException { + assertEquals("mac benson", extractor.getUploaderName()); + } + + @Test + public void getUploaderAvatarUrl() throws ParsingException { + assertTrue(extractor.getUploaderAvatarUrl().contains("f4.bcbits.com/img")); + } + + @Test + public void getStreamCount() throws ParsingException { + assertEquals(5, extractor.getStreamCount()); + } + + @Test + public void getInitialPage() throws IOException, ExtractionException { + assertNotNull(extractor.getInitialPage().getItems().get(0)); + } + + @Test + public void getName() throws ParsingException { + assertEquals("Coming of Age", extractor.getName()); + } + + @Test + public void getNextPageUrl() throws IOException, ExtractionException { + assertNull(extractor.getNextPageUrl()); + } + } } From 9bc7a470abb3189b45d147b6f09ceb54338bf1d5 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 26 Apr 2020 23:02:14 +0200 Subject: [PATCH 40/91] Replace fori with foreach loop --- .../bandcamp/extractors/BandcampSuggestionExtractor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index ac75d859a..5ef4f3d56 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -38,9 +38,8 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { ArrayList suggestions = new ArrayList<>(); - for (int i = 0; i < jsonArray.size(); i++) { - JsonObject fuzzyResult = jsonArray.getObject(i); - String res = fuzzyResult.getString("name"); + for (Object fuzzyResult : jsonArray) { + String res = ((JsonObject) fuzzyResult).getString("name"); if (!suggestions.contains(res)) suggestions.add(res); } From a3e8e1c70ed60d3b2e622d4b50eb461d289e9601 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 26 Apr 2020 23:05:56 +0200 Subject: [PATCH 41/91] Bandcamp: move own methods to helper class --- .../extractors/BandcampChannelExtractor.java | 42 ++----------------- .../extractors/BandcampExtractorHelper.java | 36 ++++++++++++++++ .../extractors/BandcampFeaturedExtractor.java | 2 - .../extractors/BandcampPlaylistExtractor.java | 2 +- .../BandcampPlaylistInfoItemExtractor.java | 2 +- .../BandcampRadioInfoItemExtractor.java | 2 +- .../BandcampRadioStreamExtractor.java | 2 +- .../extractors/BandcampStreamExtractor.java | 2 +- .../BandcampChannelLinkHandlerFactory.java | 3 +- 9 files changed, 45 insertions(+), 48 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 8b3cabfdb..4da4a44ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -4,7 +4,6 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.*; import org.jsoup.Jsoup; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -26,46 +25,11 @@ public class BandcampChannelExtractor extends ChannelExtractor { super(service, linkHandler); } - /** - * Fetch artist details from mobile endpoint. - * - * I once took a moment to note down how it works. - */ - public static JsonObject getArtistDetails(String id) throws ParsingException { - try { - return - JsonParser.object().from( - NewPipe.getDownloader().post( - "https://bandcamp.com/api/mobile/22/band_details", - null, - JsonWriter.string() - .object() - .value("band_id", id) - .end() - .done() - .getBytes() - ).responseBody() - ); - } catch (IOException | ReCaptchaException | JsonParserException e) { - throw new ParsingException("Could not download band details", e); - } - } - - /** - * @param id The image ID - * @param album Whether this is the cover of an album - * @return Url of image with this ID in size 10 which is 1200x1200 (we could also choose size 0 - * but we don't want something as large as 3460x3460 here, do we?) - */ - static String getImageUrl(long id, boolean album) { - return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; - } - @Override public String getAvatarUrl() { if (channelInfo.getLong("bio_image_id") == 0) return ""; - return getImageUrl(channelInfo.getLong("bio_image_id"), false); + return BandcampExtractorHelper.getImageUrl(channelInfo.getLong("bio_image_id"), false); } /** @@ -132,7 +96,7 @@ public class BandcampChannelExtractor extends ChannelExtractor { discograph.getLong("item_id"), discograph.getString("item_type") ), - getImageUrl( + BandcampExtractorHelper.getImageUrl( discograph.getLong("art_id"), true ), discograph.getString("band_name") @@ -154,7 +118,7 @@ public class BandcampChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - channelInfo = getArtistDetails(getId()); + channelInfo = BandcampExtractorHelper.getArtistDetails(getId()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index a9f4ec746..5371681a4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -5,6 +5,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -105,4 +106,39 @@ public class BandcampExtractorHelper { return String.valueOf(result); } + + /** + * Fetch artist details from mobile endpoint. + * + * I once took a moment to note down how it works. + */ + public static JsonObject getArtistDetails(String id) throws ParsingException { + try { + return + JsonParser.object().from( + NewPipe.getDownloader().post( + "https://bandcamp.com/api/mobile/22/band_details", + null, + JsonWriter.string() + .object() + .value("band_id", id) + .end() + .done() + .getBytes() + ).responseBody() + ); + } catch (IOException | ReCaptchaException | JsonParserException e) { + throw new ParsingException("Could not download band details", e); + } + } + + /** + * @param id The image ID + * @param album Whether this is the cover of an album + * @return Url of image with this ID in size 10 which is 1200x1200 (we could also choose size 0 + * but we don't want something as large as 3460x3460 here, do we?) + */ + static String getImageUrl(long id, boolean album) { + return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 0456b4c2e..d5d3c412d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -19,8 +19,6 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; - public class BandcampFeaturedExtractor extends KioskExtractor { public static final String KIOSK_FEATURED = "Featured"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index ddb78d3fa..ddc2b6b40 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -18,7 +18,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getJSONFromJavaScriptVariables; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index 75d841723..668dcddd7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -4,7 +4,7 @@ import com.grack.nanojson.JsonObject; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index adae3e557..808c1afd3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 98518b442..3780671ac 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index e38315af6..8353959fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -24,7 +24,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; public class BandcampStreamExtractor extends StreamExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index 94b45771e..370057dde 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import java.io.IOException; @@ -41,7 +40,7 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { try { - return BandcampChannelExtractor.getArtistDetails(id) + return BandcampExtractorHelper.getArtistDetails(id) .getString("bandcamp_url") .replace("http://", "https://"); } catch (NullPointerException e) { From fff21caa87dfe6fcac5edee14a222d18e5d68618 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 25 May 2020 17:02:49 +0200 Subject: [PATCH 42/91] Fix Bandcamp no avatar test I own the bandcamp artist account called "NewPipeExtractorTest", so it shouldn't break again. --- .../services/bandcamp/BandcampChannelExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index fe329d398..8224896cf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -28,7 +28,7 @@ public class BandcampChannelExtractorTest { .getChannelExtractor("https://zachbenson.bandcamp.com/"); extractor.fetchPage(); - noAvatarExtractor = Bandcamp.getChannelExtractor("https://powertothequeerkids.bandcamp.com/"); + noAvatarExtractor = Bandcamp.getChannelExtractor("https://npet.bandcamp.com/"); noAvatarExtractor.fetchPage(); } From 692b2d06f48d469f965b74093c14060a77f6c555 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 25 May 2020 18:51:31 +0200 Subject: [PATCH 43/91] Bandcamp: read info item data in place and not in advance --- .../extractors/BandcampChannelExtractor.java | 14 +- .../BandcampChannelInfoItemExtractor.java | 30 ++-- .../extractors/BandcampExtractorHelper.java | 2 +- .../extractors/BandcampFeaturedExtractor.java | 11 +- .../extractors/BandcampPlaylistExtractor.java | 19 +-- .../BandcampPlaylistInfoItemExtractor.java | 50 ++----- ...campPlaylistInfoItemFeaturedExtractor.java | 41 ++++++ .../BandcampRadioInfoItemExtractor.java | 2 +- .../extractors/BandcampSearchExtractor.java | 4 +- .../BandcampStreamInfoItemExtractor.java | 134 ------------------ ...campDiscographStreamInfoItemExtractor.java | 46 ++++++ ...ndcampPlaylistStreamInfoItemExtractor.java | 69 +++++++++ ...BandcampSearchStreamInfoItemExtractor.java | 48 +++++++ .../BandcampStreamInfoItemExtractor.java | 54 +++++++ .../BandcampFeaturedExtractorTest.java | 3 +- .../BandcampPlaylistExtractorTest.java | 2 - 16 files changed, 303 insertions(+), 226 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 4da4a44ef..446b962c7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -89,18 +90,7 @@ public class BandcampChannelExtractor extends ChannelExtractor { if (!discograph.getString("item_type").equals("track")) continue; - collector.commit(new BandcampStreamInfoItemExtractor( - discograph.getString("title"), - BandcampExtractorHelper.getStreamUrlFromIds( - discograph.getLong("band_id"), - discograph.getLong("item_id"), - discograph.getString("item_type") - ), - BandcampExtractorHelper.getImageUrl( - discograph.getLong("art_id"), true - ), - discograph.getString("band_name") - )); + collector.commit(new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl())); } return new InfoItemsPage<>(collector, null); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java index 2ff853407..3baf45b3e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java @@ -8,43 +8,35 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtractor { - private String name, url, image, location; + private final Element resultInfo, searchResult; public BandcampChannelInfoItemExtractor(Element searchResult) { - - Element resultInfo = searchResult.getElementsByClass("result-info").first(); - - Element img = searchResult.getElementsByClass("art").first() - .getElementsByTag("img").first(); - if (img != null) { - image = img.attr("src"); - } - - name = resultInfo.getElementsByClass("heading").text(); - - location = resultInfo.getElementsByClass("subhead").text(); - - url = resultInfo.getElementsByClass("itemurl").text(); + this.searchResult = searchResult; + resultInfo = searchResult.getElementsByClass("result-info").first(); } @Override public String getName() throws ParsingException { - return name; + return resultInfo.getElementsByClass("heading").text(); } @Override public String getUrl() throws ParsingException { - return url; + return resultInfo.getElementsByClass("itemurl").text(); } @Override public String getThumbnailUrl() throws ParsingException { - return image; + Element img = searchResult.getElementsByClass("art").first() + .getElementsByTag("img").first(); + if (img != null) { + return img.attr("src"); + } else return null; } @Override public String getDescription() { - return location; + return resultInfo.getElementsByClass("subhead").text(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 5371681a4..cfe5c2df2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -138,7 +138,7 @@ public class BandcampExtractorHelper { * @return Url of image with this ID in size 10 which is 1200x1200 (we could also choose size 0 * but we don't want something as large as 3460x3460 here, do we?) */ - static String getImageUrl(long id, boolean album) { + public static String getImageUrl(long id, boolean album) { return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index d5d3c412d..82905d239 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -14,12 +14,13 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; -public class BandcampFeaturedExtractor extends KioskExtractor { +public class BandcampFeaturedExtractor extends KioskExtractor { public static final String KIOSK_FEATURED = "Featured"; public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data"; @@ -41,9 +42,9 @@ public class BandcampFeaturedExtractor extends KioskExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - InfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); + PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); try { @@ -66,10 +67,10 @@ public class BandcampFeaturedExtractor extends KioskExtractor { continue; } - c.commit(new BandcampPlaylistInfoItemExtractor(featuredStory)); + c.commit(new BandcampPlaylistInfoItemFeaturedExtractor(featuredStory)); } - return new InfoItemsPage(c, null); + return new InfoItemsPage<>(c, null); } catch (JsonParserException e) { e.printStackTrace(); throw new ParsingException("JSON error", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index ddc2b6b40..b57644400 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampPlaylistStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -111,22 +112,12 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { if (trackInfo.size() < MAXIMUM_INDIVIDUAL_COVER_ARTS) { // Load cover art of every track individually - collector.commit(new BandcampStreamInfoItemExtractor( - track.getString("title"), - getUploaderUrl() + track.getString("title_link"), - "", - track.getLong("duration"), - getService() - )); + collector.commit(new BandcampPlaylistStreamInfoItemExtractor( + track, getUploaderUrl(), getService())); } else { // Pretend every track has the same cover art as the album - collector.commit(new BandcampStreamInfoItemExtractor( - track.getString("title"), - getUploaderUrl() + track.getString("title_link"), - getThumbnailUrl(), - "", - track.getLong("duration") - )); + collector.commit(new BandcampPlaylistStreamInfoItemExtractor( + track, getUploaderUrl(), getThumbnailUrl())); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index 668dcddd7..b28d13d72 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -4,64 +4,42 @@ import com.grack.nanojson.JsonObject; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; - public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { - - private String title, artist, url, cover; - private int trackCount; + private final Element searchResult, resultInfo; public BandcampPlaylistInfoItemExtractor(Element searchResult) { - - Element resultInfo = searchResult.getElementsByClass("result-info").first(); - - Element img = searchResult.getElementsByClass("art").first() - .getElementsByTag("img").first(); - if (img != null) { - cover = img.attr("src"); - } - - title = resultInfo.getElementsByClass("heading").text(); - url = resultInfo.getElementsByClass("itemurl").text(); - - artist = resultInfo.getElementsByClass("subhead").text() - .split(" by")[0]; - - String length = resultInfo.getElementsByClass("length").text(); - trackCount = Integer.parseInt(length.split(" track")[0]); - - } - - public BandcampPlaylistInfoItemExtractor(JsonObject featuredStory) { - title = featuredStory.getString("album_title"); - artist = featuredStory.getString("band_name"); - url = featuredStory.getString("item_url"); - cover = featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : ""; - trackCount = featuredStory.getInt("num_streamable_tracks"); + this.searchResult = searchResult; + resultInfo = searchResult.getElementsByClass("result-info").first(); } @Override public String getUploaderName() { - return artist; + return resultInfo.getElementsByClass("subhead").text() + .split(" by")[0]; } @Override public long getStreamCount() { - return trackCount; + String length = resultInfo.getElementsByClass("length").text(); + return Integer.parseInt(length.split(" track")[0]); } @Override public String getName() { - return title; + return resultInfo.getElementsByClass("heading").text(); } @Override public String getUrl() { - return url; + return resultInfo.getElementsByClass("itemurl").text(); } @Override public String getThumbnailUrl() { - return cover; + Element img = searchResult.getElementsByClass("art").first() + .getElementsByTag("img").first(); + if (img != null) { + return img.attr("src"); + } else return null; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java new file mode 100644 index 000000000..814794284 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java @@ -0,0 +1,41 @@ +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; + +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; + +public class BandcampPlaylistInfoItemFeaturedExtractor implements PlaylistInfoItemExtractor { + + private final JsonObject featuredStory; + + public BandcampPlaylistInfoItemFeaturedExtractor(JsonObject featuredStory) { + this.featuredStory = featuredStory; + } + + @Override + public String getUploaderName() { + return featuredStory.getString("band_name"); + } + + @Override + public long getStreamCount() { + return featuredStory.getInt("num_streamable_tracks"); + } + + @Override + public String getName() { + return featuredStory.getString("album_title"); + } + + @Override + public String getUrl() { + return featuredStory.getString("item_url"); + } + + @Override + public String getThumbnailUrl() { + return featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : ""; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index 808c1afd3..ebc843195 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -22,7 +22,7 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @Override public long getDuration() { - /* Duration is only present in the more detailed information that has to be queried seperately. + /* Duration is only present in the more detailed information that has to be queried separately. * Because the servers would probably not like over 300 queries every time someone opens the kiosk, * we're just providing 0 here. */ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index 3f6a35b1a..8897d0406 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -13,6 +13,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampPlaylistStreamInfoItemExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampSearchStreamInfoItemExtractor; import javax.annotation.Nonnull; import java.io.IOException; @@ -62,7 +64,7 @@ public class BandcampSearchExtractor extends SearchExtractor { break; case "TRACK": - collector.commit(new BandcampStreamInfoItemExtractor(searchResult)); + collector.commit(new BandcampSearchStreamInfoItemExtractor(searchResult, null)); break; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java deleted file mode 100644 index 4c2cd2319..000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java +++ /dev/null @@ -1,134 +0,0 @@ -// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later - -package org.schabi.newpipe.extractor.services.bandcamp.extractors; - -import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; - -import javax.annotation.Nullable; -import java.io.IOException; - -public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor { - - private String title; - private String url; - private String cover; - private String artist; - private long duration; - private StreamingService service; - - public BandcampStreamInfoItemExtractor(String title, String url, String artist, long duration, StreamingService service) { - this(title, url, null, artist, duration); - this.service = service; - } - - public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist) { - this(title, url, cover, artist, -1); - } - - public BandcampStreamInfoItemExtractor(Element searchResult) { - Element resultInfo = searchResult.getElementsByClass("result-info").first(); - - Element img = searchResult.getElementsByClass("art").first() - .getElementsByTag("img").first(); - if (img != null) { - cover = img.attr("src"); - } - - title = resultInfo.getElementsByClass("heading").text(); - url = resultInfo.getElementsByClass("itemurl").text(); - - String subhead = resultInfo.getElementsByClass("subhead").text(); - String[] splitBy = subhead.split(" by"); - if (splitBy.length > 1) { - artist = subhead.split(" by")[1]; - } - } - - public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist, long duration) { - this.title = title; - this.url = url; - this.cover = cover; - this.artist = artist; - this.duration = duration; - } - - @Override - public StreamType getStreamType() { - return StreamType.AUDIO_STREAM; - } - - @Override - public long getDuration() { - return duration; - } - - @Override - public long getViewCount() { - return -1; - } - - @Override - public String getUploaderName() { - return artist; - } - - @Override - public String getUploaderUrl() { - return null; - } - - @Nullable - @Override - public String getTextualUploadDate() { - return null; // TODO - } - - @Nullable - @Override - public DateWrapper getUploadDate() { - return null; - } - - @Override - public String getName() throws ParsingException { - return title; - } - - @Override - public String getUrl() throws ParsingException { - return url; - } - - /** - * There is no guarantee that every track of an album has the same cover art, so it needs to be fetched - * per-track if in playlist view - */ - @Override - public String getThumbnailUrl() throws ParsingException { - if (cover != null) return cover; - else { - try { - StreamExtractor extractor = service.getStreamExtractor(getUrl()); - extractor.fetchPage(); - return extractor.getThumbnailUrl(); - } catch (ExtractionException | IOException e) { - throw new ParsingException("could not download cover art location", e); - } - } - } - - /** - * There are no ads just like that, duh - */ - @Override - public boolean isAd() { - return false; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java new file mode 100644 index 000000000..4c76b404d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; + +public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { + + private final JsonObject discograph; + public BandcampDiscographStreamInfoItemExtractor(JsonObject discograph, String uploaderUrl) { + super(uploaderUrl); + + this.discograph = discograph; + } + + @Override + public String getUploaderName() { + return discograph.getString("band_name"); + } + + @Override + public String getName() { + return discograph.getString("title"); + } + + @Override + public String getUrl() throws ParsingException { + return BandcampExtractorHelper.getStreamUrlFromIds( + discograph.getLong("band_id"), + discograph.getLong("item_id"), + discograph.getString("item_type") + ); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return BandcampExtractorHelper.getImageUrl( + discograph.getLong("art_id"), true + ); + } + + @Override + public long getDuration() { + return -1; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java new file mode 100644 index 000000000..6faf397ae --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -0,0 +1,69 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamExtractor; + +import java.io.IOException; + + +public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { + + private final JsonObject track; + private String substituteCoverUrl; + private StreamingService service; + + public BandcampPlaylistStreamInfoItemExtractor(JsonObject track, String uploaderUrl, StreamingService service) { + super(uploaderUrl); + this.track = track; + this.service = service; + } + + public BandcampPlaylistStreamInfoItemExtractor(JsonObject track, String uploaderUrl, String substituteCoverUrl) { + this(track, uploaderUrl, (StreamingService) null); + this.substituteCoverUrl = substituteCoverUrl; + } + + @Override + public String getName() { + return track.getString("title"); + } + + @Override + public String getUrl() { + return getUploaderUrl() + track.getString("title_link"); + } + + @Override + public long getDuration() { + return track.getLong("duration"); + } + + @Override + public String getUploaderName() { + return ""; + } + + /** + * Each track can have its own cover art. Therefore, unless a substitute is provided, + * the thumbnail is extracted using a stream extractor. + */ + @Override + public String getThumbnailUrl() throws ParsingException { + if (substituteCoverUrl != null) { + return substituteCoverUrl; + } else { + try { + StreamExtractor extractor = service.getStreamExtractor(getUrl()); + extractor.fetchPage(); + return extractor.getThumbnailUrl(); + } catch (ExtractionException | IOException e) { + throw new ParsingException("could not download cover art location", e); + } + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java new file mode 100644 index 000000000..51fbe019d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -0,0 +1,48 @@ +package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; + +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { + + private final Element resultInfo, searchResult; + + public BandcampSearchStreamInfoItemExtractor(Element searchResult, String uploaderUrl) { + super(uploaderUrl); + this.searchResult = searchResult; + resultInfo = searchResult.getElementsByClass("result-info").first(); + } + + @Override + public String getUploaderName() throws ParsingException { + String subhead = resultInfo.getElementsByClass("subhead").text(); + String[] splitBy = subhead.split(" by"); + if (splitBy.length > 1) { + return splitBy[1]; + } else throw new ParsingException("Uploader name was not found as expected"); + } + + @Override + public String getName() throws ParsingException { + return resultInfo.getElementsByClass("heading").text(); + } + + @Override + public String getUrl() throws ParsingException { + return resultInfo.getElementsByClass("itemurl").text(); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + Element img = searchResult.getElementsByClass("art").first() + .getElementsByTag("img").first(); + if (img != null) { + return img.attr("src"); + } else return null; + } + + @Override + public long getDuration() { + return -1; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java new file mode 100644 index 000000000..a8a3ceb46 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java @@ -0,0 +1,54 @@ +package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; + +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; + +/** + * Implements methods that return a constant value for better readability in + * subclasses. + */ +public abstract class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor { + private final String uploaderUrl; + + public BandcampStreamInfoItemExtractor(String uploaderUrl) { + this.uploaderUrl = uploaderUrl; + } + + @Override + public StreamType getStreamType() { + return StreamType.AUDIO_STREAM; + } + + @Override + public long getViewCount() { + return -1; + } + + @Override + public String getUploaderUrl() { + return uploaderUrl; + } + + @Nullable + @Override + public String getTextualUploadDate() { + return null; + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return null; + } + + /** + * There are no ads just like that, duh + */ + @Override + public boolean isAd() { + return false; + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index e02e45932..09f516be9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor; import java.io.IOException; @@ -32,7 +33,7 @@ public class BandcampFeaturedExtractorTest { @Test public void testFeaturedCount() throws ExtractionException, IOException { - List list = extractor.getInitialPage().getItems(); + List list = extractor.getInitialPage().getItems(); assertTrue(list.size() > 1); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index 8af486a1d..88836efab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -11,8 +11,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; From e08256ebc6ac71f3d1b3d94db1f81359e3e69cb9 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 25 May 2020 19:17:27 +0200 Subject: [PATCH 44/91] Remove "url cleanup" in Bandcamp link handlers --- .../linkHandler/BandcampPlaylistLinkHandlerFactory.java | 7 ++----- .../linkHandler/BandcampStreamLinkHandlerFactory.java | 9 +++------ .../bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java | 1 + .../bandcamp/BandcampStreamLinkHandlerFactoryTest.java | 5 ----- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java index e7aac43e2..9a736b30f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java @@ -18,14 +18,11 @@ public class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String url, List contentFilter, String sortFilter) throws ParsingException { - if (url.endsWith("/")) - url = url.substring(0, url.length() - 1); - url = url.replace("http://", "https://").toLowerCase(); return url; } @Override - public boolean onAcceptUrl(String url) throws ParsingException { - return getUrl(url).matches("https?://.+\\..+/album/.+"); + public boolean onAcceptUrl(String url) { + return url.toLowerCase().matches("https?://.+\\..+/album/.+"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java index 2d7495b97..176c2e2ba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; /** *

Tracks don't have standalone ids, they are always in combination with the band id. - * That's why id = url. Instead, URLs are cleaned up so that they always look the same.

+ * That's why id = url.

* *

Radio (bandcamp weekly) shows do have ids.

*/ @@ -33,10 +33,7 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { public String getUrl(String input) { if (input.matches("\\d+")) return "https://bandcamp.com/?show=" + input; - if (input.endsWith("/")) - input = input.substring(0, input.length() - 1); - input = input.replace("http://", "https://").toLowerCase(); - return input; + else return input; } /** @@ -49,6 +46,6 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { */ @Override public boolean onAcceptUrl(String url) { - return getUrl(url).matches("https?://.+\\..+/track/.+") || getUrl(url).matches("https?://bandcamp\\.com/\\?show=\\d+"); + return url.toLowerCase().matches("https?://.+\\..+/track/.+") || url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java index 1da3069c9..f9cf9b1a7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java @@ -40,6 +40,7 @@ public class BandcampPlaylistLinkHandlerFactoryTest { // Tests expecting true assertTrue(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids")); assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/album/prom")); + assertTrue(linkHandler.acceptUrl("https://MACBENSON.BANDCAMP.COM/ALBUM/COMING-OF-AGE")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java index 9cbd03295..3942fb1f1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -25,11 +25,6 @@ public class BandcampStreamLinkHandlerFactoryTest { NewPipe.init(DownloaderTestImpl.getInstance()); } - @Test - public void testUrlCleanup() { - assertEquals("https://zachbenson.bandcamp.com/track/u-i-tonite", linkHandler.getUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); - } - @Test public void testGetRadioUrl() { assertEquals("https://bandcamp.com/?show=1", linkHandler.getUrl("1")); From 89b0639faa9948befa7f8c651a21e3d2e415b826 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Mon, 25 May 2020 20:22:23 +0200 Subject: [PATCH 45/91] New Bandcamp tests for channel extractor --- .../BandcampChannelExtractorTest.java | 91 +++++++++++++------ .../bandcamp/BandcampStreamExtractorTest.java | 6 ++ 2 files changed, 69 insertions(+), 28 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 8224896cf..2d1c38e30 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -8,54 +8,89 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import java.io.IOException; + +import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; -public class BandcampChannelExtractorTest { +public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { - private static BandcampChannelExtractor extractor; - private static ChannelExtractor noAvatarExtractor; + private static ChannelExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (BandcampChannelExtractor) Bandcamp - .getChannelExtractor("https://zachbenson.bandcamp.com/"); + extractor = Bandcamp.getChannelExtractor("https://npet.bandcamp.com/releases"); extractor.fetchPage(); - - noAvatarExtractor = Bandcamp.getChannelExtractor("https://npet.bandcamp.com/"); - noAvatarExtractor.fetchPage(); } @Test - public void testTranslateIdsToUrl() throws ParsingException { - assertEquals("https://zachbenson.bandcamp.com/album/covers", BandcampExtractorHelper.getStreamUrlFromIds(2862267535L, 2063639444L, "album")); - // TODO write more test cases + public void testLength() throws ExtractionException, IOException { + assertTrue(extractor.getInitialPage().getItems().size() >= 1); } + @Override @Test - public void testLength() throws ParsingException { - assertTrue(extractor.getInitialPage().getItems().size() > 2); + public void testDescription() throws Exception { + assertEquals("This string will be tested for in NewPipeExtractor tests.", extractor.getDescription()); } - @Test - public void testGetBannerUrl() throws ParsingException { - // Why is this picture in png format when all other pictures are jpg? - assertTrue(extractor.getBannerUrl().endsWith(".png")); + @Override + public void testAvatarUrl() throws Exception { + // Has no avatar + assertEquals("", extractor.getAvatarUrl()); } - @Test - public void testGetNoAvatar() throws ExtractionException { - assertEquals("", noAvatarExtractor.getAvatarUrl()); + @Override + public void testBannerUrl() throws Exception { + // Has no banner + assertEquals("", extractor.getBannerUrl()); } - @Test - public void testGetNoBanner() throws ExtractionException { - assertEquals("", noAvatarExtractor.getBannerUrl()); + @Override + public void testFeedUrl() throws Exception { + assertNull(extractor.getFeedUrl()); + } + + @Override + public void testSubscriberCount() throws Exception { + assertEquals(-1, extractor.getSubscriberCount()); + } + + @Override + public void testRelatedItems() throws Exception { + // not implemented + } + + @Override + public void testMoreRelatedItems() throws Exception { + // not implemented + } + + @Override + public void testServiceId() throws Exception { + assertEquals(4, extractor.getServiceId()); + } + + @Override + public void testName() throws Exception { + assertEquals("NewPipeExtractorTest", extractor.getName()); + } + + @Override + public void testId() throws Exception { + assertEquals("https://npet.bandcamp.com/", extractor.getId()); + } + + @Override + public void testUrl() throws Exception { + assertEquals("https://npet.bandcamp.com/releases", extractor.getUrl()); + } + + @Override + public void testOriginalUrl() throws Exception { + assertEquals("https://npet.bandcamp.com/releases", extractor.getUrl()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index ccb891ce0..c4d268226 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -84,5 +85,10 @@ public class BandcampStreamExtractorTest { assertEquals("CC BY 3.0", se.getLicence()); } + @Test + public void testTranslateIdsToUrl() throws ParsingException { + assertEquals("https://zachbenson.bandcamp.com/album/covers", BandcampExtractorHelper.getStreamUrlFromIds(2862267535L, 2063639444L, "album")); + // TODO write more test cases + } } From 9fa9d920a9e17b7703bed8fde5cd4db4be8705aa Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Wed, 3 Jun 2020 21:35:11 +0200 Subject: [PATCH 46/91] Bandcamp: Implement new methods required due to interface changes --- .../extractors/BandcampChannelExtractor.java | 15 +++++++++++++++ .../extractors/BandcampPlaylistExtractor.java | 18 ++++++++++++++++++ .../extractors/BandcampSearchExtractor.java | 5 +++++ .../extractors/BandcampStreamExtractor.java | 18 ++++++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 446b962c7..5a8d4d5f6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -76,6 +76,21 @@ public class BandcampChannelExtractor extends ChannelExtractor { return channelInfo.getString("bio"); } + @Override + public String getParentChannelName() { + return null; + } + + @Override + public String getParentChannelUrl() { + return null; + } + + @Override + public String getParentChannelAvatarUrl() { + return null; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index b57644400..7b9da51cd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -101,6 +101,24 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { return trackInfo.size(); } + @Nonnull + @Override + public String getSubChannelName() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() { + return ""; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index 8897d0406..4b5672b49 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -30,6 +30,11 @@ public class BandcampSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { // okay apparently this is where we DOWNLOAD the page and then COMMIT its ENTRIES to an INFOITEMPAGE diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 8353959fc..b6ed0a5da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -122,6 +122,24 @@ public class BandcampStreamExtractor extends StreamExtractor { } } + @Nonnull + @Override + public String getSubChannelUrl() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelName() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() { + return ""; + } + @Nonnull @Override public Description getDescription() { From 3940138fc51306c9ad9010e5776f313bf5d919e8 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Wed, 3 Jun 2020 22:41:27 +0200 Subject: [PATCH 47/91] Fix Bandcamp capitalization in one more spot --- .../newpipe/extractor/services/bandcamp/BandcampService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 1aa5b3c77..a94638a3f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -29,7 +29,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp public class BandcampService extends StreamingService { public BandcampService(int id) { - super(id, "bandcamp", Collections.singletonList(AUDIO)); + super(id, "Bandcamp", Collections.singletonList(AUDIO)); } @Override From de776561b0eeff079f614a97a2ecf1f46c6a1581 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 12:02:18 +0200 Subject: [PATCH 48/91] Bandcamp: fix loading uploader from streams in search --- .../BandcampSearchStreamInfoItemExtractor.java | 8 +++++--- .../services/bandcamp/BandcampSearchExtractorTest.java | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index 51fbe019d..6d88edc1c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -14,12 +14,14 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte } @Override - public String getUploaderName() throws ParsingException { + public String getUploaderName() { String subhead = resultInfo.getElementsByClass("subhead").text(); - String[] splitBy = subhead.split(" by"); + String[] splitBy = subhead.split("by "); if (splitBy.length > 1) { return splitBy[1]; - } else throw new ParsingException("Uploader name was not found as expected"); + } else { + return splitBy[0]; + } } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index ba8c147b5..a1b44b63b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelInfoItemExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; @@ -44,10 +45,11 @@ public class BandcampSearchExtractorTest { SearchExtractor extractor = Bandcamp.getSearchExtractor("best friend's basement"); ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - InfoItem bestFriendsBasement = page.getItems().get(0); + StreamInfoItem bestFriendsBasement = (StreamInfoItem) page.getItems().get(0); // The track by Zach Benson should be the first result, no? assertEquals("Best Friend's Basement", bestFriendsBasement.getName()); + assertEquals("Zach Benson", bestFriendsBasement.getUploaderName()); assertTrue(bestFriendsBasement.getThumbnailUrl().endsWith(".jpg")); assertTrue(bestFriendsBasement.getThumbnailUrl().contains("f4.bcbits.com/img/")); assertEquals(InfoItem.InfoType.STREAM, bestFriendsBasement.getInfoType()); From 8f6c00f8d81d16de1b2f08a72452ebc5ad03c88c Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 12:12:20 +0200 Subject: [PATCH 49/91] Use default test in BandcampSearchExtractorTest --- .../bandcamp/BandcampSearchExtractorTest.java | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index a1b44b63b..532f27a53 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -9,9 +9,11 @@ import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelInfoItemExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; @@ -19,6 +21,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; +import javax.annotation.Nullable; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; @@ -28,8 +32,6 @@ import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; */ public class BandcampSearchExtractorTest { - private static BandcampSearchExtractor extractor; - @BeforeClass public static void setUp() { NewPipe.init(DownloaderTestImpl.getInstance()); @@ -104,4 +106,25 @@ public class BandcampSearchExtractorTest { assertEquals("https://bandcamp.com/search?q=e&page=3", extractor.getPage(extractor.getNextPageUrl()).getNextPageUrl()); } + + public static class DefaultTest extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "test"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = Bandcamp.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return Bandcamp; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "bandcamp.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "bandcamp.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } } From e98bff8ae96f498987add5fbd182385f0d9899a2 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 12:30:53 +0200 Subject: [PATCH 50/91] Improved Bandcamp radio stream extractor tests --- .../BandcampRadioStreamExtractorTest.java | 75 ++++++++++++++++++- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 760a833f7..2da8ec913 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -6,8 +6,10 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamExtractor; import java.io.IOException; @@ -16,9 +18,15 @@ import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; public class BandcampRadioStreamExtractorTest { + private static StreamExtractor e; + + private static final String SHOW_URL = "https://bandcamp.com/?show=230"; + @BeforeClass - public static void setUp() { + public static void setUp() throws IOException, ExtractionException { NewPipe.init(DownloaderTestImpl.getInstance()); + e = Bandcamp.getStreamExtractor(SHOW_URL); + e.fetchPage(); } @Test @@ -28,10 +36,69 @@ public class BandcampRadioStreamExtractorTest { } @Test - public void testExtracting() throws ExtractionException, IOException { - BandcampRadioStreamExtractor e = (BandcampRadioStreamExtractor) Bandcamp.getStreamExtractor("https://bandcamp.com/?show=230"); - e.fetchPage(); + public void testGetName() throws ParsingException { assertEquals("Sound Movements", e.getName()); + } + + @Test + public void testGetUploaderUrl() { + } + + @Test + public void testGetUrl() throws ParsingException { + assertEquals(SHOW_URL, e.getUrl()); + } + + @Test + public void testGetUploaderName() throws ParsingException { assertEquals("Andrew Jervis", e.getUploaderName()); } + + @Test + public void testGetTextualUploadDate() throws ParsingException { + assertEquals("16 May 2017", e.getTextualUploadDate()); + + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertTrue(e.getThumbnailUrl().contains("bcbits.com/img")); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertTrue(e.getUploaderAvatarUrl().contains("bandcamp-button")); + + } + + @Test + public void testGetDescription() throws ParsingException { + assertEquals("Featuring special guests Nick Hakim and Elbows, plus fresh cuts from Eddie Palmieri, KRS One, Ladi6, and Moonchild.", e.getDescription().getContent()); + + } + + @Test + public void testGetLength() throws ParsingException { + assertEquals(5619, e.getLength()); + } + + @Test + public void testGetAudioStreams() throws ExtractionException, IOException { + assertEquals(2, e.getAudioStreams().size()); + } + + @Test + public void testGetLicence() throws ParsingException { + assertEquals("", e.getLicence()); + } + + @Test + public void testGetCategory() throws ParsingException { + assertEquals("", e.getCategory()); + } + + @Test + public void testGetTags() throws ParsingException { + assertEquals(0, e.getTags().size()); + } } From 6822fe3dc8c1502c7dd83d0a0dcd6154ce505cfd Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 13:03:16 +0200 Subject: [PATCH 51/91] Bandcamp: Get Stream upload date --- .../bandcamp/extractors/BandcampStreamExtractor.java | 2 +- .../services/bandcamp/BandcampStreamExtractorTest.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index b6ed0a5da..b33670ecc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -96,7 +96,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nullable @Override public String getTextualUploadDate() { - return current.getString("release_date"); + return current.getString("publish_date").replaceAll(" \\d+:\\d+:\\d+ .+", ""); } @Nullable diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index c4d268226..c280bcb77 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -60,6 +60,11 @@ public class BandcampStreamExtractorTest { assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); } + @Test + public void testUploadDate() { + assertEquals("27 Sep 2019", extractor.getTextualUploadDate()); + } + @Test public void testNoArtistProfilePicture() throws ExtractionException { assertEquals("", Bandcamp.getStreamExtractor("https://powertothequeerkids.bandcamp.com/track/human-nature").getUploaderAvatarUrl()); From 1be20ceeec6a06aa1030f3912545428e2137397c Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 14:09:21 +0200 Subject: [PATCH 52/91] Bandcamp: parse date --- .../BandcampRadioStreamExtractor.java | 2 +- .../extractors/BandcampStreamExtractor.java | 18 +++++++++++++++--- .../BandcampRadioStreamExtractorTest.java | 5 +++++ .../bandcamp/BandcampStreamExtractorTest.java | 5 +++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 3780671ac..8a69931c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -77,7 +77,7 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nullable @Override public String getTextualUploadDate() { - return showInfo.getString("published_date").replace(" 00:00:00 GMT", ""); + return showInfo.getString("published_date"); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index b33670ecc..1e69fa9f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -20,7 +20,11 @@ import org.schabi.newpipe.extractor.stream.*; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; import java.util.List; import java.util.Locale; @@ -96,13 +100,21 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nullable @Override public String getTextualUploadDate() { - return current.getString("publish_date").replaceAll(" \\d+:\\d+:\\d+ .+", ""); + return current.getString("publish_date"); } @Nullable @Override - public DateWrapper getUploadDate() { - return null; + public DateWrapper getUploadDate() throws ParsingException { + try { + Date date = new SimpleDateFormat("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH).parse(getTextualUploadDate()); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + return new DateWrapper(calendar, false); + } catch (ParseException e) { + e.printStackTrace(); + throw new ParsingException("Could not extract date", e); + } } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 2da8ec913..7dd371c28 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamE import org.schabi.newpipe.extractor.stream.StreamExtractor; import java.io.IOException; +import java.util.Calendar; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; @@ -57,7 +58,11 @@ public class BandcampRadioStreamExtractorTest { @Test public void testGetTextualUploadDate() throws ParsingException { assertEquals("16 May 2017", e.getTextualUploadDate()); + } + @Test + public void testUploadDate() throws ParsingException { + assertEquals(136, e.getUploadDate().date().get(Calendar.DAY_OF_YEAR)); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index c280bcb77..7c8d4c863 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamE import org.schabi.newpipe.extractor.stream.StreamExtractor; import java.io.IOException; +import java.util.Calendar; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -61,8 +62,8 @@ public class BandcampStreamExtractorTest { } @Test - public void testUploadDate() { - assertEquals("27 Sep 2019", extractor.getTextualUploadDate()); + public void testUploadDate() throws ParsingException { + assertEquals(270, extractor.getUploadDate().date().get(Calendar.DAY_OF_YEAR)); } @Test From 34b6928124bd0860061deea2e38cbd24dd40085c Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 14:25:38 +0200 Subject: [PATCH 53/91] Don't print stacktrace before throwing new exception --- .../services/bandcamp/extractors/BandcampStreamExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 1e69fa9f1..62b85f1d3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -112,7 +112,6 @@ public class BandcampStreamExtractor extends StreamExtractor { calendar.setTime(date); return new DateWrapper(calendar, false); } catch (ParseException e) { - e.printStackTrace(); throw new ParsingException("Could not extract date", e); } } From ea49202f64e11198a959a51c396b4bbd68370acb Mon Sep 17 00:00:00 2001 From: fynngodau Date: Thu, 4 Jun 2020 18:26:58 +0200 Subject: [PATCH 54/91] Don't display internal license id Co-authored-by: bopol --- .../services/bandcamp/extractors/BandcampStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 62b85f1d3..1ed353771 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -303,7 +303,7 @@ public class BandcampStreamExtractor extends StreamExtractor { case 6: return "CC BY 3.0"; default: - return "Unknown license (internal ID " + license + ")"; + return "Unknown"; } } From 27e7aad159c9081b6af2b84430a6f50bc01ab987 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:04:25 +0200 Subject: [PATCH 55/91] Bandcamp: Improve code style --- .../services/bandcamp/extractors/BandcampExtractorHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index cfe5c2df2..c9e21b31f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -9,6 +9,7 @@ import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.util.ArrayList; @@ -87,7 +88,7 @@ public class BandcampExtractorHelper { // Remove empty strings ArrayList list = new ArrayList<>(Arrays.asList(strings)); for (int i = list.size() - 1; i >= 0; i--) { - if (list.get(i) == null || list.get(i).isEmpty() || list.get(i).equals("null")) { + if (Utils.isNullOrEmpty(list) || list.get(i).equals("null")) { list.remove(i); } } From e6ecd919297ef9f75a67834c5b9a519113c0c7bb Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:12:10 +0200 Subject: [PATCH 56/91] Bandcamp: Test full date --- .../bandcamp/BandcampRadioStreamExtractorTest.java | 12 ++++++++++-- .../bandcamp/BandcampStreamExtractorTest.java | 9 ++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 7dd371c28..0ada81f33 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import java.io.IOException; import java.util.Calendar; +import java.util.TimeZone; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; @@ -57,12 +58,19 @@ public class BandcampRadioStreamExtractorTest { @Test public void testGetTextualUploadDate() throws ParsingException { - assertEquals("16 May 2017", e.getTextualUploadDate()); + assertEquals("16 May 2017 00:00:00 GMT", e.getTextualUploadDate()); } @Test public void testUploadDate() throws ParsingException { - assertEquals(136, e.getUploadDate().date().get(Calendar.DAY_OF_YEAR)); + final Calendar expectedCalendar = Calendar.getInstance(); + + // 16 May 2017 00:00:00 GMT + expectedCalendar.setTimeZone(TimeZone.getTimeZone("GMT")); + expectedCalendar.setTimeInMillis(0); + expectedCalendar.set(2017, Calendar.MAY, 16); + + assertEquals(expectedCalendar.getTimeInMillis(), e.getUploadDate().date().getTimeInMillis()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 7c8d4c863..c631ca964 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -14,6 +14,7 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import java.io.IOException; import java.util.Calendar; +import java.util.TimeZone; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -63,7 +64,13 @@ public class BandcampStreamExtractorTest { @Test public void testUploadDate() throws ParsingException { - assertEquals(270, extractor.getUploadDate().date().get(Calendar.DAY_OF_YEAR)); + final Calendar expectedCalendar = Calendar.getInstance(); + // 27 Sep 2019 21:49:14 GMT + expectedCalendar.setTimeZone(TimeZone.getTimeZone("GMT")); + expectedCalendar.set(2019, Calendar.SEPTEMBER, 27, 21, 49, 14); + expectedCalendar.set(Calendar.MILLISECOND, 0); + + assertEquals(expectedCalendar.getTimeInMillis(), extractor.getUploadDate().date().getTimeInMillis()); } @Test From 0950a955775aca216099f0bc9cba6d38b27fac5a Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:16:08 +0200 Subject: [PATCH 57/91] Fix for testing wrong object for null or empty --- .../services/bandcamp/extractors/BandcampExtractorHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index c9e21b31f..37c4ff96b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -88,7 +88,7 @@ public class BandcampExtractorHelper { // Remove empty strings ArrayList list = new ArrayList<>(Arrays.asList(strings)); for (int i = list.size() - 1; i >= 0; i--) { - if (Utils.isNullOrEmpty(list) || list.get(i).equals("null")) { + if (Utils.isNullOrEmpty(list.get(i)) || list.get(i).equals("null")) { list.remove(i); } } From 8fa81537c0befa7664a69114b80b989716578fd3 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:17:51 +0200 Subject: [PATCH 58/91] Better bandcamp stream extractor description test --- .../services/bandcamp/BandcampStreamExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index c631ca964..11b3fa486 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -53,7 +53,7 @@ public class BandcampStreamExtractorTest { @Test public void testDescription() { - assertEquals(831, extractor.getDescription().getContent().length()); + assertTrue(extractor.getDescription().getContent().contains("Boy, you've taken up my thoughts")); } @Test From 52103ac61fb84a6fe7bfade499baebd3e360303a Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:27:13 +0200 Subject: [PATCH 59/91] Remove "useless" comments --- .../bandcamp/BandcampChannelLinkHandlerFactoryTest.java | 2 -- .../bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java | 2 -- .../bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java | 2 -- .../services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java | 2 -- .../services/bandcamp/BandcampSuggestionExtractorTest.java | 1 - 5 files changed, 9 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index 191f096c0..ad48da1c0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -26,13 +26,11 @@ public class BandcampChannelLinkHandlerFactoryTest { @Test public void testAcceptUrl() throws ParsingException { - // Tests expecting true assertTrue(linkHandler.acceptUrl("http://interovgm.com/releases/")); assertTrue(linkHandler.acceptUrl("https://interovgm.com/releases")); assertTrue(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); - // Tests expecting false assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java index 3fd70369d..38559782b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java @@ -24,13 +24,11 @@ public class BandcampFeaturedLinkHandlerFactoryTest { @Test public void testAcceptUrl() throws ParsingException { - // Tests expecting true assertTrue(linkHandler.acceptUrl("https://bandcamp.com/?show=1")); assertTrue(linkHandler.acceptUrl("http://bandcamp.com/?show=2")); assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/mobile/24/bootstrap_data")); assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/bcweekly/1/list")); - // Tests expecting false assertFalse(linkHandler.acceptUrl("https://bandcamp.com/?show=")); assertFalse(linkHandler.acceptUrl("https://bandcamp.com/?show=a")); assertFalse(linkHandler.acceptUrl("https://bandcamp.com/")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java index f9cf9b1a7..11fe3acd7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java @@ -28,7 +28,6 @@ public class BandcampPlaylistLinkHandlerFactoryTest { @Test public void testAcceptUrl() throws ParsingException { - // Tests expecting false assertFalse(linkHandler.acceptUrl("http://interovgm.com/releases/")); assertFalse(linkHandler.acceptUrl("https://interovgm.com/releases")); assertFalse(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); @@ -37,7 +36,6 @@ public class BandcampPlaylistLinkHandlerFactoryTest { assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); assertFalse(linkHandler.acceptUrl("https://interovgm.com/track/title")); - // Tests expecting true assertTrue(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids")); assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/album/prom")); assertTrue(linkHandler.acceptUrl("https://MACBENSON.BANDCAMP.COM/ALBUM/COMING-OF-AGE")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java index 3942fb1f1..69a7c2bf1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -37,7 +37,6 @@ public class BandcampStreamLinkHandlerFactoryTest { @Test public void testAcceptUrl() throws ParsingException { - // Tests expecting false assertFalse(linkHandler.acceptUrl("http://interovgm.com/releases/")); assertFalse(linkHandler.acceptUrl("https://interovgm.com/releases")); assertFalse(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); @@ -45,7 +44,6 @@ public class BandcampStreamLinkHandlerFactoryTest { assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); assertFalse(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids")); - // Tests expecting true assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); assertTrue(linkHandler.acceptUrl("https://interovgm.com/track/title")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java index 68cdae518..393ca01c6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java @@ -32,7 +32,6 @@ public class BandcampSuggestionExtractorTest { public void testSearchExample() throws IOException, ExtractionException { List c418 = extractor.suggestionList("c418"); - // When looking for c418, one should find C418 assertTrue(c418.contains("C418")); // There should be five results, but we can't be sure of that forever From 4dd9540782a3c3bb831a87a12aa3d7cb25e16e0e Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:27:39 +0200 Subject: [PATCH 60/91] Bandcamp license switch order --- .../services/bandcamp/extractors/BandcampStreamExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 1ed353771..11bbc2ec4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -298,10 +298,10 @@ public class BandcampStreamExtractor extends StreamExtractor { return "CC BY-NC 3.0"; case 5: return "CC BY-ND 3.0"; - case 8: - return "CC BY-SA 3.0"; case 6: return "CC BY 3.0"; + case 8: + return "CC BY-SA 3.0"; default: return "Unknown"; } From e13f341a458f7bc8b6a7d54026a415f8758eff07 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:29:37 +0200 Subject: [PATCH 61/91] Bandcamp stream extractor: test service id --- .../extractor/services/bandcamp/BandcampStreamExtractorTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 11b3fa486..423a08fd1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -34,6 +34,7 @@ public class BandcampStreamExtractorTest { @Test public void testServiceId() { + assertEquals(4, extractor.getServiceId()); } @Test From 9a555d97e5fcd4c0a74a77014ab53f7a82ef6b3a Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:31:01 +0200 Subject: [PATCH 62/91] Remove useless bandcamp tests --- .../BandcampRadioStreamExtractorTest.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 0ada81f33..d9c2b9839 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -99,19 +99,4 @@ public class BandcampRadioStreamExtractorTest { public void testGetAudioStreams() throws ExtractionException, IOException { assertEquals(2, e.getAudioStreams().size()); } - - @Test - public void testGetLicence() throws ParsingException { - assertEquals("", e.getLicence()); - } - - @Test - public void testGetCategory() throws ParsingException { - assertEquals("", e.getCategory()); - } - - @Test - public void testGetTags() throws ParsingException { - assertEquals(0, e.getTags().size()); - } } From 39b55b5b42d39dabf0f2cedc463e002bbc254ca8 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:32:19 +0200 Subject: [PATCH 63/91] Bandcamp radio stream extractor: test uploader url --- .../services/bandcamp/BandcampRadioStreamExtractorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index d9c2b9839..5b35a19b3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -43,7 +43,8 @@ public class BandcampRadioStreamExtractorTest { } @Test - public void testGetUploaderUrl() { + public void testGetUploaderUrl() throws ParsingException { + assertEquals("http://bandcamp.com/andrewjervis", e.getUploaderUrl()); } @Test From 96de834b676e26d0813ff4816f88fbbce78e5f49 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 4 Jun 2020 19:36:58 +0200 Subject: [PATCH 64/91] Bandcamp: parse date from radio info item --- .../extractors/BandcampExtractorHelper.java | 17 +++++++++++++++-- .../BandcampRadioInfoItemExtractor.java | 6 +++--- .../extractors/BandcampStreamExtractor.java | 9 +-------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 37c4ff96b..707eaf062 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -9,11 +9,13 @@ import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; public class BandcampExtractorHelper { @@ -142,4 +144,15 @@ public class BandcampExtractorHelper { public static String getImageUrl(long id, boolean album) { return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; } + + static DateWrapper parseDate(String textDate) throws ParsingException { + try { + Date date = new SimpleDateFormat("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH).parse(textDate); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + return new DateWrapper(calendar, false); + } catch (ParseException e) { + throw new ParsingException("Could not extract date", e); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index ebc843195..3e6dadf01 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -33,13 +33,13 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public String getTextualUploadDate() { - return show.getString("date").replace(" 00:00:00 GMT", ""); + return show.getString("date"); } @Nullable @Override - public DateWrapper getUploadDate() { - return null; + public DateWrapper getUploadDate() throws ParsingException { + return BandcampExtractorHelper.parseDate(getTextualUploadDate()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 11bbc2ec4..282259275 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -106,14 +106,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - try { - Date date = new SimpleDateFormat("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH).parse(getTextualUploadDate()); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - return new DateWrapper(calendar, false); - } catch (ParseException e) { - throw new ParsingException("Could not extract date", e); - } + return BandcampExtractorHelper.parseDate(getTextualUploadDate()); } @Nonnull From 932d094d6ab85ab4c439e1df07c53a312fe35c2e Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 2 Aug 2020 16:55:18 +0200 Subject: [PATCH 65/91] Upgrade featured playlist urls to https --- .../BandcampPlaylistInfoItemFeaturedExtractor.java | 3 +-- .../services/bandcamp/BandcampFeaturedExtractorTest.java | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java index 814794284..968bff22a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import com.grack.nanojson.JsonObject; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; @@ -31,7 +30,7 @@ public class BandcampPlaylistInfoItemFeaturedExtractor implements PlaylistInfoIt @Override public String getUrl() { - return featuredStory.getString("item_url"); + return featuredStory.getString("item_url").replaceAll("http://", "https://"); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index 09f516be9..f0502d579 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -37,4 +37,10 @@ public class BandcampFeaturedExtractorTest { assertTrue(list.size() > 1); } + @Test + public void testHttps() throws ExtractionException, IOException { + List list = extractor.getInitialPage().getItems(); + assertTrue(list.get(0).getUrl().contains("https://")); + } + } From cfe88a74c1a9eb692ba4f5c1281a47dd04755f5b Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 9 Aug 2020 10:07:49 +0200 Subject: [PATCH 66/91] Throw ContentNotSupportedException when opening radio uploader channel --- .../bandcamp/extractors/BandcampRadioStreamExtractor.java | 6 +++--- .../bandcamp/BandcampRadioStreamExtractorTest.java | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 360009ea5..3500b862f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -56,9 +57,8 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nonnull @Override - public String getUploaderUrl() { - return Jsoup.parse(showInfo.getString("image_caption")) - .getElementsByTag("a").first().attr("href").trim(); + public String getUploaderUrl() throws ContentNotSupportedException { + throw new ContentNotSupportedException("Fan pages are not supported"); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 5b35a19b3..ecbed677c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -3,12 +3,11 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import java.io.IOException; @@ -42,9 +41,9 @@ public class BandcampRadioStreamExtractorTest { assertEquals("Sound Movements", e.getName()); } - @Test + @Test(expected = ContentNotSupportedException.class) public void testGetUploaderUrl() throws ParsingException { - assertEquals("http://bandcamp.com/andrewjervis", e.getUploaderUrl()); + e.getUploaderUrl(); } @Test From 81b5e7cf3d22015d58ef4817ec687048473dcebe Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 8 Oct 2020 17:56:03 +0200 Subject: [PATCH 67/91] Fix extractor --- .../extractors/BandcampExtractorHelper.java | 37 ++++--------------- .../extractors/BandcampPlaylistExtractor.java | 4 +- .../extractors/BandcampStreamExtractor.java | 10 ++--- .../BandcampChannelLinkHandlerFactory.java | 2 +- 4 files changed, 14 insertions(+), 39 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 707eaf062..a239c440a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -6,6 +6,8 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -20,7 +22,7 @@ import java.util.*; public class BandcampExtractorHelper { /** - *

Get JSON behind var $variable = out of web page

+ *

Get an attribute of a web page as JSON * *

Originally a part of bandcampDirect.

* @@ -29,35 +31,10 @@ public class BandcampExtractorHelper { * @param variable Name of the variable * @return The JsonObject stored in the variable with this name */ - public static JsonObject getJSONFromJavaScriptVariables(String html, String variable) throws JsonParserException, ArrayIndexOutOfBoundsException, ParsingException { - - String[] part = html.split("var " + variable + " = "); - - String firstHalfGone = part[1]; - - firstHalfGone = firstHalfGone.replaceAll("\" \\+ \"", ""); - - int position = -1; - int level = 0; - for (char character : firstHalfGone.toCharArray()) { - position++; - - switch (character) { - case '{': - level++; - continue; - case '}': - level--; - if (level == 0) { - return JsonParser.object().from(firstHalfGone.substring(0, position + 1) - .replaceAll(" {4}//.+", "") // Remove "for the curious" in JSON - .replaceAll("// xxx: note - don't internationalize this variable", "") // Remove this comment - ); - } - } - } - - throw new ParsingException("Unexpected HTML: JSON never ends"); + public static JsonObject getJsonData(String html, String variable) throws JsonParserException, ArrayIndexOutOfBoundsException, ParsingException { + Document document = Jsoup.parse(html); + String json = document.getElementsByAttribute(variable).attr(variable); + return JsonParser.object().from(json); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index f06d51d38..b7e812695 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -21,7 +21,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getJSONFromJavaScriptVariables; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getJsonData; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; public class BandcampPlaylistExtractor extends PlaylistExtractor { @@ -50,7 +50,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { trackInfo = albumJson.getArray("trackinfo"); try { - name = getJSONFromJavaScriptVariables(html, "EmbedData").getString("album_title"); + name = getJsonData(html, "data-embed").getString("album_title"); } catch (JsonParserException e) { throw new ParsingException("Faulty JSON; page likely does not contain album data", e); } catch (ArrayIndexOutOfBoundsException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index a0da730fc..ae172ed15 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -20,11 +20,7 @@ import org.schabi.newpipe.extractor.stream.*; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; import java.util.List; import java.util.Locale; @@ -63,7 +59,7 @@ public class BandcampStreamExtractor extends StreamExtractor { */ public static JsonObject getAlbumInfoJson(String html) throws ParsingException { try { - return BandcampExtractorHelper.getJSONFromJavaScriptVariables(html, "TralbumData"); + return BandcampExtractorHelper.getJsonData(html, "data-tralbum"); } catch (JsonParserException e) { throw new ParsingException("Faulty JSON; page likely does not contain album data", e); } catch (ArrayIndexOutOfBoundsException e) { @@ -264,7 +260,9 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public String getCategory() { // Get first tag from html, which is the artist's Genre - return document.getElementsByAttributeValue("itemprop", "keywords").first().text(); + return document + .getElementsByClass("tralbum-tags").first() + .getElementsByClass("tag").first().text(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index 370057dde..825add10b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -25,7 +25,7 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { String response = NewPipe.getDownloader().get(url).responseBody(); // This variable contains band data! - JsonObject bandData = BandcampExtractorHelper.getJSONFromJavaScriptVariables(response, "BandData"); + JsonObject bandData = BandcampExtractorHelper.getJsonData(response, "data-band"); return String.valueOf(bandData.getLong("id")); From 186936d04151d5d8030a46acbc0376560149f387 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Thu, 19 Nov 2020 22:33:52 +0100 Subject: [PATCH 68/91] Various changes regarding tests --- .../BandcampRadioInfoItemExtractor.java | 1 + .../extractors/BandcampStreamExtractor.java | 5 +- ...ndcampPlaylistStreamInfoItemExtractor.java | 5 +- .../BandcampChannelExtractorTest.java | 4 +- ...BandcampChannelLinkHandlerFactoryTest.java | 1 + .../BandcampFeaturedExtractorTest.java | 39 +++- .../BandcampPlaylistExtractorTest.java | 52 ++++- .../bandcamp/BandcampRadioExtractorTest.java | 41 +++- .../bandcamp/BandcampSearchExtractorTest.java | 2 +- .../bandcamp/BandcampStreamExtractorTest.java | 190 ++++++++++++------ 10 files changed, 256 insertions(+), 84 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index 3e6dadf01..f478d985e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -69,6 +69,7 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUploaderName() { + // JSON does not contain uploader name return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index ae172ed15..1561a98ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -21,6 +21,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; @@ -209,12 +210,12 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public List getVideoStreams() { - return null; + return Collections.EMPTY_LIST; } @Override public List getVideoOnlyStreams() { - return null; + return Collections.EMPTY_LIST; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index 6faf397ae..15e5116e9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -15,7 +15,7 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI private final JsonObject track; private String substituteCoverUrl; - private StreamingService service; + private final StreamingService service; public BandcampPlaylistStreamInfoItemExtractor(JsonObject track, String uploaderUrl, StreamingService service) { super(uploaderUrl); @@ -45,6 +45,9 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI @Override public String getUploaderName() { + /* Tracks can have an individual artist name, but it is not included in the + * given JSON. + */ return ""; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 2d1c38e30..a5ccabf9d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -70,8 +70,8 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { } @Override - public void testServiceId() throws Exception { - assertEquals(4, extractor.getServiceId()); + public void testServiceId() { + assertEquals(Bandcamp.getServiceId(), extractor.getServiceId()); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index ad48da1c0..e07daf7d1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -40,6 +40,7 @@ public class BandcampChannelLinkHandlerFactoryTest { assertEquals("1196681540", linkHandler.getId("https://macbenson.bandcamp.com/")); assertEquals("1581461772", linkHandler.getId("https://interovgm.com/releases")); assertEquals("3321800855", linkHandler.getId("https://infiniteammo.bandcamp.com/")); + assertEquals("3775652329", linkHandler.getId("https://npet.bandcamp.com/")); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index f0502d579..1668445c8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -9,18 +9,21 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; +import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor; import java.io.IOException; import java.util.List; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** * Tests for {@link BandcampFeaturedExtractor} */ -public class BandcampFeaturedExtractorTest { +public class BandcampFeaturedExtractorTest implements BaseListExtractorTest { private static BandcampFeaturedExtractor extractor; @@ -43,4 +46,38 @@ public class BandcampFeaturedExtractorTest { assertTrue(list.get(0).getUrl().contains("https://")); } + @Override + public void testRelatedItems() throws Exception { + DefaultTests.defaultTestRelatedItems(extractor); + } + + @Override + public void testMoreRelatedItems() throws Exception { + // more items not implemented + } + + @Override + public void testServiceId() { + assertEquals(Bandcamp.getServiceId(), extractor.getServiceId()); + } + + @Override + public void testName() throws Exception { + assertEquals("Featured", extractor.getName()); + } + + @Override + public void testId() { + assertEquals("", extractor.getId()); + } + + @Override + public void testUrl() throws Exception { + assertEquals("", extractor.getUrl()); + } + + @Override + public void testOriginalUrl() throws Exception { + assertEquals("", extractor.getOriginalUrl()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index 1b1039e6f..c360c23a7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -10,6 +10,8 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -90,7 +92,7 @@ public class BandcampPlaylistExtractorTest { } - public static class ComingOfAge { + public static class ComingOfAge implements BasePlaylistExtractorTest { private static PlaylistExtractor extractor; @@ -102,48 +104,78 @@ public class BandcampPlaylistExtractorTest { } @Test - public void getThumbnailUrl() throws ParsingException { + public void testThumbnailUrl() throws ParsingException { assertTrue(extractor.getThumbnailUrl().contains("f4.bcbits.com/img")); } @Test - public void getBannerUrl() throws ParsingException { + public void testBannerUrl() throws ParsingException { assertEquals("", extractor.getBannerUrl()); } @Test - public void getUploaderUrl() throws ParsingException { + public void testUploaderUrl() throws ParsingException { assertTrue(extractor.getUploaderUrl().contains("macbenson.bandcamp.com")); } @Test - public void getUploaderName() throws ParsingException { + public void testUploaderName() throws ParsingException { assertEquals("mac benson", extractor.getUploaderName()); } @Test - public void getUploaderAvatarUrl() throws ParsingException { + public void testUploaderAvatarUrl() throws ParsingException { assertTrue(extractor.getUploaderAvatarUrl().contains("f4.bcbits.com/img")); } @Test - public void getStreamCount() throws ParsingException { + public void testStreamCount() throws ParsingException { assertEquals(5, extractor.getStreamCount()); } @Test - public void getInitialPage() throws IOException, ExtractionException { + public void testInitialPage() throws IOException, ExtractionException { assertNotNull(extractor.getInitialPage().getItems().get(0)); } @Test - public void getName() throws ParsingException { + public void testServiceId() { + assertEquals(Bandcamp.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { assertEquals("Coming of Age", extractor.getName()); } @Test - public void getNextPageUrl() throws IOException, ExtractionException { + public void testId() throws Exception { + assertEquals("https://macbenson.bandcamp.com/album/coming-of-age", extractor.getId()); + } + + @Test + public void testUrl() throws Exception { + assertEquals("https://macbenson.bandcamp.com/album/coming-of-age", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws Exception { + assertEquals("https://macbenson.bandcamp.com/album/coming-of-age", extractor.getOriginalUrl()); + } + + @Test + public void testNextPageUrl() throws IOException, ExtractionException { assertNull(extractor.getPage(extractor.getInitialPage().getNextPage())); } + + @Test + public void testRelatedItems() throws Exception { + //DefaultTests.defaultTestRelatedItems(extractor); + // Would fail because BandcampPlaylistStreamInfoItemExtractor.getUploaderName() returns an empty String + } + + @Test + public void testMoreRelatedItems() throws Exception { + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java index c1f493c92..76f0559d0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java @@ -8,18 +8,21 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; +import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor; import java.io.IOException; import java.util.List; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; /** * Tests for {@link BandcampRadioExtractor} */ -public class BandcampRadioExtractorTest { +public class BandcampRadioExtractorTest implements BaseListExtractorTest { private static BandcampRadioExtractor extractor; @@ -36,4 +39,40 @@ public class BandcampRadioExtractorTest { List list = Bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); assertTrue(list.size() > 300); } + + @Test + public void testRelatedItems() throws Exception { + //DefaultTests.defaultTestRelatedItems(extractor); + // Would fail because BandcampRadioInfoItemExtractor.getUploaderName() returns an empty String + } + + @Test + public void testMoreRelatedItems() throws Exception { + // All items are on one page + } + + @Test + public void testServiceId() { + assertEquals(Bandcamp.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("Radio", extractor.getName()); + } + + @Test + public void testId() { + assertEquals("Radio", extractor.getId()); + } + + @Test + public void testUrl() throws Exception { + assertEquals("https://bandcamp.com/api/bcweekly/1/list", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws Exception { + assertEquals("https://bandcamp.com/api/bcweekly/1/list", extractor.getOriginalUrl()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 7d7655190..786839995 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -107,7 +107,7 @@ public class BandcampSearchExtractorTest { public static class DefaultTest extends DefaultSearchExtractorTest { private static SearchExtractor extractor; - private static final String QUERY = "test"; + private static final String QUERY = "noise"; @BeforeClass public static void setUp() throws Exception { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 423a08fd1..4c14bea5d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -6,103 +6,161 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; import java.util.Calendar; +import java.util.Collections; +import java.util.List; import java.util.TimeZone; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; -public class BandcampStreamExtractorTest { +/** + * Tests for {@link BandcampStreamExtractor} + */ +public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { private static BandcampStreamExtractor extractor; @BeforeClass - public static void setUp() throws Exception { + public static void setUp() throws ExtractionException, IOException { NewPipe.init(DownloaderTestImpl.getInstance()); + // This test track was uploaded specifically for NewPipeExtractor tests + extractor = (BandcampStreamExtractor) Bandcamp - .getStreamExtractor("https://zachbenson.bandcamp.com/track/kitchen"); + .getStreamExtractor("https://npet.bandcamp.com/track/track-1"); extractor.fetchPage(); } - @Test - public void testServiceId() { - assertEquals(4, extractor.getServiceId()); + @Override + public StreamExtractor extractor() { + return extractor; + } + + @Override + public StreamingService expectedService() { + return Bandcamp; + } + + @Override + public String expectedName() { + return "Track #1"; + } + + @Override + public String expectedId() { + return "https://npet.bandcamp.com/track/track-1"; + } + + @Override + public String expectedUrlContains() { + return "https://npet.bandcamp.com/track/track-1"; + } + + @Override + public String expectedOriginalUrlContains() { + return "https://npet.bandcamp.com/track/track-1"; + } + + @Override + public StreamType expectedStreamType() { + return StreamType.AUDIO_STREAM; + } + + @Override + public String expectedUploaderName() { + return "NewPipeExtractorTest"; + } + + @Override + public String expectedUploaderUrl() { + return "https://npet.bandcamp.com/"; + } + + @Override + public List expectedDescriptionContains() { + return Collections.singletonList("This sample track was created using MuseScore."); + } + + @Override + public long expectedLength() { + return 0; + } + + @Override + public long expectedViewCountAtLeast() { + return Long.MIN_VALUE; + } + + @Override + public String expectedUploadDate() { + return "2020-03-17 18:37:44.000"; + } + + @Override + public String expectedTextualUploadDate() { + return "17 Mar 2020 18:37:44 GMT"; + } + + @Override + public long expectedLikeCountAtLeast() { + return Long.MIN_VALUE; + } + + @Override + public long expectedDislikeCountAtLeast() { + return Long.MIN_VALUE; + } + + @Override + public boolean expectedHasVideoStreams() { + return false; + } + + @Override + public boolean expectedHasRelatedStreams() { + return false; + } + + @Override + public boolean expectedHasSubtitles() { + return false; + } + + @Override + public boolean expectedHasFrames() { + return false; + } + + @Override + public String expectedLicence() { + return "CC BY 3.0"; + } + + @Override + public String expectedCategory() { + return "acoustic"; } @Test - public void testName() throws ParsingException { - assertEquals("kitchen", extractor.getName()); - } - - @Test - public void testUrl() throws ParsingException { - assertEquals("https://zachbenson.bandcamp.com/track/kitchen", extractor.getUrl()); - } - - @Test - public void testArtistUrl() throws ParsingException { - assertEquals("https://zachbenson.bandcamp.com/", extractor.getUploaderUrl()); - } - - @Test - public void testDescription() { - assertTrue(extractor.getDescription().getContent().contains("Boy, you've taken up my thoughts")); - } - - @Test - public void testArtistProfilePicture() { - String url = extractor.getUploaderAvatarUrl(); + public void testArtistProfilePicture() throws Exception { + String url = extractor().getUploaderAvatarUrl(); assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); } - @Test - public void testUploadDate() throws ParsingException { - final Calendar expectedCalendar = Calendar.getInstance(); - // 27 Sep 2019 21:49:14 GMT - expectedCalendar.setTimeZone(TimeZone.getTimeZone("GMT")); - expectedCalendar.set(2019, Calendar.SEPTEMBER, 27, 21, 49, 14); - expectedCalendar.set(Calendar.MILLISECOND, 0); - - assertEquals(expectedCalendar.getTimeInMillis(), extractor.getUploadDate().date().getTimeInMillis()); - } - - @Test - public void testNoArtistProfilePicture() throws ExtractionException { - assertEquals("", Bandcamp.getStreamExtractor("https://powertothequeerkids.bandcamp.com/track/human-nature").getUploaderAvatarUrl()); - } - - @Test - public void testAudioStream() { - assertTrue(extractor.getAudioStreams().get(0).getUrl().contains("bcbits.com/stream")); - assertEquals(1, extractor.getAudioStreams().size()); - } - - @Test - public void testCategory() throws ExtractionException, IOException { - StreamExtractor se = Bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); - se.fetchPage(); - assertEquals("acoustic", se.getCategory()); - } - - @Test - public void testLicense() throws ExtractionException, IOException { - StreamExtractor se = Bandcamp.getStreamExtractor("https://npet.bandcamp.com/track/track-1"); - se.fetchPage(); - assertEquals("CC BY 3.0", se.getLicence()); - } - @Test public void testTranslateIdsToUrl() throws ParsingException { - assertEquals("https://zachbenson.bandcamp.com/album/covers", BandcampExtractorHelper.getStreamUrlFromIds(2862267535L, 2063639444L, "album")); - // TODO write more test cases + assertEquals("https://npet.bandcamp.com/track/track-1", BandcampExtractorHelper.getStreamUrlFromIds(3775652329L, 4207805220L, "track")); } } From 8c369b0f79d3ea01bf97a7d8f71f49d9c31b056d Mon Sep 17 00:00:00 2001 From: fynngodau Date: Tue, 24 Nov 2020 14:01:31 +0100 Subject: [PATCH 69/91] Rephrase link in javadoc Co-authored-by: Tobias Groza --- .../services/bandcamp/extractors/BandcampExtractorHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index a239c440a..c93db96d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -90,7 +90,7 @@ public class BandcampExtractorHelper { /** * Fetch artist details from mobile endpoint. * - * I once took a moment to note down how it works. + * More technical info. */ public static JsonObject getArtistDetails(String id) throws ParsingException { try { From 99e7ef013efa6b1da489262cba6f7d7e141e2d4e Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 24 Nov 2020 09:41:40 +0100 Subject: [PATCH 70/91] [Bandcamp] Apply small changes to code formatting and style Make variables final when possible Remove unused imports Improve code formatting --- .../services/bandcamp/BandcampService.java | 36 ++++++++----------- .../extractors/BandcampChannelExtractor.java | 32 +++++++++-------- .../BandcampChannelInfoItemExtractor.java | 8 +++-- .../extractors/BandcampExtractorHelper.java | 35 +++++++++--------- .../extractors/BandcampFeaturedExtractor.java | 13 +++---- .../extractors/BandcampPlaylistExtractor.java | 21 ++++++----- .../BandcampPlaylistInfoItemExtractor.java | 9 ++--- ...campPlaylistInfoItemFeaturedExtractor.java | 2 +- .../extractors/BandcampRadioExtractor.java | 18 +++++----- .../BandcampRadioInfoItemExtractor.java | 2 +- .../BandcampRadioStreamExtractor.java | 12 +++---- .../extractors/BandcampSearchExtractor.java | 23 ++++++------ .../extractors/BandcampStreamExtractor.java | 26 +++++++------- .../BandcampSuggestionExtractor.java | 18 +++++----- ...campDiscographStreamInfoItemExtractor.java | 2 +- ...ndcampPlaylistStreamInfoItemExtractor.java | 10 +++--- ...BandcampSearchStreamInfoItemExtractor.java | 12 ++++--- .../BandcampStreamInfoItemExtractor.java | 2 +- .../BandcampChannelLinkHandlerFactory.java | 15 ++++---- .../BandcampFeaturedLinkHandlerFactory.java | 27 ++++++++------ .../BandcampPlaylistLinkHandlerFactory.java | 7 ++-- .../BandcampSearchQueryHandlerFactory.java | 5 +-- .../BandcampStreamLinkHandlerFactory.java | 18 ++++++---- 23 files changed, 187 insertions(+), 166 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index a94638a3f..1d96684b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -6,7 +6,6 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; @@ -17,7 +16,6 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -import java.io.IOException; import java.util.Collections; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; @@ -28,7 +26,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp public class BandcampService extends StreamingService { - public BandcampService(int id) { + public BandcampService(final int id) { super(id, "Bandcamp", Collections.singletonList(AUDIO)); } @@ -63,7 +61,7 @@ public class BandcampService extends StreamingService { } @Override - public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { + public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) { return new BandcampSearchExtractor(this, queryHandler); } @@ -83,24 +81,20 @@ public class BandcampService extends StreamingService { KioskList kioskList = new KioskList(this); try { - kioskList.addKioskEntry(new KioskList.KioskExtractorFactory() { - @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException { - return new BandcampFeaturedExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), kioskId); - } - }, new BandcampFeaturedLinkHandlerFactory(), KIOSK_FEATURED); + kioskList.addKioskEntry((streamingService, url, kioskId) -> + new BandcampFeaturedExtractor( + BandcampService.this, + new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), kioskId), + new BandcampFeaturedLinkHandlerFactory(), KIOSK_FEATURED); - - kioskList.addKioskEntry(new KioskList.KioskExtractorFactory() { - @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException { - return new BandcampRadioExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL), kioskId); - } - }, new BandcampFeaturedLinkHandlerFactory(), KIOSK_RADIO); + kioskList.addKioskEntry((streamingService, url, kioskId) -> + new BandcampRadioExtractor(BandcampService.this, + new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL), kioskId), + new BandcampFeaturedLinkHandlerFactory(), KIOSK_RADIO); kioskList.setDefaultKiosk(KIOSK_FEATURED); - } catch (Exception e) { + } catch (final Exception e) { throw new ExtractionException(e); } @@ -108,17 +102,17 @@ public class BandcampService extends StreamingService { } @Override - public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { + public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) { return new BandcampChannelExtractor(this, linkHandler); } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { return new BandcampPlaylistExtractor(this, linkHandler); } @Override - public StreamExtractor getStreamExtractor(LinkHandler linkHandler) { + public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) { if (linkHandler.getUrl().matches("https?://bandcamp\\.com/\\?show=\\d+")) return new BandcampRadioStreamExtractor(this, linkHandler); else diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 74fa8d152..035d87a65 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -2,7 +2,8 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import com.grack.nanojson.*; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; import org.jsoup.Jsoup; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -23,7 +24,7 @@ public class BandcampChannelExtractor extends ChannelExtractor { private JsonObject channelInfo; - public BandcampChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { + public BandcampChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) { super(service, linkHandler); } @@ -34,15 +35,16 @@ public class BandcampChannelExtractor extends ChannelExtractor { return BandcampExtractorHelper.getImageUrl(channelInfo.getLong("bio_image_id"), false); } - /** - * Why does the mobile endpoint not contain the header?? Or at least not the same one? - * Anyway we're back to querying websites - */ @Override public String getBannerUrl() throws ParsingException { + /* + * Why does the mobile endpoint not contain the header?? Or at least not the same one? + * Anyway we're back to querying websites + */ try { - String html = getDownloader().get(channelInfo.getString("bandcamp_url").replace("http://", "https://")) - .responseBody(); + final String html = getDownloader() + .get(channelInfo.getString("bandcamp_url").replace("http://", "https://")) + .responseBody(); return Jsoup.parse(html) .getElementById("customHeader") @@ -50,17 +52,17 @@ public class BandcampChannelExtractor extends ChannelExtractor { .first() .attr("src"); - } catch (IOException | ReCaptchaException e) { + } catch (final IOException | ReCaptchaException e) { throw new ParsingException("Could not download artist web site", e); - } catch (NullPointerException e) { + } catch (final NullPointerException e) { // No banner available return ""; } } /** - * I had to learn bandcamp stopped providing RSS feeds when appending /feed to any URL - * because too few people used it. Bummer! + * bandcamp stopped providing RSS feeds when appending /feed to any URL + * because too few people used it. */ @Override public String getFeedUrl() { @@ -96,13 +98,13 @@ public class BandcampChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getInitialPage() throws ParsingException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray discography = channelInfo.getArray("discography"); + final JsonArray discography = channelInfo.getArray("discography"); for (int i = 0; i < discography.size(); i++) { // I define discograph as an item that can appear in a discography - JsonObject discograph = discography.getObject(i); + final JsonObject discograph = discography.getObject(i); if (!discograph.getString("item_type").equals("track")) continue; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java index 3baf45b3e..fb956791d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java @@ -10,7 +10,7 @@ public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtracto private final Element resultInfo, searchResult; - public BandcampChannelInfoItemExtractor(Element searchResult) { + public BandcampChannelInfoItemExtractor(final Element searchResult) { this.searchResult = searchResult; resultInfo = searchResult.getElementsByClass("result-info").first(); } @@ -27,11 +27,13 @@ public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtracto @Override public String getThumbnailUrl() throws ParsingException { - Element img = searchResult.getElementsByClass("art").first() + final Element img = searchResult.getElementsByClass("art").first() .getElementsByTag("img").first(); if (img != null) { return img.attr("src"); - } else return null; + } else { + return null; + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index c93db96d6..5d0fb45f7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -31,9 +31,10 @@ public class BandcampExtractorHelper { * @param variable Name of the variable * @return The JsonObject stored in the variable with this name */ - public static JsonObject getJsonData(String html, String variable) throws JsonParserException, ArrayIndexOutOfBoundsException, ParsingException { - Document document = Jsoup.parse(html); - String json = document.getElementsByAttribute(variable).attr(variable); + public static JsonObject getJsonData(final String html, final String variable) + throws JsonParserException, ArrayIndexOutOfBoundsException { + final Document document = Jsoup.parse(html); + final String json = document.getElementsByAttribute(variable).attr(variable); return JsonParser.object().from(json); } @@ -41,17 +42,18 @@ public class BandcampExtractorHelper { * Translate all these parameters together to the URL of the corresponding album or track * using the mobile api */ - public static String getStreamUrlFromIds(long bandId, long itemId, String itemType) throws ParsingException { + public static String getStreamUrlFromIds(final long bandId, final long itemId, final String itemType) + throws ParsingException { try { - String jsonString = NewPipe.getDownloader().get( + final String jsonString = NewPipe.getDownloader().get( "https://bandcamp.com/api/mobile/22/tralbum_details?band_id=" + bandId + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.substring(0, 1)) .responseBody(); return JsonParser.object().from(jsonString).getString("bandcamp_url").replace("http://", "https://"); - } catch (JsonParserException | ReCaptchaException | IOException e) { + } catch (final JsonParserException | ReCaptchaException | IOException e) { throw new ParsingException("Ids could not be translated to URL", e); } @@ -61,11 +63,11 @@ public class BandcampExtractorHelper { * Concatenate all non-null and non-empty strings together while separating them using * the comma parameter */ - public static String smartConcatenate(String[] strings, String comma) { - StringBuilder result = new StringBuilder(); + public static String smartConcatenate(final String[] strings, final String comma) { + final StringBuilder result = new StringBuilder(); // Remove empty strings - ArrayList list = new ArrayList<>(Arrays.asList(strings)); + final ArrayList list = new ArrayList<>(Arrays.asList(strings)); for (int i = list.size() - 1; i >= 0; i--) { if (Utils.isNullOrEmpty(list.get(i)) || list.get(i).equals("null")) { list.remove(i); @@ -74,8 +76,7 @@ public class BandcampExtractorHelper { // Append remaining strings to result for (int i = 0; i < list.size(); i++) { - String string = list.get(i); - result.append(string); + result.append(list.get(i)); if (i != list.size() - 1) { // This is not the last iteration yet @@ -107,7 +108,7 @@ public class BandcampExtractorHelper { .getBytes() ).responseBody() ); - } catch (IOException | ReCaptchaException | JsonParserException e) { + } catch (final IOException | ReCaptchaException | JsonParserException e) { throw new ParsingException("Could not download band details", e); } } @@ -118,17 +119,17 @@ public class BandcampExtractorHelper { * @return Url of image with this ID in size 10 which is 1200x1200 (we could also choose size 0 * but we don't want something as large as 3460x3460 here, do we?) */ - public static String getImageUrl(long id, boolean album) { + public static String getImageUrl(final long id, final boolean album) { return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; } - static DateWrapper parseDate(String textDate) throws ParsingException { + static DateWrapper parseDate(final String textDate) throws ParsingException { try { - Date date = new SimpleDateFormat("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH).parse(textDate); - Calendar calendar = Calendar.getInstance(); + final Date date = new SimpleDateFormat("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH).parse(textDate); + final Calendar calendar = Calendar.getInstance(); calendar.setTime(date); return new DateWrapper(calendar, false); - } catch (ParseException e) { + } catch (final ParseException e) { throw new ParsingException("Could not extract date", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 53781f378..2019153a6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -24,7 +24,8 @@ public class BandcampFeaturedExtractor extends KioskExtractor public static final String KIOSK_FEATURED = "Featured"; public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data"; - public BandcampFeaturedExtractor(StreamingService streamingService, ListLinkHandler listLinkHandler, String kioskId) { + public BandcampFeaturedExtractor(final StreamingService streamingService, final ListLinkHandler listLinkHandler, + final String kioskId) { super(streamingService, listLinkHandler, kioskId); } @@ -43,23 +44,23 @@ public class BandcampFeaturedExtractor extends KioskExtractor @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); + final PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); try { - JsonObject json = JsonParser.object().from( + final JsonObject json = JsonParser.object().from( getDownloader().post( FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes() ).responseBody() ); - JsonArray featuredStories = json.getObject("feed_content") + final JsonArray featuredStories = json.getObject("feed_content") .getObject("stories") .getArray("featured"); for (int i = 0; i < featuredStories.size(); i++) { - JsonObject featuredStory = featuredStories.getObject(i); + final JsonObject featuredStory = featuredStories.getObject(i); if (featuredStory.isNull("album_title")) { // Is not an album, ignore @@ -70,7 +71,7 @@ public class BandcampFeaturedExtractor extends KioskExtractor } return new InfoItemsPage<>(c, null); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { e.printStackTrace(); throw new ParsingException("JSON error", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index b7e812695..8915938fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -38,22 +38,22 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { private JsonArray trackInfo; private String name; - public BandcampPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + public BandcampPlaylistExtractor(final StreamingService service, final ListLinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String html = downloader.get(getLinkHandler().getUrl()).responseBody(); + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String html = downloader.get(getLinkHandler().getUrl()).responseBody(); document = Jsoup.parse(html); albumJson = getAlbumInfoJson(html); trackInfo = albumJson.getArray("trackinfo"); try { name = getJsonData(html, "data-embed").getString("album_title"); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Faulty JSON; page likely does not contain album data", e); - } catch (ArrayIndexOutOfBoundsException e) { + } catch (final ArrayIndexOutOfBoundsException e) { throw new ParsingException("JSON does not exist", e); } @@ -67,8 +67,11 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { - if (albumJson.isNull("art_id")) return ""; - else return getImageUrl(albumJson.getLong("art_id"), true); + if (albumJson.isNull("art_id")) { + return ""; + } else { + return getImageUrl(albumJson.getLong("art_id"), true); + } } @Override @@ -78,7 +81,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() throws ParsingException { - String[] parts = getUrl().split("/"); + final String[] parts = getUrl().split("/"); // https: (/) (/) * .bandcamp.com (/) and leave out the rest return "https://" + parts[2] + "/"; } @@ -124,7 +127,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); for (int i = 0; i < trackInfo.size(); i++) { JsonObject track = trackInfo.getObject(i); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java index b28d13d72..f6b393bfb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java @@ -1,13 +1,14 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; -import com.grack.nanojson.JsonObject; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import javax.annotation.Nonnull; + public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private final Element searchResult, resultInfo; - public BandcampPlaylistInfoItemExtractor(Element searchResult) { + public BandcampPlaylistInfoItemExtractor(@Nonnull Element searchResult) { this.searchResult = searchResult; resultInfo = searchResult.getElementsByClass("result-info").first(); } @@ -20,7 +21,7 @@ public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtrac @Override public long getStreamCount() { - String length = resultInfo.getElementsByClass("length").text(); + final String length = resultInfo.getElementsByClass("length").text(); return Integer.parseInt(length.split(" track")[0]); } @@ -36,7 +37,7 @@ public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtrac @Override public String getThumbnailUrl() { - Element img = searchResult.getElementsByClass("art").first() + final Element img = searchResult.getElementsByClass("art").first() .getElementsByTag("img").first(); if (img != null) { return img.attr("src"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java index 968bff22a..65872aab3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java @@ -9,7 +9,7 @@ public class BandcampPlaylistInfoItemFeaturedExtractor implements PlaylistInfoIt private final JsonObject featuredStory; - public BandcampPlaylistInfoItemFeaturedExtractor(JsonObject featuredStory) { + public BandcampPlaylistInfoItemFeaturedExtractor(final JsonObject featuredStory) { this.featuredStory = featuredStory; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java index 6e837d44f..61c9e7326 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java @@ -15,7 +15,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; @@ -26,12 +25,13 @@ public class BandcampRadioExtractor extends KioskExtractor { public static final String KIOSK_RADIO = "Radio"; public static final String RADIO_API_URL = "https://bandcamp.com/api/bcweekly/1/list"; - public BandcampRadioExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) { + public BandcampRadioExtractor(final StreamingService streamingService, final ListLinkHandler linkHandler, + final String kioskId) { super(streamingService, linkHandler, kioskId); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { } @@ -44,27 +44,27 @@ public class BandcampRadioExtractor extends KioskExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - InfoItemsCollector c = new StreamInfoItemsCollector(getServiceId()); + final InfoItemsCollector c = new StreamInfoItemsCollector(getServiceId()); try { - JsonObject json = JsonParser.object().from( + final JsonObject json = JsonParser.object().from( getDownloader().get( RADIO_API_URL ).responseBody() ); - JsonArray radioShows = json.getArray("results"); + final JsonArray radioShows = json.getArray("results"); for (int i = 0; i < radioShows.size(); i++) { - JsonObject radioShow = radioShows.getObject(i); + final JsonObject radioShow = radioShows.getObject(i); c.commit( new BandcampRadioInfoItemExtractor(radioShow) ); } - } catch (JsonParserException e) { + } catch (final JsonParserException e) { e.printStackTrace(); } @@ -72,7 +72,7 @@ public class BandcampRadioExtractor extends KioskExtractor { } @Override - public InfoItemsPage getPage(Page page) { + public InfoItemsPage getPage(final Page page) { return null; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index f478d985e..b2e2d3642 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -16,7 +16,7 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { private JsonObject show; - public BandcampRadioInfoItemExtractor(JsonObject radioShow) { + public BandcampRadioInfoItemExtractor(final JsonObject radioShow) { show = radioShow; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 3500b862f..57ca8b187 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -29,23 +29,23 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { private JsonObject showInfo; private LinkHandler linkHandler; - public BandcampRadioStreamExtractor(StreamingService service, LinkHandler linkHandler) { + public BandcampRadioStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); this.linkHandler = linkHandler; } - static JsonObject query(int id) throws ParsingException { + static JsonObject query(final int id) throws ParsingException { try { return JsonParser.object().from( NewPipe.getDownloader().get("https://bandcamp.com/api/bcweekly/1/get?id=" + id).responseBody() ); - } catch (IOException | ReCaptchaException | JsonParserException e) { + } catch (final IOException | ReCaptchaException | JsonParserException e) { throw new ParsingException("could not get show data", e); } } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { showInfo = query(Integer.parseInt(getId())); } @@ -105,8 +105,8 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Override public List getAudioStreams() { - ArrayList list = new ArrayList<>(); - JsonObject streams = showInfo.getObject("audio_stream"); + final ArrayList list = new ArrayList<>(); + final JsonObject streams = showInfo.getObject("audio_stream"); if (streams.has("opus-lo")) { list.add(new AudioStream( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index 9deb98c0e..23097171e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -37,21 +37,20 @@ public class BandcampSearchExtractor extends SearchExtractor { return false; } - public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { // okay apparently this is where we DOWNLOAD the page and then COMMIT its ENTRIES to an INFOITEMPAGE - String html = getDownloader().get(page.getUrl()).responseBody(); + final String html = getDownloader().get(page.getUrl()).responseBody(); - InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - Document d = Jsoup.parse(html); + final Document d = Jsoup.parse(html); - Elements searchResultsElements = d.getElementsByClass("searchresult"); + final Elements searchResultsElements = d.getElementsByClass("searchresult"); - for (Element searchResult : - searchResultsElements) { + for (final Element searchResult : searchResultsElements) { - String type = searchResult.getElementsByClass("result-info").first() + final String type = searchResult.getElementsByClass("result-info").first() .getElementsByClass("itemtype").first().text(); switch (type) { @@ -77,16 +76,16 @@ public class BandcampSearchExtractor extends SearchExtractor { } // Count pages - Elements pageLists = d.getElementsByClass("pagelist"); + final Elements pageLists = d.getElementsByClass("pagelist"); if (pageLists.size() == 0) return new InfoItemsPage<>(collector, null); - Elements pages = pageLists.first().getElementsByTag("li"); + final Elements pages = pageLists.first().getElementsByTag("li"); // Find current page int currentPage = -1; for (int i = 0; i < pages.size(); i++) { - Element pageElement = pages.get(i); + final Element pageElement = pages.get(i); if (pageElement.getElementsByTag("span").size() > 0) { currentPage = i + 1; break; @@ -112,7 +111,7 @@ public class BandcampSearchExtractor extends SearchExtractor { } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 1561a98ab..1f37176bf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -33,14 +33,14 @@ public class BandcampStreamExtractor extends StreamExtractor { private JsonObject current; private Document document; - public BandcampStreamExtractor(StreamingService service, LinkHandler linkHandler) { + public BandcampStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String html = downloader.get(getLinkHandler().getUrl()).responseBody(); + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String html = downloader.get(getLinkHandler().getUrl()).responseBody(); document = Jsoup.parse(html); albumJson = getAlbumInfoJson(html); current = albumJson.getObject("current"); @@ -58,12 +58,12 @@ public class BandcampStreamExtractor extends StreamExtractor { * @return Album metadata JSON * @throws ParsingException In case of a faulty website */ - public static JsonObject getAlbumInfoJson(String html) throws ParsingException { + public static JsonObject getAlbumInfoJson(final String html) throws ParsingException { try { return BandcampExtractorHelper.getJsonData(html, "data-tralbum"); - } catch (JsonParserException e) { + } catch (final JsonParserException e) { throw new ParsingException("Faulty JSON; page likely does not contain album data", e); - } catch (ArrayIndexOutOfBoundsException e) { + } catch (final ArrayIndexOutOfBoundsException e) { throw new ParsingException("JSON does not exist", e); } } @@ -77,7 +77,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploaderUrl() throws ParsingException { - String[] parts = getUrl().split("/"); + final String[] parts = getUrl().split("/"); // https: (/) (/) * .bandcamp.com (/) and leave out the rest return "https://" + parts[2] + "/"; } @@ -118,7 +118,7 @@ public class BandcampStreamExtractor extends StreamExtractor { public String getUploaderAvatarUrl() { try { return document.getElementsByClass("band-photo").first().attr("src"); - } catch (NullPointerException e) { + } catch (final NullPointerException e) { return ""; } } @@ -144,7 +144,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public Description getDescription() { - String s = BandcampExtractorHelper.smartConcatenate( + final String s = BandcampExtractorHelper.smartConcatenate( new String[]{ current.getString("about"), current.getString("lyrics"), @@ -198,7 +198,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public List getAudioStreams() { - List audioStreams = new ArrayList<>(); + final List audioStreams = new ArrayList<>(); audioStreams.add(new AudioStream( albumJson.getArray("trackinfo").getObject(0) @@ -303,11 +303,11 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() { - Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords"); + final Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords"); - ArrayList tags = new ArrayList<>(); + final ArrayList tags = new ArrayList<>(); - for (Element e : tagElements) { + for (final Element e : tagElements) { tags.add(e.text()); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index 5ef4f3d56..bb1ababb2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -20,32 +20,32 @@ import java.util.List; public class BandcampSuggestionExtractor extends SuggestionExtractor { private static final String AUTOCOMPLETE_URL = "https://bandcamp.com/api/fuzzysearch/1/autocomplete?q="; - public BandcampSuggestionExtractor(StreamingService service) { + public BandcampSuggestionExtractor(final StreamingService service) { super(service); } @Override - public List suggestionList(String query) throws IOException, ExtractionException { - Downloader downloader = NewPipe.getDownloader(); + public List suggestionList(final String query) throws IOException, ExtractionException { + final Downloader downloader = NewPipe.getDownloader(); try { - JsonObject fuzzyResults = JsonParser.object().from( + final JsonObject fuzzyResults = JsonParser.object().from( downloader.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, "UTF-8")).responseBody() ); - JsonArray jsonArray = fuzzyResults.getObject("auto") + final JsonArray jsonArray = fuzzyResults.getObject("auto") .getArray("results"); - ArrayList suggestions = new ArrayList<>(); + final ArrayList suggestions = new ArrayList<>(); - for (Object fuzzyResult : jsonArray) { - String res = ((JsonObject) fuzzyResult).getString("name"); + for (final Object fuzzyResult : jsonArray) { + final String res = ((JsonObject) fuzzyResult).getString("name"); if (!suggestions.contains(res)) suggestions.add(res); } return suggestions; - } catch (JsonParserException e) { + } catch (final JsonParserException e) { e.printStackTrace(); return new ArrayList<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java index 4c76b404d..15b044981 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtract public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { private final JsonObject discograph; - public BandcampDiscographStreamInfoItemExtractor(JsonObject discograph, String uploaderUrl) { + public BandcampDiscographStreamInfoItemExtractor(final JsonObject discograph, final String uploaderUrl) { super(uploaderUrl); this.discograph = discograph; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index 15e5116e9..e90a12aa1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -17,13 +17,15 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI private String substituteCoverUrl; private final StreamingService service; - public BandcampPlaylistStreamInfoItemExtractor(JsonObject track, String uploaderUrl, StreamingService service) { + public BandcampPlaylistStreamInfoItemExtractor(final JsonObject track, final String uploaderUrl, + final StreamingService service) { super(uploaderUrl); this.track = track; this.service = service; } - public BandcampPlaylistStreamInfoItemExtractor(JsonObject track, String uploaderUrl, String substituteCoverUrl) { + public BandcampPlaylistStreamInfoItemExtractor(final JsonObject track, final String uploaderUrl, + final String substituteCoverUrl) { this(track, uploaderUrl, (StreamingService) null); this.substituteCoverUrl = substituteCoverUrl; } @@ -61,10 +63,10 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI return substituteCoverUrl; } else { try { - StreamExtractor extractor = service.getStreamExtractor(getUrl()); + final StreamExtractor extractor = service.getStreamExtractor(getUrl()); extractor.fetchPage(); return extractor.getThumbnailUrl(); - } catch (ExtractionException | IOException e) { + } catch (final ExtractionException | IOException e) { throw new ParsingException("could not download cover art location", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index 6d88edc1c..adeea0519 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -7,7 +7,7 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte private final Element resultInfo, searchResult; - public BandcampSearchStreamInfoItemExtractor(Element searchResult, String uploaderUrl) { + public BandcampSearchStreamInfoItemExtractor(final Element searchResult, final String uploaderUrl) { super(uploaderUrl); this.searchResult = searchResult; resultInfo = searchResult.getElementsByClass("result-info").first(); @@ -15,8 +15,8 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte @Override public String getUploaderName() { - String subhead = resultInfo.getElementsByClass("subhead").text(); - String[] splitBy = subhead.split("by "); + final String subhead = resultInfo.getElementsByClass("subhead").text(); + final String[] splitBy = subhead.split("by "); if (splitBy.length > 1) { return splitBy[1]; } else { @@ -36,11 +36,13 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte @Override public String getThumbnailUrl() throws ParsingException { - Element img = searchResult.getElementsByClass("art").first() + final Element img = searchResult.getElementsByClass("art").first() .getElementsByTag("img").first(); if (img != null) { return img.attr("src"); - } else return null; + } else { + return null; + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java index a8a3ceb46..485433f9d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java @@ -13,7 +13,7 @@ import javax.annotation.Nullable; public abstract class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor { private final String uploaderUrl; - public BandcampStreamInfoItemExtractor(String uploaderUrl) { + public BandcampStreamInfoItemExtractor(final String uploaderUrl) { this.uploaderUrl = uploaderUrl; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index 825add10b..9fbc8ae8f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -20,16 +20,16 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { try { - String response = NewPipe.getDownloader().get(url).responseBody(); + final String response = NewPipe.getDownloader().get(url).responseBody(); // This variable contains band data! - JsonObject bandData = BandcampExtractorHelper.getJsonData(response, "data-band"); + final JsonObject bandData = BandcampExtractorHelper.getJsonData(response, "data-band"); return String.valueOf(bandData.getLong("id")); - } catch (IOException | ReCaptchaException | ArrayIndexOutOfBoundsException | JsonParserException e) { + } catch (final IOException | ReCaptchaException | ArrayIndexOutOfBoundsException | JsonParserException e) { throw new ParsingException("Download failed", e); } } @@ -38,12 +38,13 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { * Uses the mobile endpoint as a "translator" from id to url */ @Override - public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + public String getUrl(final String id, final List contentFilter, final String sortFilter) + throws ParsingException { try { return BandcampExtractorHelper.getArtistDetails(id) .getString("bandcamp_url") .replace("http://", "https://"); - } catch (NullPointerException e) { + } catch (final NullPointerException e) { throw new ParsingException("JSON does not contain URL (invalid id?) or is otherwise invalid", e); } @@ -54,7 +55,7 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { * where the profile is at * . * /releases */ @Override - public boolean onAcceptUrl(String url) { + public boolean onAcceptUrl(final String url) { // Is a subdomain of bandcamp.com? boolean isBandcampComArtistPage = url.matches("https?://.+\\.bandcamp\\.com/?"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java index 0c5cb5fe8..7aac3684b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java @@ -14,23 +14,30 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory { @Override - public String getUrl(String id, List contentFilter, String sortFilter) { - if (id.equals(KIOSK_FEATURED)) return FEATURED_API_URL; // doesn't have a website - else if (id.equals(KIOSK_RADIO)) return RADIO_API_URL; // doesn't have its own website - else return null; + public String getUrl(final String id, final List contentFilter, final String sortFilter) { + if (id.equals(KIOSK_FEATURED)) { + return FEATURED_API_URL; // doesn't have a website + } else if (id.equals(KIOSK_RADIO)) { + return RADIO_API_URL; // doesn't have its own website + } else { + return null; + } } @Override - public String getId(String url) { - if (url.matches("https?://bandcamp\\.com/\\?show=\\d+") || url.equals(RADIO_API_URL)) + public String getId(final String url) { + if (url.matches("https?://bandcamp\\.com/\\?show=\\d+") || url.equals(RADIO_API_URL)) { return KIOSK_RADIO; - else if (url.equals(FEATURED_API_URL)) + } else if (url.equals(FEATURED_API_URL)) { return KIOSK_FEATURED; - else return null; + } else { + return null; + } } @Override - public boolean onAcceptUrl(String url) { - return url.equals(FEATURED_API_URL) || (url.equals(RADIO_API_URL) || url.matches("https?://bandcamp\\.com/\\?show=\\d+")); + public boolean onAcceptUrl(final String url) { + return url.equals(FEATURED_API_URL) || (url.equals(RADIO_API_URL) + || url.matches("https?://bandcamp\\.com/\\?show=\\d+")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java index 9a736b30f..b030fcc5a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java @@ -12,17 +12,18 @@ import java.util.List; */ public class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { return getUrl(url); } @Override - public String getUrl(String url, List contentFilter, String sortFilter) throws ParsingException { + public String getUrl(final String url, final List contentFilter, final String sortFilter) + throws ParsingException { return url; } @Override - public boolean onAcceptUrl(String url) { + public boolean onAcceptUrl(final String url) { return url.toLowerCase().matches("https?://.+\\..+/album/.+"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java index 4a90c6c0e..518170b07 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java @@ -13,14 +13,15 @@ public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override - public String getUrl(String query, List contentFilter, String sortFilter) throws ParsingException { + public String getUrl(final String query, final List contentFilter, final String sortFilter) + throws ParsingException { try { return "https://bandcamp.com/search?q=" + URLEncoder.encode(query, "UTF-8") + "&page=1"; - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { throw new ParsingException("query \"" + query + "\" could not be encoded", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java index 176c2e2ba..5de345e15 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -18,11 +18,12 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { * @see BandcampStreamLinkHandlerFactory */ @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { if (url.matches("https?://bandcamp\\.com/\\?show=\\d+")) { return url.split("bandcamp.com/\\?show=")[1]; - } else + } else { return getUrl(url); + } } /** @@ -30,10 +31,12 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { * @see BandcampStreamLinkHandlerFactory */ @Override - public String getUrl(String input) { - if (input.matches("\\d+")) + public String getUrl(final String input) { + if (input.matches("\\d+")) { return "https://bandcamp.com/?show=" + input; - else return input; + } else { + return input; + } } /** @@ -45,7 +48,8 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { * https:// * . * /track/ * */ @Override - public boolean onAcceptUrl(String url) { - return url.toLowerCase().matches("https?://.+\\..+/track/.+") || url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+"); + public boolean onAcceptUrl(final String url) { + return url.toLowerCase().matches("https?://.+\\..+/track/.+") + || url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+"); } } From c91e21b37c9e17e0fc7f01d0aab45074a677df86 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 24 Nov 2020 09:55:41 +0100 Subject: [PATCH 71/91] [Bandcamp] Tests - Add finals and improve code formatting --- .../BandcampFeaturedExtractorTest.java | 4 ++-- .../BandcampPlaylistExtractorTest.java | 16 ++++++++-------- .../bandcamp/BandcampRadioExtractorTest.java | 4 ++-- .../BandcampRadioStreamExtractorTest.java | 6 ++++-- .../bandcamp/BandcampSearchExtractorTest.java | 18 +++++++++--------- .../bandcamp/BandcampStreamExtractorTest.java | 2 +- .../BandcampSuggestionExtractorTest.java | 2 +- 7 files changed, 27 insertions(+), 25 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index 1668445c8..50d992a16 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -36,13 +36,13 @@ public class BandcampFeaturedExtractorTest implements BaseListExtractorTest { @Test public void testFeaturedCount() throws ExtractionException, IOException { - List list = extractor.getInitialPage().getItems(); + final List list = extractor.getInitialPage().getItems(); assertTrue(list.size() > 1); } @Test public void testHttps() throws ExtractionException, IOException { - List list = extractor.getInitialPage().getItems(); + final List list = extractor.getInitialPage().getItems(); assertTrue(list.get(0).getUrl().contains("https://")); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index c360c23a7..b695236d9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -36,7 +36,7 @@ public class BandcampPlaylistExtractorTest { */ @Test public void testCount() throws ExtractionException, IOException { - PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); + final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://macbenson.bandcamp.com/album/coming-of-age"); extractor.fetchPage(); assertEquals(5, extractor.getStreamCount()); @@ -47,10 +47,10 @@ public class BandcampPlaylistExtractorTest { */ @Test public void testDifferentTrackCovers() throws ExtractionException, IOException { - PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachbensonarchive.bandcamp.com/album/results-of-boredom"); + final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachbensonarchive.bandcamp.com/album/results-of-boredom"); extractor.fetchPage(); - List l = extractor.getInitialPage().getItems(); + final List l = extractor.getInitialPage().getItems(); assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); assertNotEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); } @@ -60,13 +60,13 @@ public class BandcampPlaylistExtractorTest { */ @Test(timeout = 10000L) public void testDifferentTrackCoversDuration() throws ExtractionException, IOException { - PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything"); + final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything"); extractor.fetchPage(); /* All tracks in this album have the same cover art, but I don't know any albums with more than 10 tracks * that has at least one track with a cover art different from the rest. */ - List l = extractor.getInitialPage().getItems(); + final List l = extractor.getInitialPage().getItems(); assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); assertEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); } @@ -76,7 +76,7 @@ public class BandcampPlaylistExtractorTest { */ @Test(expected = ContentNotAvailableException.class) public void testLockedContent() throws ExtractionException, IOException { - PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://billwurtz.bandcamp.com/album/high-enough"); + final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://billwurtz.bandcamp.com/album/high-enough"); extractor.fetchPage(); } @@ -85,7 +85,7 @@ public class BandcampPlaylistExtractorTest { */ @Test public void testSingleStreamPlaylist() throws ExtractionException, IOException { - PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachjohnson1.bandcamp.com/album/endless"); + final PlaylistExtractor extractor = Bandcamp.getPlaylistExtractor("https://zachjohnson1.bandcamp.com/album/endless"); extractor.fetchPage(); assertEquals(1, extractor.getStreamCount()); @@ -170,7 +170,7 @@ public class BandcampPlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - //DefaultTests.defaultTestRelatedItems(extractor); + // DefaultTests.defaultTestRelatedItems(extractor); // Would fail because BandcampPlaylistStreamInfoItemExtractor.getUploaderName() returns an empty String } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java index 76f0559d0..c7b4e3579 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java @@ -36,13 +36,13 @@ public class BandcampRadioExtractorTest implements BaseListExtractorTest { @Test public void testRadioCount() throws ExtractionException, IOException { - List list = Bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); + final List list = Bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); assertTrue(list.size() > 300); } @Test public void testRelatedItems() throws Exception { - //DefaultTests.defaultTestRelatedItems(extractor); + // DefaultTests.defaultTestRelatedItems(extractor); // Would fail because BandcampRadioInfoItemExtractor.getUploaderName() returns an empty String } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index ecbed677c..b4f9382c7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -33,7 +33,8 @@ public class BandcampRadioStreamExtractorTest { @Test public void testGettingCorrectStreamExtractor() throws ExtractionException { assertTrue(Bandcamp.getStreamExtractor("https://bandcamp.com/?show=3") instanceof BandcampRadioStreamExtractor); - assertFalse(Bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/track/deflated") instanceof BandcampRadioStreamExtractor); + assertFalse(Bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/track/deflated") + instanceof BandcampRadioStreamExtractor); } @Test @@ -86,7 +87,8 @@ public class BandcampRadioStreamExtractorTest { @Test public void testGetDescription() throws ParsingException { - assertEquals("Featuring special guests Nick Hakim and Elbows, plus fresh cuts from Eddie Palmieri, KRS One, Ladi6, and Moonchild.", e.getDescription().getContent()); + assertEquals("Featuring special guests Nick Hakim and Elbows, plus fresh cuts from Eddie Palmieri, KRS One, Ladi6, and Moonchild.", + e.getDescription().getContent()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 786839995..f74fb9256 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -40,10 +40,10 @@ public class BandcampSearchExtractorTest { */ @Test public void testStreamSearch() throws ExtractionException, IOException { - SearchExtractor extractor = Bandcamp.getSearchExtractor("best friend's basement"); + final SearchExtractor extractor = Bandcamp.getSearchExtractor("best friend's basement"); - ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - StreamInfoItem bestFriendsBasement = (StreamInfoItem) page.getItems().get(0); + final ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + final StreamInfoItem bestFriendsBasement = (StreamInfoItem) page.getItems().get(0); // The track by Zach Benson should be the first result, no? assertEquals("Best Friend's Basement", bestFriendsBasement.getName()); @@ -58,8 +58,8 @@ public class BandcampSearchExtractorTest { */ @Test public void testChannelSearch() throws ExtractionException, IOException { - SearchExtractor extractor = Bandcamp.getSearchExtractor("C418"); - InfoItem c418 = extractor.getInitialPage() + final SearchExtractor extractor = Bandcamp.getSearchExtractor("C418"); + final InfoItem c418 = extractor.getInitialPage() .getItems().get(0); // C418's artist profile should be the first result, no? @@ -75,7 +75,7 @@ public class BandcampSearchExtractorTest { */ @Test public void testAlbumSearch() throws ExtractionException, IOException { - SearchExtractor extractor = Bandcamp.getSearchExtractor("minecraft volume alpha"); + final SearchExtractor extractor = Bandcamp.getSearchExtractor("minecraft volume alpha"); InfoItem minecraft = extractor.getInitialPage() .getItems().get(0); @@ -96,12 +96,12 @@ public class BandcampSearchExtractorTest { @Test public void testMultiplePages() throws ExtractionException, IOException { // A query practically guaranteed to have the maximum amount of pages - SearchExtractor extractor = Bandcamp.getSearchExtractor("e"); + final SearchExtractor extractor = Bandcamp.getSearchExtractor("e"); - Page page2 = extractor.getInitialPage().getNextPage(); + final Page page2 = extractor.getInitialPage().getNextPage(); assertEquals("https://bandcamp.com/search?q=e&page=2", page2.getUrl()); - Page page3 = extractor.getPage(page2).getNextPage(); + final Page page3 = extractor.getPage(page2).getNextPage(); assertEquals("https://bandcamp.com/search?q=e&page=3", page3.getUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 4c14bea5d..f61018820 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -154,7 +154,7 @@ public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { @Test public void testArtistProfilePicture() throws Exception { - String url = extractor().getUploaderAvatarUrl(); + final String url = extractor().getUploaderAvatarUrl(); assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java index 393ca01c6..9764d995a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java @@ -30,7 +30,7 @@ public class BandcampSuggestionExtractorTest { @Test public void testSearchExample() throws IOException, ExtractionException { - List c418 = extractor.suggestionList("c418"); + final List c418 = extractor.suggestionList("c418"); assertTrue(c418.contains("C418")); From be562b84365a9b74ce0708cae53972aade36bfaf Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Fri, 27 Nov 2020 16:48:06 +0100 Subject: [PATCH 72/91] Change tests --- .../BandcampChannelExtractorTest.java | 20 ++++++------- .../bandcamp/BandcampStreamExtractorTest.java | 29 ++++++++++--------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index a5ccabf9d..327dc6b9a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -22,31 +22,29 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = Bandcamp.getChannelExtractor("https://npet.bandcamp.com/releases"); + extractor = Bandcamp.getChannelExtractor("https://toupie.bandcamp.com/releases"); extractor.fetchPage(); } @Test public void testLength() throws ExtractionException, IOException { - assertTrue(extractor.getInitialPage().getItems().size() >= 1); + assertTrue(extractor.getInitialPage().getItems().size() >= 0); } @Override @Test public void testDescription() throws Exception { - assertEquals("This string will be tested for in NewPipeExtractor tests.", extractor.getDescription()); + assertEquals("making music:)", extractor.getDescription()); } @Override public void testAvatarUrl() throws Exception { - // Has no avatar - assertEquals("", extractor.getAvatarUrl()); + assertTrue("unexpected avatar URL", extractor.getAvatarUrl().contains("://f4.bcbits.com/")); } @Override public void testBannerUrl() throws Exception { - // Has no banner - assertEquals("", extractor.getBannerUrl()); + assertTrue("unexpected banner URL", extractor.getBannerUrl().contains("://f4.bcbits.com/")); } @Override @@ -76,21 +74,21 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { @Override public void testName() throws Exception { - assertEquals("NewPipeExtractorTest", extractor.getName()); + assertEquals("toupie", extractor.getName()); } @Override public void testId() throws Exception { - assertEquals("https://npet.bandcamp.com/", extractor.getId()); + assertEquals("https://toupie.bandcamp.com/", extractor.getId()); } @Override public void testUrl() throws Exception { - assertEquals("https://npet.bandcamp.com/releases", extractor.getUrl()); + assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); } @Override public void testOriginalUrl() throws Exception { - assertEquals("https://npet.bandcamp.com/releases", extractor.getUrl()); + assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index f61018820..dd979d554 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -35,10 +35,9 @@ public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { @BeforeClass public static void setUp() throws ExtractionException, IOException { NewPipe.init(DownloaderTestImpl.getInstance()); - // This test track was uploaded specifically for NewPipeExtractor tests extractor = (BandcampStreamExtractor) Bandcamp - .getStreamExtractor("https://npet.bandcamp.com/track/track-1"); + .getStreamExtractor("https://teaganbear.bandcamp.com/track/just-for-the-halibut-creative-commons-attribution"); extractor.fetchPage(); } @@ -54,22 +53,22 @@ public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { @Override public String expectedName() { - return "Track #1"; + return "Just for the Halibut [Creative Commons: Attribution]"; } @Override public String expectedId() { - return "https://npet.bandcamp.com/track/track-1"; + return "https://teaganbear.bandcamp.com/track/just-for-the-halibut-creative-commons-attribution"; } @Override public String expectedUrlContains() { - return "https://npet.bandcamp.com/track/track-1"; + return "https://teaganbear.bandcamp.com/track/just-for-the-halibut-creative-commons-attribution"; } @Override public String expectedOriginalUrlContains() { - return "https://npet.bandcamp.com/track/track-1"; + return "https://teaganbear.bandcamp.com/track/just-for-the-halibut-creative-commons-attribution"; } @Override @@ -79,17 +78,17 @@ public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { @Override public String expectedUploaderName() { - return "NewPipeExtractorTest"; + return "Teaganbear"; } @Override public String expectedUploaderUrl() { - return "https://npet.bandcamp.com/"; + return "https://teaganbear.bandcamp.com/"; } @Override public List expectedDescriptionContains() { - return Collections.singletonList("This sample track was created using MuseScore."); + return Collections.singletonList("it's Creative Commons so feel free to use it in whatever"); } @Override @@ -104,12 +103,12 @@ public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { @Override public String expectedUploadDate() { - return "2020-03-17 18:37:44.000"; + return "2019-03-10 23:00:42.000"; } @Override public String expectedTextualUploadDate() { - return "17 Mar 2020 18:37:44 GMT"; + return "10 Mar 2019 23:00:42 GMT"; } @Override @@ -149,7 +148,7 @@ public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { @Override public String expectedCategory() { - return "acoustic"; + return "dance"; } @Test @@ -160,7 +159,11 @@ public class BandcampStreamExtractorTest extends DefaultStreamExtractorTest { @Test public void testTranslateIdsToUrl() throws ParsingException { - assertEquals("https://npet.bandcamp.com/track/track-1", BandcampExtractorHelper.getStreamUrlFromIds(3775652329L, 4207805220L, "track")); + // To add tests: look at website's source, search for `band_id` and `item_id` + assertEquals( + "https://teaganbear.bandcamp.com/track/just-for-the-halibut-creative-commons-attribution", + BandcampExtractorHelper.getStreamUrlFromIds(3877364987L, 3486455278L, "track") + ); } } From 04dd3d4d3266d3b2cf2d8afca8518209431c17aa Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sat, 5 Dec 2020 15:08:26 +0100 Subject: [PATCH 73/91] Rework link handlers to correctly accept external websites --- .../extractors/BandcampExtractorHelper.java | 22 +++++++++++++++++++ .../BandcampChannelLinkHandlerFactory.java | 16 ++++++-------- .../BandcampPlaylistLinkHandlerFactory.java | 13 +++++++++-- .../BandcampStreamLinkHandlerFactory.java | 22 +++++++++++-------- ...BandcampChannelLinkHandlerFactoryTest.java | 10 +++++++-- ...andcampPlaylistLinkHandlerFactoryTest.java | 1 + .../BandcampStreamLinkHandlerFactoryTest.java | 1 + 7 files changed, 63 insertions(+), 22 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 5d0fb45f7..6508de10d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -123,6 +123,28 @@ public class BandcampExtractorHelper { return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; } + /** + * @return true if the given url looks like it comes from a bandcamp custom domain + * or if it comes from bandcamp.com itself + */ + public static boolean isSupportedDomain(final String url) throws ParsingException { + + // Accept all bandcamp.com URLs + if (url.toLowerCase().matches("https?://.+\\.bandcamp\\.com(/.*)?")) return true; + + try { + // Accept all other URLs if they contain a tag that says they are generated by bandcamp + return Jsoup.parse( + NewPipe.getDownloader().get(url).responseBody() + ) + .getElementsByAttributeValue("name", "generator") + .attr("content").equals("Bandcamp"); + } catch (IOException | ReCaptchaException e) { + throw new ParsingException("Could not determine whether URL is custom domain " + + "(not available? network error?)"); + } + } + static DateWrapper parseDate(final String textDate) throws ParsingException { try { final Date date = new SimpleDateFormat("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH).parse(textDate); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index 9fbc8ae8f..762c5d9b3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -24,7 +24,7 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { try { final String response = NewPipe.getDownloader().get(url).responseBody(); - // This variable contains band data! + // Use band data embedded in website to extract ID final JsonObject bandData = BandcampExtractorHelper.getJsonData(response, "data-band"); return String.valueOf(bandData.getLong("id")); @@ -51,17 +51,15 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { } /** - * Matches * .bandcamp.com as well as custom domains - * where the profile is at * . * /releases + * Accepts only pages that do not lead to an album or track. Supports external pages. */ @Override - public boolean onAcceptUrl(final String url) { + public boolean onAcceptUrl(final String url) throws ParsingException { - // Is a subdomain of bandcamp.com? - boolean isBandcampComArtistPage = url.matches("https?://.+\\.bandcamp\\.com/?"); + // Exclude URLs that lead to a track or album + if (url.matches(".*/(album|track)/.*")) return false; - boolean isCustomDomainReleases = url.matches("https?://.+\\..+/releases/?(?!.)"); - - return isBandcampComArtistPage || isCustomDomainReleases; + // Test whether domain is supported + return BandcampExtractorHelper.isSupportedDomain(url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java index b030fcc5a..b0fd2d50a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java @@ -4,6 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import java.util.List; @@ -22,8 +23,16 @@ public class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFactory { return url; } + /** + * Accepts all bandcamp URLs that contain /album/ behind their domain name. + */ @Override - public boolean onAcceptUrl(final String url) { - return url.toLowerCase().matches("https?://.+\\..+/album/.+"); + public boolean onAcceptUrl(final String url) throws ParsingException { + + // Exclude URLs which do not lead to an album + if (!url.toLowerCase().matches("https?://.+\\..+/album/.+")) return false; + + // Test whether domain is supported + return BandcampExtractorHelper.isSupportedDomain(url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java index 5de345e15..fa0435daa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -4,6 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; /** *

Tracks don't have standalone ids, they are always in combination with the band id. @@ -40,16 +41,19 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { } /** - * Sometimes, the root page of an artist is also an album or track - * page. In that case, it is assumed that one actually wants to open - * the profile and not the track it has set as the default one. - *

Urls are expected to be in this format to account for - * custom domains:

- * https:// * . * /track/ * + * Accepts URLs that point to a bandcamp radio show or that are a bandcamp + * domain and point to a track. */ @Override - public boolean onAcceptUrl(final String url) { - return url.toLowerCase().matches("https?://.+\\..+/track/.+") - || url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+"); + public boolean onAcceptUrl(final String url) throws ParsingException { + + // Accept Bandcamp radio + if (url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+")) return true; + + // Don't accept URLs that don't point to a track + if (!url.toLowerCase().matches("https?://.+\\..+/track/.+")) return false; + + // Test whether domain is supported + return BandcampExtractorHelper.isSupportedDomain(url); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index e07daf7d1..8886f649f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -26,13 +26,19 @@ public class BandcampChannelLinkHandlerFactoryTest { @Test public void testAcceptUrl() throws ParsingException { - assertTrue(linkHandler.acceptUrl("http://interovgm.com/releases/")); - assertTrue(linkHandler.acceptUrl("https://interovgm.com/releases")); + // Bandcamp URLs assertTrue(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); + assertTrue(linkHandler.acceptUrl("https://billwurtz.bandcamp.com/releases")); assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); + + // External URLs + assertTrue(linkHandler.acceptUrl("http://interovgm.com/releases/")); + assertTrue(linkHandler.acceptUrl("https://interovgm.com/releases")); + + assertFalse(linkHandler.acceptUrl("https://example.com/releases")); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java index 11fe3acd7..561d8f7b6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java @@ -35,6 +35,7 @@ public class BandcampPlaylistLinkHandlerFactoryTest { assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); assertFalse(linkHandler.acceptUrl("https://interovgm.com/track/title")); + assertFalse(linkHandler.acceptUrl("https://example.com/album/samplealbum")); assertTrue(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids")); assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/album/prom")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java index 69a7c2bf1..a65b09567 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -43,6 +43,7 @@ public class BandcampStreamLinkHandlerFactoryTest { assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); assertFalse(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids")); + assertFalse(linkHandler.acceptUrl("https://example.com/track/sampletrack")); assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); From df16a8646fb971242a6cbc8b2aaae3baba36b415 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 18 Dec 2020 23:15:54 +0100 Subject: [PATCH 74/91] [Bandcamp] Improve radio stream extractor test --- .../BandcampRadioStreamExtractorTest.java | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index b4f9382c7..c216f9f23 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -4,30 +4,32 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; -import java.util.Calendar; -import java.util.TimeZone; +import java.util.*; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; -public class BandcampRadioStreamExtractorTest { +public class BandcampRadioStreamExtractorTest extends DefaultStreamExtractorTest { - private static StreamExtractor e; + private static StreamExtractor extractor; - private static final String SHOW_URL = "https://bandcamp.com/?show=230"; + private static final String URL = "https://bandcamp.com/?show=230"; @BeforeClass public static void setUp() throws IOException, ExtractionException { NewPipe.init(DownloaderTestImpl.getInstance()); - e = Bandcamp.getStreamExtractor(SHOW_URL); - e.fetchPage(); + extractor = Bandcamp.getStreamExtractor(URL); + extractor.fetchPage(); } @Test @@ -37,31 +39,43 @@ public class BandcampRadioStreamExtractorTest { instanceof BandcampRadioStreamExtractor); } - @Test - public void testGetName() throws ParsingException { - assertEquals("Sound Movements", e.getName()); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public String expectedName() throws Exception { return "Sound Movements"; } + @Override public String expectedId() throws Exception { return "230"; } + @Override public String expectedUrlContains() throws Exception { return URL; } + @Override public String expectedOriginalUrlContains() throws Exception { return URL; } + @Override public boolean expectedHasVideoStreams() { return false; } + @Override public boolean expectedHasSubtitles() { return false; } + @Override public boolean expectedHasFrames() { return false; } + @Override public boolean expectedHasRelatedStreams() { return false; } + @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } + @Override public StreamingService expectedService() { return Bandcamp; } + @Override public String expectedUploaderName() { return "Andrew Jervis"; } @Test(expected = ContentNotSupportedException.class) public void testGetUploaderUrl() throws ParsingException { - e.getUploaderUrl(); + extractor.getUploaderUrl(); } - @Test - public void testGetUrl() throws ParsingException { - assertEquals(SHOW_URL, e.getUrl()); + @Test(expected = ContentNotSupportedException.class) + @Override + public void testUploaderUrl() throws Exception { + super.testUploaderUrl(); + } + @Override public String expectedUploaderUrl() { return null; } + + @Override + public List expectedDescriptionContains() { + return Collections.singletonList("Featuring special guests Nick Hakim and Elbows, plus fresh cuts from Eddie Palmieri, KRS One, Ladi6, and Moonchild."); } - @Test - public void testGetUploaderName() throws ParsingException { - assertEquals("Andrew Jervis", e.getUploaderName()); - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - assertEquals("16 May 2017 00:00:00 GMT", e.getTextualUploadDate()); - } + @Override public long expectedLength() { return 5619; } + @Override public long expectedViewCountAtLeast() { return -1; } + @Override public long expectedLikeCountAtLeast() { return -1; } + @Override public long expectedDislikeCountAtLeast() { return -1; } + @Override public String expectedUploadDate() { return "16 May 2017 00:00:00 GMT"; } + @Override public String expectedTextualUploadDate() { return "16 May 2017 00:00:00 GMT"; } @Test public void testUploadDate() throws ParsingException { final Calendar expectedCalendar = Calendar.getInstance(); @@ -71,34 +85,20 @@ public class BandcampRadioStreamExtractorTest { expectedCalendar.setTimeInMillis(0); expectedCalendar.set(2017, Calendar.MAY, 16); - assertEquals(expectedCalendar.getTimeInMillis(), e.getUploadDate().date().getTimeInMillis()); + assertEquals(expectedCalendar.getTimeInMillis(), extractor.getUploadDate().offsetDateTime().toInstant().toEpochMilli()); } @Test public void testGetThumbnailUrl() throws ParsingException { - assertTrue(e.getThumbnailUrl().contains("bcbits.com/img")); + assertTrue(extractor.getThumbnailUrl().contains("bcbits.com/img")); } @Test public void testGetUploaderAvatarUrl() throws ParsingException { - assertTrue(e.getUploaderAvatarUrl().contains("bandcamp-button")); - + assertTrue(extractor.getUploaderAvatarUrl().contains("bandcamp-button")); } - @Test - public void testGetDescription() throws ParsingException { - assertEquals("Featuring special guests Nick Hakim and Elbows, plus fresh cuts from Eddie Palmieri, KRS One, Ladi6, and Moonchild.", - e.getDescription().getContent()); - - } - - @Test - public void testGetLength() throws ParsingException { - assertEquals(5619, e.getLength()); - } - - @Test - public void testGetAudioStreams() throws ExtractionException, IOException { - assertEquals(2, e.getAudioStreams().size()); + @Test public void testGetAudioStreams() throws ExtractionException, IOException { + assertEquals(2, extractor.getAudioStreams().size()); } } From 74b46fed2d36da002b612fb8952cdd22f19ba8aa Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 19 Dec 2020 10:40:21 +0100 Subject: [PATCH 75/91] [Bandcamp] Fix deprecation in parseDate helper function --- .../extractors/BandcampExtractorHelper.java | 21 +++++++++++-------- .../BandcampRadioStreamExtractorTest.java | 5 ++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 6508de10d..1d76c4612 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -15,9 +15,12 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; +import java.time.DateTimeException; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Locale; public class BandcampExtractorHelper { @@ -147,12 +150,12 @@ public class BandcampExtractorHelper { static DateWrapper parseDate(final String textDate) throws ParsingException { try { - final Date date = new SimpleDateFormat("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH).parse(textDate); - final Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - return new DateWrapper(calendar, false); - } catch (final ParseException e) { - throw new ParsingException("Could not extract date", e); + final ZonedDateTime zonedDateTime = ZonedDateTime.parse( + textDate, DateTimeFormatter.ofPattern("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH)); + return new DateWrapper(zonedDateTime.toOffsetDateTime(), false); + } catch (final DateTimeException e) { + throw new ParsingException("Could not parse date '" + textDate + "'", e); } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index c216f9f23..e024fefa7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -14,7 +14,10 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; -import java.util.*; +import java.util.Calendar; +import java.util.Collections; +import java.util.List; +import java.util.TimeZone; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; From 50903730b13e1b60e297f11047cfbd3dc13fd9c7 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 22 Dec 2020 08:53:10 +0100 Subject: [PATCH 76/91] [Bandcamp] Fix accepting HTTP URLs --- .../BandcampFeaturedLinkHandlerFactory.java | 11 +++++++---- .../BandcampChannelLinkHandlerFactoryTest.java | 3 +++ .../BandcampFeaturedLinkHandlerFactoryTest.java | 2 ++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java index 7aac3684b..6286e31da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; import java.util.List; @@ -25,8 +26,9 @@ public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory { } @Override - public String getId(final String url) { - if (url.matches("https?://bandcamp\\.com/\\?show=\\d+") || url.equals(RADIO_API_URL)) { + public String getId(String url) { + url = Utils.replaceHttpWithHttps(url); + if (url.matches("https://bandcamp\\.com/\\?show=\\d+") || url.equals(RADIO_API_URL)) { return KIOSK_RADIO; } else if (url.equals(FEATURED_API_URL)) { return KIOSK_FEATURED; @@ -36,8 +38,9 @@ public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory { } @Override - public boolean onAcceptUrl(final String url) { + public boolean onAcceptUrl(String url) { + url = Utils.replaceHttpWithHttps(url); return url.equals(FEATURED_API_URL) || (url.equals(RADIO_API_URL) - || url.matches("https?://bandcamp\\.com/\\?show=\\d+")); + || url.matches("https://bandcamp\\.com/\\?show=\\d+")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index 8886f649f..499704b99 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -31,6 +31,8 @@ public class BandcampChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); assertTrue(linkHandler.acceptUrl("https://billwurtz.bandcamp.com/releases")); + assertTrue(linkHandler.acceptUrl("http://zachbenson.bandcamp.com/")); + assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); @@ -44,6 +46,7 @@ public class BandcampChannelLinkHandlerFactoryTest { @Test public void testGetId() throws ParsingException { assertEquals("1196681540", linkHandler.getId("https://macbenson.bandcamp.com/")); + assertEquals("1196681540", linkHandler.getId("http://macbenson.bandcamp.com/")); assertEquals("1581461772", linkHandler.getId("https://interovgm.com/releases")); assertEquals("3321800855", linkHandler.getId("https://infiniteammo.bandcamp.com/")); assertEquals("3775652329", linkHandler.getId("https://npet.bandcamp.com/")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java index 38559782b..b337ded7d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java @@ -24,6 +24,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest { @Test public void testAcceptUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("http://bandcamp.com/?show=1")); assertTrue(linkHandler.acceptUrl("https://bandcamp.com/?show=1")); assertTrue(linkHandler.acceptUrl("http://bandcamp.com/?show=2")); assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/mobile/24/bootstrap_data")); @@ -42,6 +43,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest { @Test public void testGetId() { + assertEquals("Featured", linkHandler.getId("http://bandcamp.com/api/mobile/24/bootstrap_data")); assertEquals("Featured", linkHandler.getId("https://bandcamp.com/api/mobile/24/bootstrap_data")); assertEquals("Radio", linkHandler.getId("http://bandcamp.com/?show=1")); assertEquals("Radio", linkHandler.getId("https://bandcamp.com/api/bcweekly/1/list")); From c9e9953bb085a1231817b9d29675587d10d92470 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Tue, 22 Dec 2020 20:02:53 +0100 Subject: [PATCH 77/91] [Bandcamp] Fix channel link handler factory --- .../BandcampChannelLinkHandlerFactory.java | 26 +++++++++++++++---- ...BandcampChannelLinkHandlerFactoryTest.java | 2 ++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index 762c5d9b3..2cf956cc9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -51,15 +51,31 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { } /** - * Accepts only pages that do not lead to an album or track. Supports external pages. + * Accepts only pages that lead to the root of an artist profile. Supports external pages. */ @Override public boolean onAcceptUrl(final String url) throws ParsingException { - // Exclude URLs that lead to a track or album - if (url.matches(".*/(album|track)/.*")) return false; + // https: | | artist.bandcamp.com | releases + // 0 1 2 3 + String[] splitUrl = url.split("/"); - // Test whether domain is supported - return BandcampExtractorHelper.isSupportedDomain(url); + // URL is too short + if (splitUrl.length < 3) return false; + + // Must have "releases" as segment after url or none at all + if (splitUrl.length > 3 && !splitUrl[3].equals("releases")) { + + return false; + + } else { + if (splitUrl[2].equals("daily.bandcamp.com")) { + // Refuse links to daily.bandcamp.com as that is not an artist + return false; + } + + // Test whether domain is supported + return BandcampExtractorHelper.isSupportedDomain(url); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index 499704b99..6052a2549 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -35,6 +35,8 @@ public class BandcampChannelLinkHandlerFactoryTest { assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); + assertFalse(linkHandler.acceptUrl("https://daily.bandcamp.com/")); + assertFalse(linkHandler.acceptUrl("https://daily.bandcamp.com/best-of-2020/bandcamp-daily-staffers-on-their-favorite-albums-of-2020")); // External URLs assertTrue(linkHandler.acceptUrl("http://interovgm.com/releases/")); From 07570554b6a6ecdfc395901a9c827a583a00d1d5 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 15 Jan 2021 21:55:40 +0100 Subject: [PATCH 78/91] Fix build and optimize imports --- .../extractors/BandcampSearchExtractor.java | 10 ++++++++++ .../extractors/BandcampStreamExtractor.java | 13 +++++++++++++ .../bandcamp/BandcampChannelExtractorTest.java | 2 +- .../BandcampChannelLinkHandlerFactoryTest.java | 2 +- .../bandcamp/BandcampFeaturedExtractorTest.java | 3 +-- .../bandcamp/BandcampPlaylistExtractorTest.java | 3 +-- .../BandcampPlaylistLinkHandlerFactoryTest.java | 3 +-- .../bandcamp/BandcampRadioExtractorTest.java | 3 +-- .../bandcamp/BandcampRadioStreamExtractorTest.java | 2 +- .../bandcamp/BandcampSearchExtractorTest.java | 7 ++----- .../BandcampSearchQueryHandlerFactoryTest.java | 2 +- .../bandcamp/BandcampStreamExtractorTest.java | 4 +--- .../BandcampStreamLinkHandlerFactoryTest.java | 2 +- .../bandcamp/BandcampSuggestionExtractorTest.java | 2 +- 14 files changed, 36 insertions(+), 22 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java index 23097171e..6888d6932 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -8,10 +8,12 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; @@ -19,6 +21,8 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem. import javax.annotation.Nonnull; import java.io.IOException; +import java.util.Collections; +import java.util.List; public class BandcampSearchExtractor extends SearchExtractor { @@ -37,6 +41,12 @@ public class BandcampSearchExtractor extends SearchExtractor { return false; } + @Nonnull + @Override + public List getMetaInfo() throws ParsingException { + return Collections.emptyList(); + } + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { // okay apparently this is where we DOWNLOAD the page and then COMMIT its ENTRIES to an INFOITEMPAGE final String html = getDownloader().get(page.getUrl()).responseBody(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 1f37176bf..3409a4cd2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -9,6 +9,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -319,4 +320,16 @@ public class BandcampStreamExtractor extends StreamExtractor { public String getSupportInfo() { return ""; } + + @Nonnull + @Override + public List getStreamSegments() throws ParsingException { + return Collections.emptyList(); + } + + @Nonnull + @Override + public List getMetaInfo() throws ParsingException { + return Collections.emptyList(); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 327dc6b9a..b67be1832 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index 6052a2549..89c81fb9e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index 50d992a16..8a9756f91 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -4,8 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index b695236d9..ce6a54cf0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -4,14 +4,13 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; -import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java index 561d8f7b6..e1b7949f8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistLinkHandlerFactoryTest.java @@ -4,11 +4,10 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampPlaylistLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java index c7b4e3579..064521d9b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java @@ -4,12 +4,11 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; -import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor; import java.io.IOException; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index e024fefa7..8773c5564 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index f74fb9256..124265168 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -4,20 +4,17 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelInfoItemExtractor; -import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import java.io.IOException; - import javax.annotation.Nullable; +import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java index 4271bc97c..c39115247 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index dd979d554..a080de674 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -16,10 +16,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; -import java.util.Calendar; import java.util.Collections; import java.util.List; -import java.util.TimeZone; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java index a65b09567..17d983cec 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java index 9764d995a..95f7de3a9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSuggestionExtractorTest.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSuggestionExtractor; From 5bf9fddba930fb0555b5ec77221bf6faa00d79e6 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 15 Feb 2021 22:45:02 +0100 Subject: [PATCH 79/91] Code improvements --- .../bandcamp/extractors/BandcampRadioStreamExtractor.java | 4 +--- .../bandcamp/BandcampChannelLinkHandlerFactoryTest.java | 1 - .../bandcamp/BandcampStreamLinkHandlerFactoryTest.java | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 57ca8b187..a0c109810 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -27,11 +27,9 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { private JsonObject showInfo; - private LinkHandler linkHandler; public BandcampRadioStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); - this.linkHandler = linkHandler; } static JsonObject query(final int id) throws ParsingException { @@ -64,7 +62,7 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nonnull @Override public String getUrl() throws ParsingException { - return linkHandler.getUrl(); + return getLinkHandler().getUrl(); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java index 89c81fb9e..aa7a93d8a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; import static org.junit.Assert.*; -import static org.junit.Assert.assertTrue; /** * Test for {@link BandcampChannelLinkHandlerFactory} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java index 17d983cec..85094827e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; import static org.junit.Assert.*; -import static org.junit.Assert.assertFalse; /** * Test for {@link BandcampStreamLinkHandlerFactory} From 16973126a0b2c68f5cfbfad2618f8248ff11e654 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 16 Feb 2021 20:25:48 +0100 Subject: [PATCH 80/91] Use Collections.emptyList() instead of Collections.EMPTY_LIST or new ArrayList() --- .../extractor/linkhandler/ListLinkHandlerFactory.java | 3 ++- .../bandcamp/extractors/BandcampRadioStreamExtractor.java | 3 ++- .../bandcamp/extractors/BandcampStreamExtractor.java | 8 ++++---- .../bandcamp/extractors/BandcampSuggestionExtractor.java | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 86991c4df..11b94c7a5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Utils; import java.util.ArrayList; +import java.util.Collections; import java.util.List; public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { @@ -13,7 +14,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /////////////////////////////////// public List getContentFilter(String url) throws ParsingException { - return new ArrayList<>(0); + return Collections.emptyList(); } public String getSortFilter(String url) throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index a0c109810..01ed5ad5a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -20,6 +20,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; @@ -137,6 +138,6 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nonnull @Override public List getTags() { - return new ArrayList<>(); + return Collections.emptyList(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 3409a4cd2..856ad54ba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -211,24 +211,24 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public List getVideoStreams() { - return Collections.EMPTY_LIST; + return Collections.emptyList(); } @Override public List getVideoOnlyStreams() { - return Collections.EMPTY_LIST; + return Collections.emptyList(); } @Nonnull @Override public List getSubtitlesDefault() { - return new ArrayList<>(); + return Collections.emptyList(); } @Nonnull @Override public List getSubtitles(MediaFormat format) { - return new ArrayList<>(); + return Collections.emptyList(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index bb1ababb2..cc7d384f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.io.IOException; import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class BandcampSuggestionExtractor extends SuggestionExtractor { @@ -48,7 +49,7 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { } catch (final JsonParserException e) { e.printStackTrace(); - return new ArrayList<>(); + return Collections.emptyList(); } From 54b8e54f8007df6e6820a5f9d42f4bd6e141c9c2 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 16 Feb 2021 20:59:13 +0100 Subject: [PATCH 81/91] Fix potential NPE --- .../services/bandcamp/extractors/BandcampStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 856ad54ba..d8e6fdede 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -194,7 +194,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public String getHlsUrl() { - return null; + return ""; } @Override From 54aa5b3042b1f4cc53a06d78ef9aac4be6996a4a Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 16 Feb 2021 21:09:35 +0100 Subject: [PATCH 82/91] Use propper structure in KioskExtractors Made BandCampRadioExtractor a Kiosk which holds StreamInfoItems and not InfoItems. --- .../extractors/BandcampFeaturedExtractor.java | 48 +++++++++---------- .../extractors/BandcampRadioExtractor.java | 45 +++++++---------- .../BandcampFeaturedExtractorTest.java | 1 + .../bandcamp/BandcampRadioExtractorTest.java | 5 +- 4 files changed, 45 insertions(+), 54 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 2019153a6..12214601a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -24,6 +24,8 @@ public class BandcampFeaturedExtractor extends KioskExtractor public static final String KIOSK_FEATURED = "Featured"; public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data"; + private JsonObject json; + public BandcampFeaturedExtractor(final StreamingService streamingService, final ListLinkHandler listLinkHandler, final String kioskId) { super(streamingService, listLinkHandler, kioskId); @@ -31,7 +33,15 @@ public class BandcampFeaturedExtractor extends KioskExtractor @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - + try { + json = JsonParser.object().from( + getDownloader().post( + FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes() + ).responseBody() + ); + } catch (final JsonParserException e) { + throw new ParsingException("Could not parse Bandcamp featured API response", e); + } } @Nonnull @@ -46,35 +56,23 @@ public class BandcampFeaturedExtractor extends KioskExtractor final PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); - try { + final JsonArray featuredStories = json.getObject("feed_content") + .getObject("stories") + .getArray("featured"); + for (int i = 0; i < featuredStories.size(); i++) { + final JsonObject featuredStory = featuredStories.getObject(i); - final JsonObject json = JsonParser.object().from( - getDownloader().post( - FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes() - ).responseBody() - ); - - final JsonArray featuredStories = json.getObject("feed_content") - .getObject("stories") - .getArray("featured"); - - for (int i = 0; i < featuredStories.size(); i++) { - final JsonObject featuredStory = featuredStories.getObject(i); - - if (featuredStory.isNull("album_title")) { - // Is not an album, ignore - continue; - } - - c.commit(new BandcampPlaylistInfoItemFeaturedExtractor(featuredStory)); + if (featuredStory.isNull("album_title")) { + // Is not an album, ignore + continue; } - return new InfoItemsPage<>(c, null); - } catch (final JsonParserException e) { - e.printStackTrace(); - throw new ParsingException("JSON error", e); + c.commit(new BandcampPlaylistInfoItemFeaturedExtractor(featuredStory)); } + + return new InfoItemsPage<>(c, null); + } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java index 61c9e7326..8c3a7baf3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java @@ -6,8 +6,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -15,16 +13,19 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; -public class BandcampRadioExtractor extends KioskExtractor { +public class BandcampRadioExtractor extends KioskExtractor { public static final String KIOSK_RADIO = "Radio"; public static final String RADIO_API_URL = "https://bandcamp.com/api/bcweekly/1/list"; + private JsonObject json = null; + public BandcampRadioExtractor(final StreamingService streamingService, final ListLinkHandler linkHandler, final String kioskId) { super(streamingService, linkHandler, kioskId); @@ -32,7 +33,12 @@ public class BandcampRadioExtractor extends KioskExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - + try { + json = JsonParser.object().from( + getDownloader().get(RADIO_API_URL).responseBody()); + } catch (final JsonParserException e) { + throw new ExtractionException("Could not parse Bandcamp Radio API response", e); + } } @Nonnull @@ -43,36 +49,21 @@ public class BandcampRadioExtractor extends KioskExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final InfoItemsCollector c = new StreamInfoItemsCollector(getServiceId()); + public InfoItemsPage getInitialPage() { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - try { + final JsonArray radioShows = json.getArray("results"); - final JsonObject json = JsonParser.object().from( - getDownloader().get( - RADIO_API_URL - ).responseBody() - ); - - final JsonArray radioShows = json.getArray("results"); - - for (int i = 0; i < radioShows.size(); i++) { - final JsonObject radioShow = radioShows.getObject(i); - - c.commit( - new BandcampRadioInfoItemExtractor(radioShow) - ); - } - - } catch (final JsonParserException e) { - e.printStackTrace(); + for (int i = 0; i < radioShows.size(); i++) { + final JsonObject radioShow = radioShows.getObject(i); + collector.commit(new BandcampRadioInfoItemExtractor(radioShow)); } - return new InfoItemsPage(c, null); + return new InfoItemsPage<>(collector, null); } @Override - public InfoItemsPage getPage(final Page page) { + public InfoItemsPage getPage(final Page page) { return null; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index 8a9756f91..1be966dea 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -31,6 +31,7 @@ public class BandcampFeaturedExtractorTest implements BaseListExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (BandcampFeaturedExtractor) Bandcamp .getKioskList().getDefaultKioskExtractor(); + extractor.fetchPage(); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java index 064521d9b..70838bd7e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java @@ -5,11 +5,11 @@ package org.schabi.newpipe.extractor.services.bandcamp; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; -import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; import java.util.List; @@ -31,11 +31,12 @@ public class BandcampRadioExtractorTest implements BaseListExtractorTest { extractor = (BandcampRadioExtractor) Bandcamp .getKioskList() .getExtractorById("Radio", null); + extractor.fetchPage(); } @Test public void testRadioCount() throws ExtractionException, IOException { - final List list = Bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); + final List list = extractor.getInitialPage().getItems(); assertTrue(list.size() > 300); } From 98268e351c06d30009965bd2698246cff6a2a405 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 19 Feb 2021 13:35:44 +0100 Subject: [PATCH 83/91] Move radio URL check into a function --- .../bandcamp/extractors/BandcampExtractorHelper.java | 9 +++++++++ .../linkHandler/BandcampFeaturedLinkHandlerFactory.java | 6 +++--- .../linkHandler/BandcampStreamLinkHandlerFactory.java | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 1d76c4612..b19c41dd1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -148,6 +148,15 @@ public class BandcampExtractorHelper { } } + /** + * Whether the URL points to a radio kiosk. + * @param url the URL to check + * @return true if the URL is https://bandcamp.com/?show=SHOW_ID + */ + public static boolean isRadioUrl(final String url) { + return url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+"); + } + static DateWrapper parseDate(final String textDate) throws ParsingException { try { final ZonedDateTime zonedDateTime = ZonedDateTime.parse( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java index 6286e31da..ea6254979 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import org.schabi.newpipe.extractor.utils.Utils; import java.util.List; @@ -28,7 +29,7 @@ public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) { url = Utils.replaceHttpWithHttps(url); - if (url.matches("https://bandcamp\\.com/\\?show=\\d+") || url.equals(RADIO_API_URL)) { + if (BandcampExtractorHelper.isRadioUrl(url) || url.equals(RADIO_API_URL)) { return KIOSK_RADIO; } else if (url.equals(FEATURED_API_URL)) { return KIOSK_FEATURED; @@ -40,7 +41,6 @@ public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(String url) { url = Utils.replaceHttpWithHttps(url); - return url.equals(FEATURED_API_URL) || (url.equals(RADIO_API_URL) - || url.matches("https://bandcamp\\.com/\\?show=\\d+")); + return url.equals(FEATURED_API_URL) || (url.equals(RADIO_API_URL) || BandcampExtractorHelper.isRadioUrl(url)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java index fa0435daa..37ff29c0c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -20,7 +20,7 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { */ @Override public String getId(final String url) throws ParsingException { - if (url.matches("https?://bandcamp\\.com/\\?show=\\d+")) { + if (BandcampExtractorHelper.isRadioUrl(url)) { return url.split("bandcamp.com/\\?show=")[1]; } else { return getUrl(url); @@ -48,7 +48,7 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { public boolean onAcceptUrl(final String url) throws ParsingException { // Accept Bandcamp radio - if (url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+")) return true; + if (BandcampExtractorHelper.isRadioUrl(url)) return true; // Don't accept URLs that don't point to a track if (!url.toLowerCase().matches("https?://.+\\..+/track/.+")) return false; From fa61b864f20e756156be96a4b073f871b3b1c785 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 19 Feb 2021 13:37:24 +0100 Subject: [PATCH 84/91] Code improvements --- .../bandcamp/extractors/BandcampExtractorHelper.java | 6 +++--- .../bandcamp/extractors/BandcampRadioInfoItemExtractor.java | 2 +- .../bandcamp/extractors/BandcampSuggestionExtractor.java | 3 +-- .../bandcamp/BandcampStreamLinkHandlerFactoryTest.java | 1 + 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index b19c41dd1..c9a82a075 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -43,7 +43,7 @@ public class BandcampExtractorHelper { /** * Translate all these parameters together to the URL of the corresponding album or track - * using the mobile api + * using the mobile API */ public static String getStreamUrlFromIds(final long bandId, final long itemId, final String itemType) throws ParsingException { @@ -51,7 +51,7 @@ public class BandcampExtractorHelper { try { final String jsonString = NewPipe.getDownloader().get( "https://bandcamp.com/api/mobile/22/tralbum_details?band_id=" + bandId - + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.substring(0, 1)) + + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.charAt(0)) .responseBody(); return JsonParser.object().from(jsonString).getString("bandcamp_url").replace("http://", "https://"); @@ -88,7 +88,7 @@ public class BandcampExtractorHelper { } - return String.valueOf(result); + return result.toString(); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index b2e2d3642..c575a35fe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -14,7 +14,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { - private JsonObject show; + private final JsonObject show; public BandcampRadioInfoItemExtractor(final JsonObject radioShow) { show = radioShow; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index cc7d384f3..9196e7c75 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -37,7 +37,7 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { final JsonArray jsonArray = fuzzyResults.getObject("auto") .getArray("results"); - final ArrayList suggestions = new ArrayList<>(); + final List suggestions = new ArrayList<>(); for (final Object fuzzyResult : jsonArray) { final String res = ((JsonObject) fuzzyResult).getString("name"); @@ -52,6 +52,5 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { return Collections.emptyList(); } - } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java index 85094827e..bf99fa308 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -48,5 +48,6 @@ public class BandcampStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); assertTrue(linkHandler.acceptUrl("https://interovgm.com/track/title")); assertTrue(linkHandler.acceptUrl("http://bandcamP.com/?show=38")); + assertTrue(linkHandler.acceptUrl("https://goodgoodblood-tl.bandcamp.com/track/when-it-all-wakes-up")); } } From 02920fafa88b3cdba4f4f94ca11c4ebdee52554b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 19 Feb 2021 15:51:02 +0100 Subject: [PATCH 85/91] Add isUploaderVerified() --- .../bandcamp/extractors/BandcampChannelExtractor.java | 5 +++++ .../extractors/BandcampChannelInfoItemExtractor.java | 5 +++++ .../bandcamp/extractors/BandcampPlaylistExtractor.java | 5 +++++ .../bandcamp/extractors/BandcampRadioInfoItemExtractor.java | 5 +++++ .../bandcamp/extractors/BandcampStreamExtractor.java | 5 +++++ .../BandcampDiscographStreamInfoItemExtractor.java | 5 +++++ .../BandcampPlaylistStreamInfoItemExtractor.java | 5 +++++ .../BandcampSearchStreamInfoItemExtractor.java | 5 +++++ .../services/bandcamp/BandcampChannelExtractorTest.java | 5 +++++ .../services/bandcamp/BandcampPlaylistExtractorTest.java | 5 +++++ 10 files changed, 50 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 035d87a65..ef2835d26 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -94,6 +94,11 @@ public class BandcampChannelExtractor extends ChannelExtractor { return null; } + @Override + public boolean isVerified() throws ParsingException { + return false; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java index fb956791d..d25c82656 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java @@ -50,4 +50,9 @@ public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtracto public long getStreamCount() { return -1; } + + @Override + public boolean isVerified() throws ParsingException { + return false; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 8915938fc..58a701d90 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -100,6 +100,11 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { } } + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + @Override public long getStreamCount() { return trackInfo.size(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index c575a35fe..a684e7eb8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -78,6 +78,11 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { return ""; } + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + @Override public boolean isAd() { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index d8e6fdede..53e6d3970 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -95,6 +95,11 @@ public class BandcampStreamExtractor extends StreamExtractor { return albumJson.getString("artist"); } + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + @Nullable @Override public String getTextualUploadDate() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java index 15b044981..16b82d8c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -18,6 +18,11 @@ public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInf return discograph.getString("band_name"); } + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + @Override public String getName() { return discograph.getString("title"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index e90a12aa1..12bf6816c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -53,6 +53,11 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI return ""; } + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + /** * Each track can have its own cover art. Therefore, unless a substitute is provided, * the thumbnail is extracted using a stream extractor. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index adeea0519..d096d098e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -24,6 +24,11 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte } } + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + @Override public String getName() throws ParsingException { return resultInfo.getElementsByClass("heading").text(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index b67be1832..78b392558 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -57,6 +57,11 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { assertEquals(-1, extractor.getSubscriberCount()); } + @Override + public void testVerified() throws Exception { + assertFalse(extractor.isVerified()); + } + @Override public void testRelatedItems() throws Exception { // not implemented diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index ce6a54cf0..31109951a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -132,6 +132,11 @@ public class BandcampPlaylistExtractorTest { assertEquals(5, extractor.getStreamCount()); } + @Override + public void testUploaderVerified() throws Exception { + assertFalse(extractor.isUploaderVerified()); + } + @Test public void testInitialPage() throws IOException, ExtractionException { assertNotNull(extractor.getInitialPage().getItems().get(0)); From ea120a463725bdddf22577888f85dd42c7d82db2 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 20 Feb 2021 00:23:32 +0100 Subject: [PATCH 86/91] remove print stacktrace use interface --- .../services/bandcamp/extractors/BandcampStreamExtractor.java | 2 +- .../bandcamp/extractors/BandcampSuggestionExtractor.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 53e6d3970..005078a7a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -311,7 +311,7 @@ public class BandcampStreamExtractor extends StreamExtractor { public List getTags() { final Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords"); - final ArrayList tags = new ArrayList<>(); + final List tags = new ArrayList<>(); for (final Element e : tagElements) { tags.add(e.text()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index 9196e7c75..4eaa830cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -47,8 +47,6 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { return suggestions; } catch (final JsonParserException e) { - e.printStackTrace(); - return Collections.emptyList(); } From adde4332d1f21220f18d5b0079210a733324f203 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 20 Feb 2021 23:58:49 +0100 Subject: [PATCH 87/91] Code improvements --- .../bandcamp/extractors/BandcampExtractorHelper.java | 10 +++++----- .../BandcampDiscographStreamInfoItemExtractor.java | 5 ----- .../BandcampPlaylistStreamInfoItemExtractor.java | 5 ----- .../BandcampSearchStreamInfoItemExtractor.java | 5 ----- .../BandcampStreamInfoItemExtractor.java | 9 ++++++--- 5 files changed, 11 insertions(+), 23 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index c9a82a075..c4011fff6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -119,16 +119,16 @@ public class BandcampExtractorHelper { /** * @param id The image ID * @param album Whether this is the cover of an album - * @return Url of image with this ID in size 10 which is 1200x1200 (we could also choose size 0 - * but we don't want something as large as 3460x3460 here, do we?) + * @return URL of image with this ID in size 10 which is 1200x1200 (we could also choose size 0 + * but we don't want something as large as 3460x3460 here) */ public static String getImageUrl(final long id, final boolean album) { return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg"; } /** - * @return true if the given url looks like it comes from a bandcamp custom domain - * or if it comes from bandcamp.com itself + * @return true if the given URL looks like it comes from a bandcamp custom domain + * or if it comes from bandcamp.com itself */ public static boolean isSupportedDomain(final String url) throws ParsingException { @@ -151,7 +151,7 @@ public class BandcampExtractorHelper { /** * Whether the URL points to a radio kiosk. * @param url the URL to check - * @return true if the URL is https://bandcamp.com/?show=SHOW_ID + * @return true if the URL matches https://bandcamp.com/?show=SHOW_ID */ public static boolean isRadioUrl(final String url) { return url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java index 16b82d8c8..15b044981 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -18,11 +18,6 @@ public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInf return discograph.getString("band_name"); } - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Override public String getName() { return discograph.getString("title"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index 12bf6816c..e90a12aa1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -53,11 +53,6 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI return ""; } - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - /** * Each track can have its own cover art. Therefore, unless a substitute is provided, * the thumbnail is extracted using a stream extractor. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index d096d098e..adeea0519 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -24,11 +24,6 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte } } - @Override - public boolean isUploaderVerified() throws ParsingException { - return false; - } - @Override public String getName() throws ParsingException { return resultInfo.getElementsByClass("heading").text(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java index 485433f9d..5f792fdef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; @@ -44,9 +45,11 @@ public abstract class BandcampStreamInfoItemExtractor implements StreamInfoItemE return null; } - /** - * There are no ads just like that, duh - */ + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + @Override public boolean isAd() { return false; From c07db80ef068f885ca10709551e4591bd385cdac Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 21 Feb 2021 12:15:45 +0100 Subject: [PATCH 88/91] Add BASE_URL and BASE_API_URL to BandcampExtractorHelper --- .../services/bandcamp/BandcampService.java | 5 +++-- .../extractors/BandcampExtractorHelper.java | 14 +++++++++----- .../extractors/BandcampFeaturedExtractor.java | 4 +++- .../extractors/BandcampRadioExtractor.java | 4 +++- .../extractors/BandcampRadioInfoItemExtractor.java | 3 ++- .../extractors/BandcampRadioStreamExtractor.java | 6 +++--- .../extractors/BandcampSuggestionExtractor.java | 4 +++- .../BandcampSearchQueryHandlerFactory.java | 4 +++- .../BandcampStreamLinkHandlerFactory.java | 4 +++- 9 files changed, 32 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 1d96684b2..145693285 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.util.Collections; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO; @@ -32,7 +33,7 @@ public class BandcampService extends StreamingService { @Override public String getBaseUrl() { - return "https://bandcamp.com"; + return BASE_URL; } @Override @@ -113,7 +114,7 @@ public class BandcampService extends StreamingService { @Override public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) { - if (linkHandler.getUrl().matches("https?://bandcamp\\.com/\\?show=\\d+")) + if (BandcampExtractorHelper.isRadioUrl(linkHandler.getUrl())) return new BandcampRadioStreamExtractor(this, linkHandler); else return new BandcampStreamExtractor(this, linkHandler); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index c4011fff6..dad376d58 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -20,10 +20,14 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Locale; public class BandcampExtractorHelper { + public static final String BASE_URL = "https://bandcamp.com"; + public static final String BASE_API_URL = BASE_URL + "/api"; + /** *

Get an attribute of a web page as JSON * @@ -50,11 +54,12 @@ public class BandcampExtractorHelper { try { final String jsonString = NewPipe.getDownloader().get( - "https://bandcamp.com/api/mobile/22/tralbum_details?band_id=" + bandId + BASE_API_URL + "/mobile/22/tralbum_details?band_id=" + bandId + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.charAt(0)) .responseBody(); - return JsonParser.object().from(jsonString).getString("bandcamp_url").replace("http://", "https://"); + return JsonParser.object().from(jsonString) + .getString("bandcamp_url").replace("http://", "https://"); } catch (final JsonParserException | ReCaptchaException | IOException e) { throw new ParsingException("Ids could not be translated to URL", e); @@ -70,7 +75,7 @@ public class BandcampExtractorHelper { final StringBuilder result = new StringBuilder(); // Remove empty strings - final ArrayList list = new ArrayList<>(Arrays.asList(strings)); + final List list = new ArrayList<>(Arrays.asList(strings)); for (int i = list.size() - 1; i >= 0; i--) { if (Utils.isNullOrEmpty(list.get(i)) || list.get(i).equals("null")) { list.remove(i); @@ -101,7 +106,7 @@ public class BandcampExtractorHelper { return JsonParser.object().from( NewPipe.getDownloader().post( - "https://bandcamp.com/api/mobile/22/band_details", + BASE_API_URL + "/mobile/22/band_details", null, JsonWriter.string() .object() @@ -165,6 +170,5 @@ public class BandcampExtractorHelper { } catch (final DateTimeException e) { throw new ParsingException("Could not parse date '" + textDate + "'", e); } - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 12214601a..93b1b0297 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -19,10 +19,12 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL; + public class BandcampFeaturedExtractor extends KioskExtractor { public static final String KIOSK_FEATURED = "Featured"; - public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data"; + public static final String FEATURED_API_URL = BASE_API_URL + "/mobile/24/bootstrap_data"; private JsonObject json; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java index 8c3a7baf3..da062ab04 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java @@ -19,10 +19,12 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL; + public class BandcampRadioExtractor extends KioskExtractor { public static final String KIOSK_RADIO = "Radio"; - public static final String RADIO_API_URL = "https://bandcamp.com/api/bcweekly/1/list"; + public static final String RADIO_API_URL = BASE_API_URL + "/bcweekly/1/list"; private JsonObject json = null; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index a684e7eb8..2331f258c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @@ -49,7 +50,7 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUrl() { - return "https://bandcamp.com/?show=" + show.getInt("id"); + return BASE_URL + "/?show=" + show.getInt("id"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 01ed5ad5a..7d2543c8a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -23,7 +23,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.*; public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @@ -36,7 +36,7 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { static JsonObject query(final int id) throws ParsingException { try { return JsonParser.object().from( - NewPipe.getDownloader().get("https://bandcamp.com/api/bcweekly/1/get?id=" + id).responseBody() + NewPipe.getDownloader().get(BASE_API_URL + "/bcweekly/1/get?id=" + id).responseBody() ); } catch (final IOException | ReCaptchaException | JsonParserException e) { throw new ParsingException("could not get show data", e); @@ -88,7 +88,7 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nonnull @Override public String getUploaderAvatarUrl() { - return "https://bandcamp.com/img/buttons/bandcamp-button-circle-whitecolor-512.png"; + return BASE_URL + "/img/buttons/bandcamp-button-circle-whitecolor-512.png"; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index 4eaa830cb..46283345b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -18,9 +18,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL; + public class BandcampSuggestionExtractor extends SuggestionExtractor { - private static final String AUTOCOMPLETE_URL = "https://bandcamp.com/api/fuzzysearch/1/autocomplete?q="; + private static final String AUTOCOMPLETE_URL = BASE_API_URL + "/fuzzysearch/1/autocomplete?q="; public BandcampSuggestionExtractor(final StreamingService service) { super(service); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java index 518170b07..9e295d5de 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java @@ -9,6 +9,8 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.List; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; + public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory { @@ -17,7 +19,7 @@ public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory throws ParsingException { try { - return "https://bandcamp.com/search?q=" + + return BASE_URL + "/search?q=" + URLEncoder.encode(query, "UTF-8") + "&page=1"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java index 37ff29c0c..96b9f9d8e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -6,6 +6,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; + /** *

Tracks don't have standalone ids, they are always in combination with the band id. * That's why id = url.

@@ -34,7 +36,7 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getUrl(final String input) { if (input.matches("\\d+")) { - return "https://bandcamp.com/?show=" + input; + return BASE_URL + "/?show=" + input; } else { return input; } From b9e8ee8450c5222c80d42af59d1e9f197766be87 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 21 Feb 2021 12:35:14 +0100 Subject: [PATCH 89/91] Rename BandcampExtractorHelper.smartConcatenate(String[], String) to Utils.nonEmptyAndNullJoin(String, String[]) --- .../extractors/BandcampExtractorHelper.java | 29 ------------------- .../extractors/BandcampStreamExtractor.java | 6 ++-- .../schabi/newpipe/extractor/utils/Utils.java | 13 +++++++++ 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index dad376d58..34bd3c6a2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -67,35 +67,6 @@ public class BandcampExtractorHelper { } - /** - * Concatenate all non-null and non-empty strings together while separating them using - * the comma parameter - */ - public static String smartConcatenate(final String[] strings, final String comma) { - final StringBuilder result = new StringBuilder(); - - // Remove empty strings - final List list = new ArrayList<>(Arrays.asList(strings)); - for (int i = list.size() - 1; i >= 0; i--) { - if (Utils.isNullOrEmpty(list.get(i)) || list.get(i).equals("null")) { - list.remove(i); - } - } - - // Append remaining strings to result - for (int i = 0; i < list.size(); i++) { - result.append(list.get(i)); - - if (i != list.size() - 1) { - // This is not the last iteration yet - result.append(comma); - } - - } - - return result.toString(); - } - /** * Fetch artist details from mobile endpoint. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 005078a7a..99a19cecd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -150,12 +151,13 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public Description getDescription() { - final String s = BandcampExtractorHelper.smartConcatenate( + final String s = Utils.nonEmptyAndNullJoin( + "\n\n", new String[]{ current.getString("about"), current.getString("lyrics"), current.getString("credits") - }, "\n\n" + } ); return new Description(s, Description.PLAIN_TEXT); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 721bc96c3..09574ab51 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -271,4 +271,17 @@ public class Utils { return join(delimiter, list); } + /** + * Concatenate all non-null, non-empty and strings which are not equal to "null". + */ + public static String nonEmptyAndNullJoin(final String delimiter, final String[] elements) { + final List list = Arrays.asList(elements); + for (int i = list.size() - 1; i >= 0; i--) { + if (isNullOrEmpty(list.get(i)) || list.get(i).equals("null")) { + list.remove(i); + } + } + + return join(delimiter, list); + } } From 70814dcfefbc6eaae9aa88a9190d3fa582b0376b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 21 Feb 2021 12:51:47 +0100 Subject: [PATCH 90/91] Fix Utils.nonEmptyAndNullJoin When using the index here, it the index needs to be decremented once an element is removed. To cirecumvent this, the native Collections.removeIf() method is used. --- .../org/schabi/newpipe/extractor/utils/Utils.java | 11 +++-------- .../org/schabi/newpipe/extractor/utils/UtilsTest.java | 1 + 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 09574ab51..82ff58263 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -274,14 +274,9 @@ public class Utils { /** * Concatenate all non-null, non-empty and strings which are not equal to "null". */ - public static String nonEmptyAndNullJoin(final String delimiter, final String[] elements) { - final List list = Arrays.asList(elements); - for (int i = list.size() - 1; i >= 0; i--) { - if (isNullOrEmpty(list.get(i)) || list.get(i).equals("null")) { - list.remove(i); - } - } - + public static String nonEmptyAndNullJoin(final CharSequence delimiter, final String[] elements) { + final List list = new java.util.ArrayList<>(Arrays.asList(elements)); + list.removeIf(s -> isNullOrEmpty(s) || s.equals("null")); return join(delimiter, list); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java index e4a65505b..a81270de0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java @@ -20,6 +20,7 @@ public class UtilsTest { @Test public void testJoin() { assertEquals("some,random,stuff", Utils.join(",", Arrays.asList("some", "random", "stuff"))); + assertEquals("some,random,not-null,stuff", Utils.nonEmptyAndNullJoin(",", new String[]{"some", "null", "random", "", "not-null", null, "stuff"})); } @Test From a1688fe9534983b27899bbdf685f99ff6f25d4c9 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 21 Feb 2021 13:51:12 +0100 Subject: [PATCH 91/91] Move BandcampExtractorHelper.getJsonData(String, String) to JsonUtils --- .../extractors/BandcampExtractorHelper.java | 17 ---------- .../extractors/BandcampPlaylistExtractor.java | 2 +- .../extractors/BandcampStreamExtractor.java | 3 +- .../BandcampChannelLinkHandlerFactory.java | 3 +- .../newpipe/extractor/utils/JsonUtils.java | 34 +++++++++++++++++++ 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index 34bd3c6a2..547a0356e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -28,23 +28,6 @@ public class BandcampExtractorHelper { public static final String BASE_URL = "https://bandcamp.com"; public static final String BASE_API_URL = BASE_URL + "/api"; - /** - *

Get an attribute of a web page as JSON - * - *

Originally a part of bandcampDirect.

- * - * @param html The HTML where the JSON we're looking for is stored inside a - * variable inside some JavaScript block - * @param variable Name of the variable - * @return The JsonObject stored in the variable with this name - */ - public static JsonObject getJsonData(final String html, final String variable) - throws JsonParserException, ArrayIndexOutOfBoundsException { - final Document document = Jsoup.parse(html); - final String json = document.getElementsByAttribute(variable).attr(variable); - return JsonParser.object().from(json); - } - /** * Translate all these parameters together to the URL of the corresponding album or track * using the mobile API diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 58a701d90..8cc21a6e7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -21,7 +21,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getJsonData; +import static org.schabi.newpipe.extractor.utils.JsonUtils.getJsonData; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; public class BandcampPlaylistExtractor extends PlaylistExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 99a19cecd..344663525 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; @@ -62,7 +63,7 @@ public class BandcampStreamExtractor extends StreamExtractor { */ public static JsonObject getAlbumInfoJson(final String html) throws ParsingException { try { - return BandcampExtractorHelper.getJsonData(html, "data-tralbum"); + return JsonUtils.getJsonData(html, "data-tralbum"); } catch (final JsonParserException e) { throw new ParsingException("Faulty JSON; page likely does not contain album data", e); } catch (final ArrayIndexOutOfBoundsException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java index 2cf956cc9..727aec404 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; import java.util.List; @@ -25,7 +26,7 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { final String response = NewPipe.getDownloader().get(url).responseBody(); // Use band data embedded in website to extract ID - final JsonObject bandData = BandcampExtractorHelper.getJsonData(response, "data-band"); + final JsonObject bandData = JsonUtils.getJsonData(response, "data-band"); return String.valueOf(bandData.getLong("id")); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ad132b5ac..7a5bd6c42 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -2,6 +2,10 @@ package org.schabi.newpipe.extractor.utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.exceptions.ParsingException; import javax.annotation.Nonnull; @@ -99,4 +103,34 @@ public class JsonUtils { return result; } + /** + *

Get an attribute of a web page as JSON + * + *

Originally a part of bandcampDirect.

+ *

Example HTML:

+ * 
+     * {@code
+     * 
This is Sparta!

+     * }
+     *
+ *

Calling this function to get the attribute data-town returns the JsonObject for

+ * 
+     * {@code
+     *   {
+     *     "name": "Mycenae",
+     *     "country": "Greece"
+     *   }
+     * }
+     *
+ * @param html The HTML where the JSON we're looking for is stored inside a + * variable inside some JavaScript block + * @param variable Name of the variable + * @return The JsonObject stored in the variable with this name + */ + public static JsonObject getJsonData(final String html, final String variable) + throws JsonParserException, ArrayIndexOutOfBoundsException { + final Document document = Jsoup.parse(html); + final String json = document.getElementsByAttribute(variable).attr(variable); + return JsonParser.object().from(json); + } }