kavin/NewPipeExtractor
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2024-12-15 06:40:33 +05:30
Code Issues Projects Releases Wiki Activity

Workaround enourmous load times for long bandcamp playlists

Browse Source 
Additionally, get cover art from json instead of html
This commit is contained in:
Fynn Godau 2020-01-03 13:13:42 +01:00
parent 46e1f3922c
commit ba967f1a15
4 changed files with 50 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java
View File

@ -18,11 +18,19 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getJSONFromJavaScriptVariables;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson;
public class BandcampPlaylistExtractor extends PlaylistExtractor {
/**
* An arbitrarily chosen number above which cover arts won't be fetched individually for each track;
* instead, it will be assumed that every track has the same cover art as the album, which is not
* always the case.
*/
private static final int MAXIMUM_INDIVIDUAL_COVER_ARTS = 10;
private Document document;
private JSONObject albumJson;
private JSONArray trackInfo;
@ -57,11 +65,8 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content");
} catch (NullPointerException e) {
return "";
}
if (albumJson.isNull("art_id")) return "";
else return getImageUrl(albumJson.getLong("art_id"), true);
}
@Override
@ -104,13 +109,26 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
for (int i = 0; i < trackInfo.length(); i++) {
JSONObject track = trackInfo.getJSONObject(i);
collector.commit(new BandcampStreamInfoItemExtractor(
track.getString("title"),
getUploaderUrl() + track.getString("title_link"),
"",
track.getLong("duration"),
getService()
));
if (trackInfo.length() < MAXIMUM_INDIVIDUAL_COVER_ARTS) {
// Load cover art of every track individually
collector.commit(new BandcampStreamInfoItemExtractor(
track.getString("title"),
getUploaderUrl() + track.getString("title_link"),
"",
track.getLong("duration"),
getService()
));
} else {
// Pretend every track has the same cover art as the album
collector.commit(new BandcampStreamInfoItemExtractor(
track.getString("title"),
getUploaderUrl() + track.getString("title_link"),
getThumbnailUrl(),
"",
track.getLong("duration")
));
}
}
return new InfoItemsPage<>(collector, null);

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java
View File

@ -21,6 +21,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl;
public class BandcampStreamExtractor extends StreamExtractor {
private JSONObject albumJson;
@ -103,11 +105,8 @@ public class BandcampStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content");
} catch (NullPointerException e) {
return "";
}
if (albumJson.isNull("art_id")) return "";
else return getImageUrl(albumJson.getLong("art_id"), true);
}
@Nonnull

16
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java
View File

@ -52,6 +52,22 @@ public class BandcampPlaylistExtractorTest {
assertNotEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl());
}
/**
* Tests that no attempt to load every track's cover individually is made
*/
@Test(timeout = 10000L)
public void testDifferentTrackCoversDuration() throws ExtractionException, IOException {
PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything");
extractor.fetchPage();
/* All tracks in this album have the same cover art, but I don't know any albums with more than 10 tracks
* that has at least one track with a cover art different from the rest.
*/
List<StreamInfoItem> l = extractor.getInitialPage().getItems();
assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl());
assertEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl());
}
/**
* Test playlists with locked content
*/

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java
View File

@ -34,7 +34,6 @@ public class BandcampRadioExtractorTest {
@Test
public void testRadioCount() throws ExtractionException, IOException {
List<InfoItem> list = bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems();
System.out.println(list.size());
assertTrue(list.size() > 300);
}
}