From 979c5a75029275ebdd206db726d690a11b9d1db0 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 16 Apr 2020 16:08:14 +0200 Subject: [PATCH] Use our fork of nanojson --- build.gradle | 1 + extractor/build.gradle | 4 +- .../extractors/PeertubeAccountExtractor.java | 4 +- .../extractors/PeertubeStreamExtractor.java | 2 +- .../SoundcloudChannelExtractor.java | 9 +- .../SoundcloudChannelInfoItemExtractor.java | 5 +- .../soundcloud/SoundcloudParsingHelper.java | 7 +- .../SoundcloudPlaylistInfoItemExtractor.java | 11 +- .../soundcloud/SoundcloudSearchExtractor.java | 3 +- .../soundcloud/SoundcloudStreamExtractor.java | 8 +- .../SoundcloudStreamInfoItemExtractor.java | 3 +- .../extractors/YoutubeChannelExtractor.java | 62 ++++---- .../YoutubeChannelInfoItemExtractor.java | 19 +-- .../YoutubeMusicSearchExtractor.java | 38 ++--- .../extractors/YoutubePlaylistExtractor.java | 61 +++----- .../extractors/YoutubeSearchExtractor.java | 12 +- .../extractors/YoutubeStreamExtractor.java | 145 +++++++----------- .../YoutubeStreamInfoItemExtractor.java | 121 ++++++--------- .../YoutubeSuggestionExtractor.java | 2 +- .../extractors/YoutubeTrendingExtractor.java | 9 +- .../linkHandler/YoutubeParsingHelper.java | 12 +- timeago-parser/build.gradle | 4 +- 22 files changed, 219 insertions(+), 323 deletions(-) diff --git a/build.gradle b/build.gradle index a62598060..6f349545f 100644 --- a/build.gradle +++ b/build.gradle @@ -10,6 +10,7 @@ allprojects { repositories { jcenter() + maven { url "https://jitpack.io" } } } diff --git a/extractor/build.gradle b/extractor/build.gradle index 1b7fbf001..2138df88e 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -1,11 +1,11 @@ dependencies { implementation project(':timeago-parser') - implementation 'com.grack:nanojson:1.1' + implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' implementation 'org.jsoup:jsoup:1.9.2' implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' implementation 'org.nibor.autolink:autolink:0.8.0' testImplementation 'junit:junit:4.12' -} \ No newline at end of file +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index ecde0d43c..81cb0afae 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -109,7 +109,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if (null != response && !StringUtil.isBlank(response.responseBody())) { + if (response != null && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { @@ -169,7 +169,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor { try { json = JsonParser.object().from(responseBody); } catch (JsonParserException e) { - throw new ExtractionException("Unable to extract peertube channel data", e); + throw new ExtractionException("Unable to extract PeerTube channel data", e); } if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 2b7fa0945..3cf65154d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -167,7 +167,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { assertPageFetched(); List videoStreams = new ArrayList<>(); try { - JsonArray streams = json.getArray("files", new JsonArray()); + JsonArray streams = json.getArray("files"); for (Object s : streams) { if (!(s instanceof JsonObject)) continue; JsonObject stream = (JsonObject) s; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 853b6fc1a..3c51271c9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -16,6 +16,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; + @SuppressWarnings("WeakerAccess") public class SoundcloudChannelExtractor extends ChannelExtractor { private String userId; @@ -62,10 +64,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() { - return user.getObject("visuals", new JsonObject()) - .getArray("visuals", new JsonArray()) - .getObject(0, new JsonObject()) - .getString("visual_url"); + return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url"); } @Override @@ -80,7 +79,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getDescription() { - return user.getString("description", ""); + return user.getString("description", EMPTY_STRING); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java index 118b59cc0..641438e79 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { @@ -24,7 +25,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac @Override public String getThumbnailUrl() { - String avatarUrl = itemObject.getString("avatar_url", ""); + String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING); String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg"); return avatarUrlBetterResolution; } @@ -41,6 +42,6 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac @Override public String getDescription() { - return itemObject.getString("description", ""); + return itemObject.getString("description", EMPTY_STRING); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index a957ed810..4ea615168 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -28,6 +28,7 @@ import java.util.*; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { @@ -256,17 +257,17 @@ public class SoundcloudParsingHelper { @Nonnull static String getUploaderUrl(JsonObject object) { - String url = object.getObject("user").getString("permalink_url", ""); + String url = object.getObject("user").getString("permalink_url", EMPTY_STRING); return replaceHttpWithHttps(url); } @Nonnull static String getAvatarUrl(JsonObject object) { - String url = object.getObject("user", new JsonObject()).getString("avatar_url", ""); + String url = object.getObject("user").getString("avatar_url", EMPTY_STRING); return replaceHttpWithHttps(url); } public static String getUploaderName(JsonObject object) { - return object.getObject("user").getString("username", ""); + return object.getObject("user").getString("username", EMPTY_STRING); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java index 53123859b..ae6875e73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { @@ -31,7 +32,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr public String getThumbnailUrl() throws ParsingException { // Over-engineering at its finest if (itemObject.isString(ARTWORK_URL_KEY)) { - final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, ""); + final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING); if (!artworkUrl.isEmpty()) { String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); return artworkUrlBetterResolution; @@ -45,7 +46,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr // First look for track artwork url if (trackObject.isString(ARTWORK_URL_KEY)) { - String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, ""); + String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING); if (!artworkUrl.isEmpty()) { String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); return artworkUrlBetterResolution; @@ -53,8 +54,8 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr } // Then look for track creator avatar url - final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject()); - final String creatorAvatar = creator.getString(AVATAR_URL_KEY, ""); + final JsonObject creator = trackObject.getObject(USER_KEY); + final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING); if (!creatorAvatar.isEmpty()) return creatorAvatar; } } catch (Exception ignored) { @@ -63,7 +64,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr try { // Last resort, use user avatar url. If still not found, then throw exception. - return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, ""); + return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING); } catch (Exception e) { throw new ParsingException("Failed to extract playlist thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index 780e7e2b1..1e20818db 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -23,6 +23,7 @@ import java.net.MalformedURLException; import java.net.URL; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; public class SoundcloudSearchExtractor extends SearchExtractor { @@ -84,7 +85,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor { if (!(result instanceof JsonObject)) continue; //noinspection ConstantConditions JsonObject searchResult = (JsonObject) result; - String kind = searchResult.getString("kind", ""); + String kind = searchResult.getString("kind", EMPTY_STRING); switch (kind) { case "user": collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 9ebb3dd6e..1ee7ca334 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -34,6 +34,8 @@ import java.util.Locale; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; + public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -45,7 +47,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl()); - String policy = track.getString("policy", ""); + String policy = track.getString("policy", EMPTY_STRING); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { throw new ContentNotAvailableException("Content not available: policy " + policy); } @@ -78,9 +80,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public String getThumbnailUrl() { - String artworkUrl = track.getString("artwork_url", ""); + String artworkUrl = track.getString("artwork_url", EMPTY_STRING); if (artworkUrl.isEmpty()) { - artworkUrl = track.getObject("user").getString("avatar_url", ""); + artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING); } String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); return artworkUrlBetterResolution; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java index da0ca1b54..31a719ea4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -62,7 +63,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto @Override public String getThumbnailUrl() { - String artworkUrl = itemObject.getString("artwork_url", ""); + String artworkUrl = itemObject.getString("artwork_url", EMPTY_STRING); if (artworkUrl.isEmpty()) { artworkUrl = itemObject.getObject("user").getString("avatar_url"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 6a570da53..5a65f0569 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -72,22 +72,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor { while (level < 3) { final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization()); - final JsonObject endpoint = jsonResponse.getObject(1, EMPTY_OBJECT) - .getObject("response", EMPTY_OBJECT).getArray("onResponseReceivedActions", EMPTY_ARRAY) - .getObject(0, EMPTY_OBJECT).getObject("navigateAction", EMPTY_OBJECT) - .getObject("endpoint", EMPTY_OBJECT); + final JsonObject endpoint = jsonResponse.getObject(1).getObject("response") + .getArray("onResponseReceivedActions").getObject(0).getObject("navigateAction") + .getObject("endpoint"); - final String webPageType = endpoint - .getObject("commandMetadata", EMPTY_OBJECT) - .getObject("webCommandMetadata", EMPTY_OBJECT) + final String webPageType = endpoint.getObject("commandMetadata").getObject("webCommandMetadata") .getString("webPageType", EMPTY_STRING); - final String browseId = endpoint - .getObject("browseEndpoint", EMPTY_OBJECT) - .getString("browseId", EMPTY_STRING); + final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING); if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) { - if (!browseId.startsWith("UC")) { throw new ExtractionException("Redirected id is not pointing to a channel"); } @@ -131,10 +125,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public String getId() throws ParsingException { - final String channelId = initialData - .getObject("header", EMPTY_OBJECT) - .getObject("c4TabbedHeaderRenderer", EMPTY_OBJECT) - .getString("channelId", EMPTY_STRING); + final String channelId = initialData.getObject("header").getObject("c4TabbedHeaderRenderer") + .getString("channelId", EMPTY_STRING); if (!channelId.isEmpty()) { return channelId; @@ -170,11 +162,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { try { - String url = null; - try { - url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner") + String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner") .getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception ignored) {} + if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; } @@ -196,19 +186,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); - if (subscriberInfo != null) { + final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header").getObject("c4TabbedHeaderRenderer"); + if (c4TabbedHeaderRenderer.has("subscriberCountText")) { try { - return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo)); + return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer.getObject("subscriberCountText"))); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } } else { // If there's no subscribe button, the channel has the subscriber count disabled - if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) { - return -1; - } else { + if (c4TabbedHeaderRenderer.has("subscribeButton")) { return 0; + } else { + return -1; } } } @@ -260,7 +250,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private String getNextPageUrlFrom(JsonArray continuations) { - if (continuations == null) return ""; + if (continuations == null || continuations.isEmpty()) { + return ""; + } JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); @@ -277,7 +269,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object video : videos) { - if (((JsonObject) video).getObject("gridVideoRenderer") != null) { + if (((JsonObject) video).has("gridVideoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor( ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) { @Override @@ -302,8 +294,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { JsonObject videoTab = null; for (Object tab : tabs) { - if (((JsonObject) tab).getObject("tabRenderer") != null) { - if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) { + if (((JsonObject) tab).has("tabRenderer")) { + if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) { videoTab = ((JsonObject) tab).getObject("tabRenderer"); break; } @@ -314,13 +306,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ContentNotSupportedException("This channel has no Videos tab"); } - try { - if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("messageRenderer") - .getObject("text")).equals("This channel has no videos.")) - return null; - } catch (Exception ignored) {} + if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("messageRenderer") + .getObject("text")).equals("This channel has no videos.")) + return null; this.videoTab = videoTab; return videoTab; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 27c082359..68d1c48b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -70,14 +70,12 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getSubscriberCount() throws ParsingException { try { - final JsonObject subscriberCountObject = channelInfoItem.getObject("subscriberCountText"); - - if (subscriberCountObject == null) { + if (!channelInfoItem.has("subscriberCountText")) { // Subscription count is not available for this channel item. return -1; } - return Utils.mixedNumberWordToLong(getTextFromObject(subscriberCountObject)); + return Utils.mixedNumberWordToLong(getTextFromObject(channelInfoItem.getObject("subscriberCountText"))); } catch (Exception e) { throw new ParsingException("Could not get subscriber count", e); } @@ -86,14 +84,13 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getStreamCount() throws ParsingException { try { - final JsonObject videoCountObject = channelInfoItem.getObject("videoCountText"); - - if (videoCountObject == null) { + if (!channelInfoItem.has("videoCountText")) { // Video count is not available, channel probably has no public uploads. return -1; } - return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(videoCountObject))); + return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( + channelInfoItem.getObject("videoCountText")))); } catch (Exception e) { throw new ParsingException("Could not get stream count", e); } @@ -102,14 +99,12 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getDescription() throws ParsingException { try { - final JsonObject descriptionObject = channelInfoItem.getObject("descriptionSnippet"); - - if (descriptionObject == null) { + if (!channelInfoItem.has("descriptionSnippet")) { // Channel have no description. return null; } - return getTextFromObject(descriptionObject); + return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); } catch (Exception e) { throw new ParsingException("Could not get description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 51918e9ce..14ff1f11f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -37,6 +37,7 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeS import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; public class YoutubeMusicSearchExtractor extends SearchExtractor { private JsonObject initialData; @@ -128,14 +129,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer == null) { - return ""; - } - final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") - .getObject(0).getObject("didYouMeanRenderer"); - if (didYouMeanRenderer == null) { + final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("didYouMeanRenderer"); + if (!didYouMeanRenderer.has("correctedQuery")) { return ""; } return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); @@ -149,7 +146,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); for (Object content : contents) { - if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + if (((JsonObject) content).has("musicShelfRenderer")) { collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); } } @@ -162,7 +159,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); for (Object content : contents) { - if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + if (((JsonObject) content).has("musicShelfRenderer")) { return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); } } @@ -224,10 +221,6 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { throw new ParsingException("Could not parse JSON", e); } - if (ajaxJson.getObject("continuationContents") == null) { - return InfoItemsPage.emptyPage(); - } - final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); @@ -240,7 +233,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { - final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); + final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer", null); if (info != null) { final String searchType = getLinkHandler().getContentFilters().get(0); if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { @@ -290,7 +283,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); for (Object item : items) { final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { + if (menuNavigationItemRenderer.getObject("icon").getString("iconType", EMPTY_STRING).equals("ARTIST")) { return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); } } @@ -301,16 +294,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); - if (navigationEndpoint == null) { - return null; - } - - final String url = getUrlFromNavigationEndpoint(navigationEndpoint); - - if (url != null && !url.isEmpty()) { - return url; - } - throw new ParsingException("Could not get uploader url"); + return getUrlFromNavigationEndpoint(navigationEndpoint); } } @@ -480,7 +464,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { - if (continuations == null) { + if (continuations == null || continuations.isEmpty()) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index eef85aa9d..de6fd74c7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -47,23 +47,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private JsonObject getUploaderInfo() throws ParsingException { JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items"); - try { - JsonObject uploaderInfo = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer") - .getObject("videoOwner").getObject("videoOwnerRenderer"); - if (uploaderInfo != null) { - return uploaderInfo; - } - } catch (Exception ignored) {} + + JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); + if (videoOwner.has("videoOwnerRenderer")) { + return videoOwner.getObject("videoOwnerRenderer"); + } // we might want to create a loop here instead of using duplicated code - try { - JsonObject uploaderInfo = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer") - .getObject("videoOwner").getObject("videoOwnerRenderer"); - if (uploaderInfo != null) { - return uploaderInfo; - } - } catch (Exception e) { - throw new ParsingException("Could not get uploader info", e); + videoOwner = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); + if (videoOwner.has("videoOwnerRenderer")) { + return videoOwner.getObject("videoOwnerRenderer"); } throw new ParsingException("Could not get uploader info"); } @@ -89,33 +82,22 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public String getName() throws ParsingException { - try { - String name = getTextFromObject(playlistInfo.getObject("title")); - if (name != null) return name; - } catch (Exception ignored) {} - try { - return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); - } catch (Exception e) { - throw new ParsingException("Could not get playlist name", e); - } + String name = getTextFromObject(playlistInfo.getObject("title")); + if (name == null || !name.isEmpty()) return name; + + return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); } @Override public String getThumbnailUrl() throws ParsingException { - String url = null; + String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - try { - url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception ignored) {} + if (url == null || url.isEmpty()) { + url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") + .getArray("thumbnails").getObject(0).getString("url"); - if (url == null) { - try { - url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") - .getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception ignored) {} - - if (url == null) throw new ParsingException("Could not get playlist thumbnail"); + if (url == null || url.isEmpty()) throw new ParsingException("Could not get playlist thumbnail"); } return fixThumbnailUrl(url); @@ -123,8 +105,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getBannerUrl() { - return ""; // Banner can't be handled by frontend right now. + // Banner can't be handled by frontend right now. // Whoever is willing to implement this should also implement it in the frontend. + return ""; } @Override @@ -199,7 +182,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } private String getNextPageUrlFrom(JsonArray continuations) { - if (continuations == null) { + if (continuations == null || continuations.isEmpty()) { return ""; } @@ -216,7 +199,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object video : videos) { - if (((JsonObject) video).getObject("playlistVideoRenderer") != null) { + if (((JsonObject) video).has("playlistVideoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) { @Override public long getViewCount() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 70c47f6b1..633cfda6f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -69,7 +69,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (showingResultsForRenderer == null) { + if (!showingResultsForRenderer.has("correctedQuery")) { return ""; } return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); @@ -119,21 +119,21 @@ public class YoutubeSearchExtractor extends SearchExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { - if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { + if (((JsonObject) item).has("backgroundPromoRenderer")) { throw new NothingFoundException(getTextFromObject(((JsonObject) item) .getObject("backgroundPromoRenderer").getObject("bodyText"))); - } else if (((JsonObject) item).getObject("videoRenderer") != null) { + } else if (((JsonObject) item).has("videoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); - } else if (((JsonObject) item).getObject("channelRenderer") != null) { + } else if (((JsonObject) item).has("channelRenderer")) { collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer"))); - } else if (((JsonObject) item).getObject("playlistRenderer") != null) { + } else if (((JsonObject) item).has("playlistRenderer")) { collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer"))); } } } private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { - if (continuations == null) { + if (continuations == null || continuations.isEmpty()) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f6b0750c2..6c3c761c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -57,6 +57,7 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; /* * Created by Christian Schabesberger on 06.08.15. @@ -117,18 +118,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getName() throws ParsingException { assertPageFetched(); - String title = null; + String title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); - try { - title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); - } catch (Exception ignored) {} + if (title == null || title.isEmpty()) { + title = playerResponse.getObject("videoDetails").getString("title"); - if (title == null) { - try { - title = playerResponse.getObject("videoDetails").getString("title"); - } catch (Exception ignored) {} - - if (title == null) throw new ParsingException("Could not get name"); + if (title == null || title.isEmpty()) throw new ParsingException("Could not get name"); } return title; @@ -140,35 +135,31 @@ public class YoutubeStreamExtractor extends StreamExtractor { return null; } - try { - JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); - if (micro.getString("uploadDate") != null && !micro.getString("uploadDate").isEmpty()) { - return micro.getString("uploadDate"); - } - if (micro.getString("publishDate") != null && !micro.getString("publishDate").isEmpty()) { - return micro.getString("publishDate"); - } - } catch (Exception ignored) {} + JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); + if (micro.isString("uploadDate") && !micro.getString("uploadDate").isEmpty()) { + return micro.getString("uploadDate"); + } + if (micro.isString("publishDate") && !micro.getString("publishDate").isEmpty()) { + return micro.getString("publishDate"); + } + + if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { + String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); + + try { // Premiered 20 hours ago + TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); + Calendar parsedTime = timeAgoParser.parse(time).date(); + return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); + } catch (Exception ignored) {} + + try { // Premiered Feb 21, 2020 + Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); + return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); + } catch (Exception ignored) {} + } try { - if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { - String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); - - try { // Premiered 20 hours ago - TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); - Calendar parsedTime = timeAgoParser.parse(time).date(); - return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); - } catch (Exception ignored) {} - - try { // Premiered Feb 21, 2020 - Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); - return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); - } catch (Exception ignored) {} - } - } catch (Exception ignored) {} - - try { - // TODO this parses English formatted dates only, we need a better approach to parse the textual date + // TODO: this parses English formatted dates only, we need a better approach to parse the textual date Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); return new SimpleDateFormat("yyyy-MM-dd").format(d); @@ -180,7 +171,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public DateWrapper getUploadDate() throws ParsingException { final String textualUploadDate = getTextualUploadDate(); - if (textualUploadDate == null) { + if (textualUploadDate == null || textualUploadDate.isEmpty()) { return null; } @@ -208,17 +199,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { public Description getDescription() throws ParsingException { assertPageFetched(); // description with more info on links - try { - String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); - return new Description(description, Description.HTML); - } catch (Exception ignored) { } + String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); + if (description != null && !description.isEmpty()) return new Description(description, Description.HTML); // raw non-html description - try { - return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); - } catch (Exception ignored) { - throw new ParsingException("Could not get description"); - } + return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); } @Override @@ -264,19 +249,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getViewCount() throws ParsingException { assertPageFetched(); - String views = null; - - try { - views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") + String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") .getObject("videoViewCountRenderer").getObject("viewCount")); - } catch (Exception ignored) {} - if (views == null) { - try { - views = playerResponse.getObject("videoDetails").getString("viewCount"); - } catch (Exception ignored) {} + if (views == null || views.isEmpty()) { + views = playerResponse.getObject("videoDetails").getString("viewCount"); - if (views == null) throw new ParsingException("Could not get view count"); + if (views == null || views.isEmpty()) throw new ParsingException("Could not get view count"); } if (views.toLowerCase().contains("no views")) return 0; @@ -334,16 +313,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getUploaderUrl() throws ParsingException { assertPageFetched(); - try { + String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer() .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint")); - if (uploaderUrl != null) return uploaderUrl; - } catch (Exception ignored) {} - try { - String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); - if (uploaderId != null) - return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); - } catch (Exception ignored) {} + if (uploaderUrl != null && !uploaderUrl.isEmpty()) return uploaderUrl; + + + String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); + if (uploaderId != null && !uploaderId.isEmpty()) + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); + throw new ParsingException("Could not get uploader url"); } @@ -351,19 +330,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getUploaderName() throws ParsingException { assertPageFetched(); - String uploaderName = null; - - try { - uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") + String uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") .getObject("videoOwnerRenderer").getObject("title")); - } catch (Exception ignored) {} - if (uploaderName == null) { - try { - uploaderName = playerResponse.getObject("videoDetails").getString("author"); - } catch (Exception ignored) {} + if (uploaderName == null || uploaderName.isEmpty()) { + uploaderName = playerResponse.getObject("videoDetails").getString("author"); - if (uploaderName == null) throw new ParsingException("Could not get uploader name"); + if (uploaderName == null || uploaderName.isEmpty()) throw new ParsingException("Could not get uploader name"); } return uploaderName; @@ -392,7 +365,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (videoInfoPage.containsKey("dashmpd")) { dashManifestUrl = videoInfoPage.get("dashmpd"); } else if (playerArgs != null && playerArgs.isString("dashmpd")) { - dashManifestUrl = playerArgs.getString("dashmpd", ""); + dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING); } else { return ""; } @@ -561,9 +534,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object ul : results) { - final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer"); - - if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); + if (((JsonObject) ul).has("compactVideoRenderer")) { + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul).getObject("compactVideoRenderer"), timeAgoParser)); + } } return collector; } catch (Exception e) { @@ -612,7 +585,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String playerUrl; - if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos + if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos initialData = initialAjaxJson.getObject(2).getObject("response"); ageLimit = 18; @@ -631,7 +604,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerResponse = getPlayerResponse(); - final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.EMPTY_OBJECT); + final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus"); final String status = playabilityStatus.getString("status"); // If status exist, and is not "OK", throw a ContentNotAvailableException with the reason. if (status != null && !status.toLowerCase().equals("ok")) { @@ -808,10 +781,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } captions = playerResponse.getObject("captions"); - final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject()); - final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray()); + final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer"); + final JsonArray captionsArray = renderer.getArray("captionTracks"); // todo: use this to apply auto translation to different language from a source language -// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray()); +// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages"); // This check is necessary since there may be cases where subtitles metadata do not contain caption track info // e.g. https://www.youtube.com/watch?v=-Vpwatutnko @@ -876,7 +849,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { JsonObject videoPrimaryInfoRenderer = null; for (Object content : contents) { - if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) { + if (((JsonObject) content).has("videoPrimaryInfoRenderer")) { videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer"); break; } @@ -898,7 +871,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { JsonObject videoSecondaryInfoRenderer = null; for (Object content : contents) { - if (((JsonObject) content).getObject("videoSecondaryInfoRenderer") != null) { + if (((JsonObject) content).has("videoSecondaryInfoRenderer")) { videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer"); break; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 2330130eb..e5d97fdde 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -17,6 +17,7 @@ import java.util.Calendar; import java.util.Date; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; /* * Copyright (C) Christian Schabesberger 2016 @@ -37,7 +38,6 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP */ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - private JsonObject videoInfo; private final TimeAgoParser timeAgoParser; private StreamType cachedStreamType; @@ -59,23 +59,18 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return cachedStreamType; } - try { - JsonArray badges = videoInfo.getArray("badges"); - for (Object badge : badges) { - if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { - return cachedStreamType = StreamType.LIVE_STREAM; - } - } - - } catch (Exception ignored) {} - - try { - final String style = videoInfo.getArray("thumbnailOverlays").getObject(0) - .getObject("thumbnailOverlayTimeStatusRenderer").getString("style"); - if (style.equalsIgnoreCase("LIVE")) { + final JsonArray badges = videoInfo.getArray("badges"); + for (Object badge : badges) { + if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("LIVE NOW")) { return cachedStreamType = StreamType.LIVE_STREAM; } - } catch (Exception ignored) {} + } + + final String style = videoInfo.getArray("thumbnailOverlays").getObject(0) + .getObject("thumbnailOverlayTimeStatusRenderer").getString("style", EMPTY_STRING); + if (style.equalsIgnoreCase("LIVE")) { + return cachedStreamType = StreamType.LIVE_STREAM; + } return cachedStreamType = StreamType.VIDEO_STREAM; } @@ -98,7 +93,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getName() throws ParsingException { String name = getTextFromObject(videoInfo.getObject("title")); - if (name != null && !name.isEmpty()) return name; + if (name == null || !name.isEmpty()) return name; throw new ParsingException("Could not get name"); } @@ -108,23 +103,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return -1; } - String duration = null; + String duration = getTextFromObject(videoInfo.getObject("lengthText")); - try { - duration = getTextFromObject(videoInfo.getObject("lengthText")); - } catch (Exception ignored) {} - - if (duration == null) { - try { - for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { - if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) { - duration = getTextFromObject(((JsonObject) thumbnailOverlay) - .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text")); - } + if (duration == null || duration.isEmpty()) { + for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { + if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) { + duration = getTextFromObject(((JsonObject) thumbnailOverlay) + .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text")); } - } catch (Exception ignored) {} + } - if (duration == null) throw new ParsingException("Could not get duration"); + if (duration == null || duration.isEmpty()) throw new ParsingException("Could not get duration"); } return YoutubeParsingHelper.parseDurationString(duration); @@ -132,23 +121,15 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUploaderName() throws ParsingException { - String name = null; + String name = getTextFromObject(videoInfo.getObject("longBylineText")); - try { - name = getTextFromObject(videoInfo.getObject("longBylineText")); - } catch (Exception ignored) {} + if (name == null || name.isEmpty()) { + name = getTextFromObject(videoInfo.getObject("ownerText")); - if (name == null) { - try { - name = getTextFromObject(videoInfo.getObject("ownerText")); - } catch (Exception ignored) {} + if (name == null || name.isEmpty()) { + name = getTextFromObject(videoInfo.getObject("shortBylineText")); - if (name == null) { - try { - name = getTextFromObject(videoInfo.getObject("shortBylineText")); - } catch (Exception ignored) {} - - if (name == null) throw new ParsingException("Could not get uploader name"); + if (name == null || name.isEmpty()) throw new ParsingException("Could not get uploader name"); } } @@ -157,26 +138,18 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUploaderUrl() throws ParsingException { - String url = null; + String url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); - try { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + if (url == null || url.isEmpty()) { + url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); - } catch (Exception ignored) {} - if (url == null) { - try { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") + if (url == null || url.isEmpty()) { + url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); - } catch (Exception ignored) {} - if (url == null) { - try { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - } catch (Exception ignored) {} - - if (url == null) throw new ParsingException("Could not get uploader url"); + if (url == null || url.isEmpty()) throw new ParsingException("Could not get uploader url"); } } @@ -195,12 +168,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date); } - try { - return getTextFromObject(videoInfo.getObject("publishedTimeText")); - } catch (Exception e) { - // upload date is not always available, e.g. in playlists - return null; - } + final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); + if (publishedTimeText != null && !publishedTimeText.isEmpty()) return publishedTimeText; + + return null; } @Nullable @@ -228,17 +199,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public long getViewCount() throws ParsingException { try { - if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { + if (videoInfo.has("topStandaloneBadge") || isPremium()) { return -1; } - final JsonObject viewCountObject = videoInfo.getObject("viewCountText"); - if (viewCountObject == null) { + if (!videoInfo.has("viewCountText")) { // This object is null when a video has its views hidden. return -1; } - final String viewCount = getTextFromObject(viewCountObject); + final String viewCount = getTextFromObject(videoInfo.getObject("viewCountText")); if (viewCount.toLowerCase().contains("no views")) { return 0; @@ -266,14 +236,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } private boolean isPremium() { - try { - JsonArray badges = videoInfo.getArray("badges"); - for (Object badge : badges) { - if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("Premium")) { - return true; - } + JsonArray badges = videoInfo.getArray("badges"); + for (Object badge : badges) { + if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("Premium")) { + return true; } - } catch (Exception ignored) { } return false; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index 8ed35c194..2c4c74bee 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -59,7 +59,7 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { // trim JSONP part "JP(...)" response = response.substring(3, response.length() - 1); try { - JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray()); + JsonArray collection = JsonParser.array().from(response).getArray(1); for (Object suggestion : collection) { if (!(suggestion instanceof JsonArray)) continue; String suggestionStr = ((JsonArray) suggestion).getString(0); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index e15463b7f..b2c27dbae 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -97,14 +97,11 @@ public class YoutubeTrendingExtractor extends KioskExtractor { JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer") .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content") .getObject("expandedShelfContentsRenderer"); - if (expandedShelfContentsRenderer != null) { - for (Object ul : expandedShelfContentsRenderer.getArray("items")) { - final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); - collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); - } + for (Object ul : expandedShelfContentsRenderer.getArray("items")) { + final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); } } return new InfoItemsPage<>(collector, getNextPageUrl()); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 54bfe95c5..79d01d8a8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -334,7 +334,7 @@ public class YoutubeParsingHelper { } public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { - if (navigationEndpoint.getObject("urlEndpoint") != null) { + if (navigationEndpoint.has("urlEndpoint")) { String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); if (internUrl.startsWith("/redirect?")) { // q parameter can be the first parameter @@ -354,7 +354,7 @@ public class YoutubeParsingHelper { } else if (internUrl.startsWith("http")) { return internUrl; } - } else if (navigationEndpoint.getObject("browseEndpoint") != null) { + } else if (navigationEndpoint.has("browseEndpoint")) { final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl"); final String browseId = browseEndpoint.getString("browseId"); @@ -369,7 +369,7 @@ public class YoutubeParsingHelper { } throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")"); - } else if (navigationEndpoint.getObject("watchEndpoint") != null) { + } else if (navigationEndpoint.has("watchEndpoint")) { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) @@ -377,7 +377,7 @@ public class YoutubeParsingHelper { if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); return url.toString(); - } else if (navigationEndpoint.getObject("watchPlaylistEndpoint") != null) { + } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId"); } @@ -390,7 +390,7 @@ public class YoutubeParsingHelper { StringBuilder textBuilder = new StringBuilder(); for (Object textPart : textObject.getArray("runs")) { String text = ((JsonObject) textPart).getString("text"); - if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) { + if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); if (url != null && !url.isEmpty()) { textBuilder.append("").append(text).append(""); @@ -486,7 +486,7 @@ public class YoutubeParsingHelper { */ public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException { final JsonArray alerts = initialData.getArray("alerts"); - if (alerts != null && !alerts.isEmpty()) { + if (!alerts.isEmpty()) { final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); final String alertText = alertRenderer.getObject("text").getString("simpleText"); final String alertType = alertRenderer.getString("type"); diff --git a/timeago-parser/build.gradle b/timeago-parser/build.gradle index bd99b48bb..ff23db9ea 100644 --- a/timeago-parser/build.gradle +++ b/timeago-parser/build.gradle @@ -1,6 +1,6 @@ dependencies { testImplementation 'junit:junit:4.12' - implementation 'com.grack:nanojson:1.1' + implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' -} \ No newline at end of file +}