From 0769fb635828713f12566cc2a4d85fe3fada90a6 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Sun, 14 May 2023 20:43:35 +0100 Subject: [PATCH 1/4] Update NewPipeExtractor --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 401b144..291848a 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation 'it.unimi.dsi:fastutil-core:8.5.12' implementation 'commons-codec:commons-codec:1.15' implementation 'org.bouncycastle:bcprov-jdk15on:1.70' - implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:a0037b718448d744b151412953ed28254acf5389' + implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:251b204a23742c3413426c9b6131d084b190636f' implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc' implementation 'com.fasterxml.jackson.core:jackson-core:2.15.1' implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.1' From f6639511aa9d5296552cd701ed644f24d3fca63d Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Sun, 14 May 2023 21:17:16 +0100 Subject: [PATCH 2/4] Implement support for extracting playlist description. --- .../me/kavin/piped/server/handlers/PlaylistHandlers.java | 2 +- .../piped/server/handlers/auth/AuthPlaylistHandlers.java | 2 +- src/main/java/me/kavin/piped/utils/obj/Playlist.java | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java b/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java index b1ae76c..94d771e 100644 --- a/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java @@ -61,7 +61,7 @@ public class PlaylistHandlers { } final Playlist playlist = new Playlist(info.getName(), rewriteURL(info.getThumbnailUrl()), - rewriteURL(info.getBannerUrl()), nextpage, + info.getDescription().getContent(), rewriteURL(info.getBannerUrl()), nextpage, info.getUploaderName().isEmpty() ? null : info.getUploaderName(), substringYouTube(info.getUploaderUrl()), rewriteURL(info.getUploaderAvatarUrl()), (int) info.getStreamCount(), relatedStreams); diff --git a/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java b/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java index 05b1ff4..a9a8b05 100644 --- a/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java @@ -67,7 +67,7 @@ public class AuthPlaylistHandlers { video.getDuration(), -1, -1, channel.isVerified(), false)); } - final Playlist playlist = new Playlist(pl.getName(), rewriteURL(pl.getThumbnail()), null, null, pl.getOwner().getUsername(), + final Playlist playlist = new Playlist(pl.getName(), rewriteURL(pl.getThumbnail()), pl.getShortDescription(), null, null, pl.getOwner().getUsername(), null, null, videos.size(), relatedStreams); return mapper.writeValueAsBytes(playlist); diff --git a/src/main/java/me/kavin/piped/utils/obj/Playlist.java b/src/main/java/me/kavin/piped/utils/obj/Playlist.java index 4d30103..8675e81 100644 --- a/src/main/java/me/kavin/piped/utils/obj/Playlist.java +++ b/src/main/java/me/kavin/piped/utils/obj/Playlist.java @@ -4,14 +4,15 @@ import java.util.List; public class Playlist { - public String name, thumbnailUrl, bannerUrl, nextpage, uploader, uploaderUrl, uploaderAvatar; + public String name, thumbnailUrl, description, bannerUrl, nextpage, uploader, uploaderUrl, uploaderAvatar; public int videos; public List relatedStreams; - public Playlist(String name, String thumbnailUrl, String bannerUrl, String nextpage, String uploader, + public Playlist(String name, String thumbnailUrl, String description, String bannerUrl, String nextpage, String uploader, String uploaderUrl, String uploaderAvatar, int videos, List relatedStreams) { this.name = name; this.thumbnailUrl = thumbnailUrl; + this.description = description; this.bannerUrl = bannerUrl; this.nextpage = nextpage; this.videos = videos; From 234adf1255d93bdb986b44eaa29ae8936e870a17 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 22 May 2023 20:59:25 +0100 Subject: [PATCH 3/4] Fixes for the new new channel tab handler. --- .../server/handlers/ChannelHandlers.java | 98 +++++++++++-------- .../me/kavin/piped/utils/CollectionUtils.java | 12 +++ .../me/kavin/piped/utils/DatabaseHelper.java | 30 ++++-- 3 files changed, 92 insertions(+), 48 deletions(-) diff --git a/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java b/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java index 541f14d..7feb05a 100644 --- a/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java @@ -11,6 +11,7 @@ import me.kavin.piped.utils.obj.federation.FederatedVideoInfo; import me.kavin.piped.utils.resp.InvalidRequestResponse; import org.apache.commons.lang3.StringUtils; import org.hibernate.StatelessSession; +import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.channel.ChannelInfo; @@ -28,6 +29,7 @@ import java.util.stream.Collectors; import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static me.kavin.piped.consts.Constants.mapper; +import static me.kavin.piped.utils.CollectionUtils.collectPreloadedTabs; import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; import static me.kavin.piped.utils.URLUtils.rewriteURL; @@ -38,24 +40,30 @@ public class ChannelHandlers { final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath); - final List relatedStreams = collectRelatedItems(info.getRelatedItems()); + final ChannelTabInfo tabInfo = ChannelTabInfo.getInfo(YOUTUBE_SERVICE, collectPreloadedTabs(info.getTabs()).get(0)); - Multithreading.runAsync(() -> info.getRelatedItems().forEach(infoItem -> { - if ( - infoItem.getUploadDate() != null && - System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli() - < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION) - ) - try { - MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo( - StringUtils.substring(infoItem.getUrl(), -11), StringUtils.substring(infoItem.getUploaderUrl(), -24), - infoItem.getName(), - infoItem.getDuration(), infoItem.getViewCount()) - ); - } catch (IOException e) { - throw new RuntimeException(e); - } - })); + final List relatedStreams = collectRelatedItems(tabInfo.getRelatedItems()); + + Multithreading.runAsync(() -> tabInfo.getRelatedItems() + .stream().filter(StreamInfoItem.class::isInstance) + .map(StreamInfoItem.class::cast) + .forEach(infoItem -> { + if ( + infoItem.getUploadDate() != null && + System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli() + < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION) + ) + try { + MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo( + StringUtils.substring(infoItem.getUrl(), -11), StringUtils.substring(infoItem.getUploaderUrl(), -24), + infoItem.getName(), + infoItem.getDuration(), infoItem.getViewCount()) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + ); Multithreading.runAsync(() -> { try { @@ -75,8 +83,10 @@ public class ChannelHandlers { ChannelHelpers.updateChannel(s, channel, info.getName(), info.getAvatarUrl(), info.isVerified()); - Set ids = info.getRelatedItems() + Set ids = tabInfo.getRelatedItems() .stream() + .filter(StreamInfoItem.class::isInstance) + .map(StreamInfoItem.class::cast) .filter(item -> { long time = item.getUploadDate() != null ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() @@ -94,32 +104,35 @@ public class ChannelHandlers { List