kavin/Piped-Backend
1
0
Fork 0
mirror of https://github.com/TeamPiped/Piped-Backend.git synced 2025-04-29 16:30:29 +05:30
Code Issues Projects Releases Wiki Activity

Fixes for the new new channel tab handler.

Browse Source
This commit is contained in:
Kavin 2023-05-22 20:59:25 +01:00
parent f6639511aa
commit 234adf1255
No known key found for this signature in database
GPG Key ID: 49451E4482CC5BCD
3 changed files with 92 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java
View File

@ -11,6 +11,7 @@ import me.kavin.piped.utils.obj.federation.FederatedVideoInfo;
import me.kavin.piped.utils.resp.InvalidRequestResponse; import me.kavin.piped.utils.resp.InvalidRequestResponse;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.hibernate.StatelessSession; import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.channel.ChannelInfo; import org.schabi.newpipe.extractor.channel.ChannelInfo;
@ -28,6 +29,7 @@ import java.util.stream.Collectors;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
import static me.kavin.piped.consts.Constants.mapper; import static me.kavin.piped.consts.Constants.mapper;
import static me.kavin.piped.utils.CollectionUtils.collectPreloadedTabs;
import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems;
import static me.kavin.piped.utils.URLUtils.rewriteURL; import static me.kavin.piped.utils.URLUtils.rewriteURL;
@ -38,24 +40,30 @@ public class ChannelHandlers {
final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath); final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath);
final List<ContentItem> relatedStreams = collectRelatedItems(info.getRelatedItems()); final ChannelTabInfo tabInfo = ChannelTabInfo.getInfo(YOUTUBE_SERVICE, collectPreloadedTabs(info.getTabs()).get(0));
Multithreading.runAsync(() -> info.getRelatedItems().forEach(infoItem -> { final List<ContentItem> relatedStreams = collectRelatedItems(tabInfo.getRelatedItems());
if (
infoItem.getUploadDate() != null && Multithreading.runAsync(() -> tabInfo.getRelatedItems()
System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli() .stream().filter(StreamInfoItem.class::isInstance)
< TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION) .map(StreamInfoItem.class::cast)
) .forEach(infoItem -> {
try { if (
MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo( infoItem.getUploadDate() != null &&
StringUtils.substring(infoItem.getUrl(), -11), StringUtils.substring(infoItem.getUploaderUrl(), -24), System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
infoItem.getName(), < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)
infoItem.getDuration(), infoItem.getViewCount()) )
); try {
} catch (IOException e) { MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo(
throw new RuntimeException(e); StringUtils.substring(infoItem.getUrl(), -11), StringUtils.substring(infoItem.getUploaderUrl(), -24),
} infoItem.getName(),
})); infoItem.getDuration(), infoItem.getViewCount())
);
} catch (IOException e) {
throw new RuntimeException(e);
}
})
);
Multithreading.runAsync(() -> { Multithreading.runAsync(() -> {
try { try {
@ -75,8 +83,10 @@ public class ChannelHandlers {
ChannelHelpers.updateChannel(s, channel, info.getName(), info.getAvatarUrl(), info.isVerified()); ChannelHelpers.updateChannel(s, channel, info.getName(), info.getAvatarUrl(), info.isVerified());
Set<String> ids = info.getRelatedItems() Set<String> ids = tabInfo.getRelatedItems()
.stream() .stream()
.filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast)
.filter(item -> { .filter(item -> {
long time = item.getUploadDate() != null long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
@ -94,32 +104,35 @@ public class ChannelHandlers {
List<Video> videos = DatabaseHelper.getVideosFromIds(s, ids); List<Video> videos = DatabaseHelper.getVideosFromIds(s, ids);
for (StreamInfoItem item : info.getRelatedItems()) { tabInfo.getRelatedItems()
long time = item.getUploadDate() != null .stream()
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() .filter(StreamInfoItem.class::isInstance)
: System.currentTimeMillis(); .map(StreamInfoItem.class::cast).forEach(item -> {
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) long time = item.getUploadDate() != null
try { ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl()); : System.currentTimeMillis();
var video = videos.stream() if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
.filter(v -> v.getId().equals(id)) try {
.findFirst(); String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl());
if (video.isPresent()) { var video = videos.stream()
VideoHelpers.updateVideo(id, item); .filter(v -> v.getId().equals(id))
} else { .findFirst();
VideoHelpers.handleNewVideo("https://youtube.com/watch?v=" + id, time, channel); if (video.isPresent()) {
} VideoHelpers.updateVideo(id, item);
} catch (Exception e) { } else {
ExceptionHandler.handle(e); VideoHelpers.handleNewVideo("https://youtube.com/watch?v=" + id, time, channel);
} }
} } catch (Exception e) {
ExceptionHandler.handle(e);
}
});
} }
} }
}); });
String nextpage = null; String nextpage = null;
if (info.hasNextPage()) { if (tabInfo.hasNextPage()) {
Page page = info.getNextPage(); Page page = tabInfo.getNextPage();
nextpage = mapper.writeValueAsString(page); nextpage = mapper.writeValueAsString(page);
} }
@ -152,8 +165,11 @@ public class ChannelHandlers {
if (prevpage == null) if (prevpage == null)
ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter")); ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter"));
ListExtractor.InfoItemsPage<StreamInfoItem> info = ChannelInfo.getMoreItems(YOUTUBE_SERVICE, String channelUrl = "https://youtube.com/channel/" + channelId;
"https://youtube.com/channel/" + channelId, prevpage); String url = channelUrl + "/videos";
ListExtractor.InfoItemsPage<InfoItem> info = ChannelTabInfo.getMoreItems(YOUTUBE_SERVICE,
new ListLinkHandler(url, url, channelId, List.of("videos"), ""), prevpage);
final List<ContentItem> relatedStreams = collectRelatedItems(info.getItems()); final List<ContentItem> relatedStreams = collectRelatedItems(info.getItems());

12
src/main/java/me/kavin/piped/utils/CollectionUtils.java
View File

@ -4,6 +4,9 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import me.kavin.piped.utils.obj.*; import me.kavin.piped.utils.obj.*;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -13,6 +16,7 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Optional; import java.util.Optional;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
import static me.kavin.piped.utils.URLUtils.*; import static me.kavin.piped.utils.URLUtils.*;
public class CollectionUtils { public class CollectionUtils {
@ -122,4 +126,12 @@ public class CollectionUtils {
item.getDescription(), item.getSubscriberCount(), item.getStreamCount(), item.getDescription(), item.getSubscriberCount(), item.getStreamCount(),
item.isVerified()); item.isVerified());
} }
public static List<ReadyChannelTabListLinkHandler> collectPreloadedTabs(List<ListLinkHandler> tabs) {
return tabs
.stream()
.filter(ReadyChannelTabListLinkHandler.class::isInstance)
.map(ReadyChannelTabListLinkHandler.class::cast)
.toList();
}
} }

30
src/main/java/me/kavin/piped/utils/DatabaseHelper.java
View File

@ -9,7 +9,9 @@ import me.kavin.piped.utils.obj.db.*;
import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.exception.ExceptionUtils;
import org.hibernate.SharedSessionContract; import org.hibernate.SharedSessionContract;
import org.hibernate.StatelessSession; import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfo; import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -19,6 +21,8 @@ import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
public class DatabaseHelper { public class DatabaseHelper {
public static User getUserFromSession(String session) { public static User getUserFromSession(String session) {
@ -207,13 +211,25 @@ public class DatabaseHelper {
}); });
Multithreading.runAsync(() -> { Multithreading.runAsync(() -> {
for (StreamInfoItem item : info.getRelatedItems()) { CollectionUtils.collectPreloadedTabs(info.getTabs())
long time = item.getUploadDate() != null .stream()
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() .parallel()
: System.currentTimeMillis(); .map(tab -> {
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) try {
VideoHelpers.handleNewVideo(item.getUrl(), time, channel); return ChannelTabInfo.getInfo(YOUTUBE_SERVICE, tab).getRelatedItems();
} } catch (ExtractionException | IOException e) {
throw new RuntimeException(e);
}
})
.filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast)
.forEach(item -> {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
VideoHelpers.handleNewVideo(item.getUrl(), time, channel);
});
}); });
return channel; return channel;