diff --git a/Dockerfile b/Dockerfile index f0fa377..922e831 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,10 +11,12 @@ FROM eclipse-temurin:17-jre WORKDIR /app/ +COPY hotspot-entrypoint.sh / + COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar COPY VERSION . EXPOSE 8080 -CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -Xshare:on -jar /app/piped.jar +ENTRYPOINT ["/hotspot-entrypoint.sh"] diff --git a/Dockerfile.azul b/Dockerfile.azul index d7d34cc..b376720 100644 --- a/Dockerfile.azul +++ b/Dockerfile.azul @@ -11,10 +11,12 @@ FROM azul/zulu-openjdk:17-jre-headless-latest WORKDIR /app/ +COPY hotspot-entrypoint.sh / + COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar COPY VERSION . EXPOSE 8080 -CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar +ENTRYPOINT ["/hotspot-entrypoint.sh"] diff --git a/Dockerfile.azul.ci b/Dockerfile.azul.ci index d79d85f..6c7c24d 100644 --- a/Dockerfile.azul.ci +++ b/Dockerfile.azul.ci @@ -2,10 +2,12 @@ FROM azul/zulu-openjdk:17-jre-headless-latest WORKDIR /app/ +COPY hotspot-entrypoint.sh / + COPY ./piped.jar /app/piped.jar COPY VERSION . EXPOSE 8080 -CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar +ENTRYPOINT ["/hotspot-entrypoint.sh"] diff --git a/Dockerfile.ci b/Dockerfile.ci index 525d716..94acf71 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -2,10 +2,12 @@ FROM eclipse-temurin:17-jre WORKDIR /app/ +COPY hotspot-entrypoint.sh / + COPY ./piped.jar /app/piped.jar COPY VERSION . EXPOSE 8080 -CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -Xshare:on -jar /app/piped.jar +ENTRYPOINT ["/hotspot-entrypoint.sh"] diff --git a/build.gradle b/build.gradle index 2118c6c..d5cb1e7 100644 --- a/build.gradle +++ b/build.gradle @@ -11,14 +11,14 @@ repositories { } dependencies { - implementation 'org.apache.commons:commons-lang3:3.12.0' + implementation 'org.apache.commons:commons-lang3:3.13.0' implementation 'org.apache.commons:commons-text:1.10.0' implementation 'commons-io:commons-io:2.12.0' implementation 'it.unimi.dsi:fastutil-core:8.5.12' implementation 'commons-codec:commons-codec:1.16.0' implementation 'org.bouncycastle:bcprov-jdk15on:1.70' - implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:bdd1366285614f56a245e3baceb26fb8864bda27' - implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc' + implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:88ceba0da4a48b5f4ffecb3b5b2f36f95ec53afe' + implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2' implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2' implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2' @@ -31,17 +31,17 @@ dependencies { implementation 'io.activej:activej-launchers-http:5.5' implementation 'org.hsqldb:hsqldb:2.7.2' implementation 'org.postgresql:postgresql:42.6.0' - implementation 'org.hibernate:hibernate-core:6.2.5.Final' - implementation 'org.hibernate:hibernate-hikaricp:6.2.5.Final' + implementation 'org.hibernate:hibernate-core:6.2.7.Final' + implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final' implementation 'com.zaxxer:HikariCP:5.0.1' - implementation 'org.springframework.security:spring-security-crypto:6.1.1' + implementation 'org.springframework.security:spring-security-crypto:6.1.2' implementation 'commons-logging:commons-logging:1.2' implementation(platform("com.squareup.okhttp3:okhttp-bom:4.11.0")) implementation 'com.squareup.okhttp3:okhttp' implementation 'com.squareup.okhttp3:okhttp-brotli' implementation 'com.nimbusds:oauth2-oidc-sdk:10.9.1' - implementation 'io.sentry:sentry:6.25.0' - implementation 'rocks.kavin:reqwest4j:1.0.6' + implementation 'io.sentry:sentry:6.28.0' + implementation 'rocks.kavin:reqwest4j:1.0.7' implementation 'io.minio:minio:8.5.4' } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 83d5b95..c33e302 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2-bin.zip +distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2.1-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/hotspot-entrypoint.sh b/hotspot-entrypoint.sh new file mode 100755 index 0000000..eea1684 --- /dev/null +++ b/hotspot-entrypoint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +MAX_MEMORY=${MAX_MEMORY:-1G} + +java -server -Xmx"$MAX_MEMORY" -XX:+UnlockExperimentalVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index a8647c0..067f229 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -23,6 +23,8 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExt import rocks.kavin.reqwest4j.ReqwestUtils; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -36,8 +38,6 @@ public class Main { YoutubeStreamExtractor.forceFetchAndroidClient(true); YoutubeStreamExtractor.forceFetchIosClient(true); - ReqwestUtils.init(Constants.REQWEST_PROXY); - Sentry.init(options -> { options.setDsn(Constants.SENTRY_DSN); options.setRelease(Constants.VERSION); @@ -102,15 +102,39 @@ public class Main { Collections.shuffle(channelIds); - channelIds.stream() - .parallel() - .forEach(id -> Multithreading.runAsyncLimitedPubSub(() -> { + var queue = new ConcurrentLinkedQueue<>(channelIds); + + System.out.println("PubSub: queue size - " + queue.size() + " channels"); + + for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) { + new Thread(() -> { + + Object o = new Object(); + + String channelId; + while ((channelId = queue.poll()) != null) { try { - PubSubHelper.subscribePubSub(id); + CompletableFuture future = PubSubHelper.subscribePubSub(channelId); + + if (future == null) + continue; + + future.whenComplete((resp, throwable) -> { + synchronized (o) { + o.notify(); + } + }); + + synchronized (o) { + o.wait(); + } + } catch (Exception e) { ExceptionHandler.handle(e); } - })); + } + }, "PubSub-" + i).start(); + } } catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index ac983d4..3340cc6 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -21,6 +21,7 @@ import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.ContentCountry; +import rocks.kavin.reqwest4j.ReqwestUtils; import java.io.File; import java.io.FileReader; @@ -134,6 +135,7 @@ public class Constants { PUBSUB_URL = getProperty(prop, "PUBSUB_URL", PUBLIC_URL); PUBSUB_HUB_URL = getProperty(prop, "PUBSUB_HUB_URL", "https://pubsubhubbub.appspot.com/subscribe"); REQWEST_PROXY = getProperty(prop, "REQWEST_PROXY"); + ReqwestUtils.init(REQWEST_PROXY); FRONTEND_URL = getProperty(prop, "FRONTEND_URL", "https://piped.video"); COMPROMISED_PASSWORD_CHECK = Boolean.parseBoolean(getProperty(prop, "COMPROMISED_PASSWORD_CHECK", "true")); DISABLE_REGISTRATION = Boolean.parseBoolean(getProperty(prop, "DISABLE_REGISTRATION", "false")); @@ -195,6 +197,7 @@ public class Constants { .map(JsonNodeFactory.instance::textNode).toList() ); frontendProperties.put("s3Enabled", S3_CLIENT != null); + frontendProperties.put("registrationDisabled", DISABLE_REGISTRATION); // transform hibernate properties for legacy configurations hibernateProperties.replace("hibernate.dialect", diff --git a/src/main/java/me/kavin/piped/server/ServerLauncher.java b/src/main/java/me/kavin/piped/server/ServerLauncher.java index f88e7bc..a62b996 100644 --- a/src/main/java/me/kavin/piped/server/ServerLauncher.java +++ b/src/main/java/me/kavin/piped/server/ServerLauncher.java @@ -42,6 +42,7 @@ import com.nimbusds.oauth2.sdk.id.*; import java.io.ByteArrayInputStream; import java.net.InetSocketAddress; import java.net.URI; +import java.util.List; import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -85,18 +86,21 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher { .map(GET, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> { var topic = request.getQueryParameter("hub.topic"); if (topic != null) - Multithreading.runAsync(() -> { + Multithreading.runAsyncLimited(() -> { String channelId = StringUtils.substringAfter(topic, "channel_id="); PubSubHelper.updatePubSub(channelId); }); - return HttpResponse.ok200().withPlainText(Objects.requireNonNull(request.getQueryParameter("hub.challenge"))); + + var challenge = request.getQueryParameter("hub.challenge"); + return HttpResponse.ok200() + .withPlainText(Objects.requireNonNullElse(challenge, "ok")); })).map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> { try { SyndFeed feed = new SyndFeedInput().build( new InputSource(new ByteArrayInputStream(request.loadBody().getResult().asArray()))); - Multithreading.runAsync(() -> { + Multithreading.runAsyncLimited(() -> { for (var entry : feed.getEntries()) { String url = entry.getLinks().get(0).getHref(); String videoId = StringUtils.substring(url, -11); @@ -104,7 +108,7 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher { if (DatabaseHelper.doesVideoExist(s, videoId)) continue; } - Multithreading.runAsync(() -> { + Multithreading.runAsyncLimited(() -> { try { Sentry.setExtra("videoId", videoId); var extractor = YOUTUBE_SERVICE.getStreamExtractor("https://youtube.com/watch?v=" + videoId); @@ -155,6 +159,23 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher { } catch (Exception e) { return getErrorResponse(e, request.getPath()); } + })).map(GET, "/dearrow", AsyncServlet.ofBlocking(executor, request -> { + try { + var videoIds = getArray(request.getQueryParameter("videoIds")); + + return getJsonResponse( + SponsorBlockUtils.getDeArrowedInfo(videoIds) + .thenApplyAsync(json -> { + try { + return mapper.writeValueAsBytes(json); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }).get(), + "public, max-age=3600"); + } catch (Exception e) { + return getErrorResponse(e, request.getPath()); + } })).map(GET, "/streams/:videoId", AsyncServlet.ofBlocking(executor, request -> { try { return getJsonResponse(StreamHandlers.streamsResponse(request.getPathParameter("videoId")), diff --git a/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java b/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java index 2606477..e086df9 100644 --- a/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java @@ -15,10 +15,10 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.channel.ChannelInfo; -import org.schabi.newpipe.extractor.channel.ChannelTabInfo; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; diff --git a/src/main/java/me/kavin/piped/utils/Alea.java b/src/main/java/me/kavin/piped/utils/Alea.java new file mode 100644 index 0000000..4e5059f --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/Alea.java @@ -0,0 +1,50 @@ +package me.kavin.piped.utils; + +public class Alea { + + private static final double NORM32 = 2.3283064365386963e-10; // 2^-32 + private double s0, s1, s2; + private int c = 1; + + public double next() { + double t = 2091639.0 * s0 + c * NORM32; // 2^-32 + s0 = s1; + s1 = s2; + return s2 = t - (c = (int) t); + } + + public Alea(String seed) { + s0 = mash(" "); + s1 = mash(" "); + s2 = mash(" "); + + s0 -= mash(seed); + + if (s0 < 0) + s0 += 1; + s1 -= mash(seed); + if (s1 < 0) + s1 += 1; + s2 -= mash(seed); + if (s2 < 0) + s2 += 1; + } + + private long n = 0xefc8249dL; + + public double mash(String x) { + double h; + + for (char c : x.toCharArray()) { + n += c; + h = 0.02519603282416938 * n; + n = (long) h; + h -= n; + h *= n; + n = (long) h; + h -= n; + n += h * 0x100000000L; + } + return n * 2.3283064365386963e-10; // 2^-32 + } +} diff --git a/src/main/java/me/kavin/piped/utils/CollectionUtils.java b/src/main/java/me/kavin/piped/utils/CollectionUtils.java index 0c46fdf..bd09132 100644 --- a/src/main/java/me/kavin/piped/utils/CollectionUtils.java +++ b/src/main/java/me/kavin/piped/utils/CollectionUtils.java @@ -4,7 +4,6 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList; import me.kavin.piped.utils.obj.*; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.channel.ChannelTabInfo; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; @@ -16,7 +15,6 @@ import java.util.List; import java.util.Locale; import java.util.Optional; -import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static me.kavin.piped.utils.URLUtils.*; public class CollectionUtils { @@ -66,12 +64,18 @@ public class CollectionUtils { final List relatedStreams = collectRelatedItems(info.getRelatedItems()); + final List metaInfo = new ObjectArrayList<>(); + info.getMetaInfo().forEach(metaInfoItem -> metaInfo.add(new MetaInfo(metaInfoItem.getTitle(), metaInfoItem.getContent().getContent(), + metaInfoItem.getUrls(), metaInfoItem.getUrlTexts() + ))); + return new Streams(info.getName(), info.getDescription().getContent(), info.getTextualUploadDate(), info.getUploaderName(), substringYouTube(info.getUploaderUrl()), rewriteURL(info.getUploaderAvatarUrl()), rewriteURL(info.getThumbnailUrl()), info.getDuration(), info.getViewCount(), info.getLikeCount(), info.getDislikeCount(), info.getUploaderSubscriberCount(), info.isUploaderVerified(), audioStreams, videoStreams, relatedStreams, subtitles, livestream, rewriteVideoURL(info.getHlsUrl()), - rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), chapters, previewFrames); + rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), info.getLicence(), + info.getPrivacy().name().toLowerCase(), info.getTags(), metaInfo, chapters, previewFrames); } public static List collectRelatedItems(List items) { diff --git a/src/main/java/me/kavin/piped/utils/DatabaseHelper.java b/src/main/java/me/kavin/piped/utils/DatabaseHelper.java index a168c7c..9b9ae91 100644 --- a/src/main/java/me/kavin/piped/utils/DatabaseHelper.java +++ b/src/main/java/me/kavin/piped/utils/DatabaseHelper.java @@ -10,9 +10,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.hibernate.SharedSessionContract; import org.hibernate.StatelessSession; -import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfo; -import org.schabi.newpipe.extractor.channel.ChannelTabInfo; +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; diff --git a/src/main/java/me/kavin/piped/utils/Multithreading.java b/src/main/java/me/kavin/piped/utils/Multithreading.java index ccc54fd..2e4efd4 100644 --- a/src/main/java/me/kavin/piped/utils/Multithreading.java +++ b/src/main/java/me/kavin/piped/utils/Multithreading.java @@ -1,8 +1,6 @@ package me.kavin.piped.utils; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; import java.util.function.Supplier; public class Multithreading { @@ -12,11 +10,16 @@ public class Multithreading { .newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 8); private static final ExecutorService esLimitedPubSub = Executors .newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + private static final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors()); public static void runAsync(final Runnable runnable) { es.submit(runnable); } + public static void runAsyncTask(final ForkJoinTask task) { + forkJoinPool.submit(task); + } + public static void runAsyncLimited(final Runnable runnable) { esLimited.submit(runnable); } diff --git a/src/main/java/me/kavin/piped/utils/PubSubHelper.java b/src/main/java/me/kavin/piped/utils/PubSubHelper.java index 23976b7..f74bb5d 100644 --- a/src/main/java/me/kavin/piped/utils/PubSubHelper.java +++ b/src/main/java/me/kavin/piped/utils/PubSubHelper.java @@ -6,16 +6,21 @@ import okhttp3.FormBody; import okio.Buffer; import org.hibernate.StatelessSession; import rocks.kavin.reqwest4j.ReqwestUtils; +import rocks.kavin.reqwest4j.Response; +import javax.annotation.Nullable; import java.io.IOException; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; public class PubSubHelper { - public static void subscribePubSub(String channelId) throws IOException { + + @Nullable + public static CompletableFuture subscribePubSub(String channelId) throws IOException { if (!ChannelHelpers.isValidId(channelId)) - return; + return null; PubSub pubsub = DatabaseHelper.getPubSubFromId(channelId); @@ -44,30 +49,32 @@ public class PubSubHelper { var buffer = new Buffer(); formBuilder.build().writeTo(buffer); - ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of()) - .thenAccept(resp -> { - if (resp.status() != 202) + var completableFuture = ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of()); + + completableFuture + .whenComplete((resp, e) -> { + if (e != null) { + ExceptionHandler.handle((Exception) e); + return; + } + if (resp != null && resp.status() != 202) System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body())); - }) - .exceptionally(e -> { - ExceptionHandler.handle((Exception) e); - return null; }); + + return completableFuture; } + return null; } public static void updatePubSub(String channelId) { - var pubsub = DatabaseHelper.getPubSubFromId(channelId); try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { - s.beginTransaction(); - if (pubsub == null) { - pubsub = new PubSub(channelId, System.currentTimeMillis()); - s.insert(pubsub); - } else { - pubsub.setSubbedAt(System.currentTimeMillis()); - s.update(pubsub); - } - s.getTransaction().commit(); + var tr = s.beginTransaction(); + s.createNativeMutationQuery("INSERT INTO pubsub (id, subbed_at) VALUES (?, ?) " + + "ON CONFLICT (id) DO UPDATE SET subbed_at = excluded.subbed_at") + .setParameter(1, channelId) + .setParameter(2, System.currentTimeMillis()) + .executeUpdate(); + tr.commit(); } } } diff --git a/src/main/java/me/kavin/piped/utils/RequestUtils.java b/src/main/java/me/kavin/piped/utils/RequestUtils.java index d886418..f5c25b0 100644 --- a/src/main/java/me/kavin/piped/utils/RequestUtils.java +++ b/src/main/java/me/kavin/piped/utils/RequestUtils.java @@ -42,13 +42,13 @@ public class RequestUtils { } } - public static CompletableFuture sendGetJson(String url) throws Exception { + public static CompletableFuture sendGetJson(String url) { return ReqwestUtils.fetch(url, "GET", null, Map.of()).thenApply(Response::body).thenApplyAsync(resp -> { try { return mapper.readTree(resp); } catch (Exception e) { throw new RuntimeException("Failed to parse JSON", e); } - }, Multithreading.getCachedExecutor()); + }); } } diff --git a/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java b/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java index bfe580f..4151575 100644 --- a/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java +++ b/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java @@ -1,5 +1,10 @@ package me.kavin.piped.utils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import me.kavin.piped.consts.Constants; import me.kavin.piped.utils.resp.InvalidRequestResponse; import me.kavin.piped.utils.resp.SimpleErrorMessage; @@ -7,7 +12,12 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinTask; +import static me.kavin.piped.consts.Constants.SPONSORBLOCK_SERVERS; import static me.kavin.piped.consts.Constants.mapper; public class SponsorBlockUtils { @@ -46,4 +56,76 @@ public class SponsorBlockUtils { return null; } + public static CompletableFuture getDeArrowedInfo(String[] videoIds) { + ObjectNode objectNode = mapper.createObjectNode(); + + var futures = Arrays.stream(videoIds) + .map(id -> getDeArrowedInfo(id, SPONSORBLOCK_SERVERS.toArray(new String[0])) + .thenAcceptAsync(jsonNode -> objectNode.set(id, jsonNode.orElse(NullNode.getInstance()))) + ) + .toArray(CompletableFuture[]::new); + + return CompletableFuture.allOf(futures) + .thenApplyAsync(v -> objectNode, Multithreading.getCachedExecutor()); + } + + private static CompletableFuture> getDeArrowedInfo(String videoId, String[] servers) { + + String hash = DigestUtils.sha256Hex(videoId); + + CompletableFuture> future = new CompletableFuture<>(); + + var task = ForkJoinTask.adapt(() -> { + fetchDeArrowedCf(future, videoId, hash, servers); + }); + + Multithreading.runAsyncTask(task); + + return future; + + } + + private static final ObjectNode EMPTY_DEARROWED_INFO; + + static { + EMPTY_DEARROWED_INFO = mapper.createObjectNode(); + EMPTY_DEARROWED_INFO.putArray("titles"); + EMPTY_DEARROWED_INFO.putArray("thumbnails"); + EMPTY_DEARROWED_INFO.set("videoDuration", NullNode.getInstance()); + } + + private static void fetchDeArrowedCf(CompletableFuture> future, String videoId, String hash, String[] servers) { + + var completableFuture = RequestUtils.sendGetJson(servers[0] + "/api/branding/" + URLUtils.silentEncode(hash.substring(0, 4))) + .thenApplyAsync(json -> json.has(videoId) ? Optional.of(json.get(videoId)) : Optional.empty()); + + completableFuture.thenAcceptAsync(optional -> optional.ifPresent(jsonNode -> { + ArrayNode nodes = (ArrayNode) jsonNode.get("thumbnails"); + for (JsonNode node : nodes) { + if (!node.get("original").booleanValue()) + ((ObjectNode) node).set("thumbnail", new TextNode(URLUtils.rewriteURL("https://dearrow-thumb.ajay.app/api/v1/getThumbnail?videoID=" + videoId + "&time=" + node.get("timestamp").asText()))); + } + })); + + completableFuture = completableFuture.thenApplyAsync(optional -> { + if (optional.isEmpty()) { + var clone = EMPTY_DEARROWED_INFO.deepCopy(); + clone.put("randomTime", new Alea(videoId).next()); + return Optional.of(clone); + } else + return optional; + }); + + + completableFuture.whenComplete((optional, throwable) -> { + if (throwable == null) + future.complete(optional); + else { + if (servers.length == 1) + future.completeExceptionally(new Exception("All SponsorBlock servers are down")); + else + fetchDeArrowedCf(future, videoId, hash, Arrays.copyOfRange(servers, 1, servers.length)); + } + }); + } } diff --git a/src/main/java/me/kavin/piped/utils/VideoHelpers.java b/src/main/java/me/kavin/piped/utils/VideoHelpers.java index a305aa0..6c80834 100644 --- a/src/main/java/me/kavin/piped/utils/VideoHelpers.java +++ b/src/main/java/me/kavin/piped/utils/VideoHelpers.java @@ -6,7 +6,6 @@ import me.kavin.piped.consts.Constants; import me.kavin.piped.utils.obj.db.Video; import org.apache.commons.lang3.StringUtils; import org.hibernate.StatelessSession; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -49,14 +48,7 @@ public class VideoHelpers { Video video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(), Math.max(infoTime, time), info.getThumbnailUrl(), info.isShortFormContent(), channel); - var tr = s.beginTransaction(); - try { - s.insert(video); - tr.commit(); - } catch (Exception e) { - tr.rollback(); - ExceptionHandler.handle(e); - } + insertVideo(video); return; } } @@ -87,14 +79,7 @@ public class VideoHelpers { Video video = new Video(extractor.getId(), extractor.getName(), extractor.getViewCount(), extractor.getLength(), Math.max(infoTime, time), extractor.getThumbnailUrl(), isShort, channel); - var tr = s.beginTransaction(); - try { - s.insert(video); - tr.commit(); - } catch (Exception e) { - tr.rollback(); - ExceptionHandler.handle(e); - } + insertVideo(video); } } @@ -169,4 +154,30 @@ public class VideoHelpers { return updated > 0; } } + + public static void insertVideo(Video video) { + try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { + var tr = s.beginTransaction(); + try { + s.createNativeMutationQuery( + "INSERT INTO videos (uploader_id,duration,is_short,thumbnail,title,uploaded,views,id) values " + + "(:uploader_id,:duration,:is_short,:thumbnail,:title,:uploaded,:views,:id) ON CONFLICT (id) DO UPDATE SET " + + "duration = excluded.duration, title = excluded.title, views = excluded.views" + ) + .setParameter("uploader_id", video.getChannel().getUploaderId()) + .setParameter("duration", video.getDuration()) + .setParameter("is_short", video.isShort()) + .setParameter("thumbnail", video.getThumbnail()) + .setParameter("title", video.getTitle()) + .setParameter("uploaded", video.getUploaded()) + .setParameter("views", video.getViews()) + .setParameter("id", video.getId()) + .executeUpdate(); + tr.commit(); + } catch (Exception e) { + tr.rollback(); + ExceptionHandler.handle(e); + } + } + } } diff --git a/src/main/java/me/kavin/piped/utils/obj/MetaInfo.java b/src/main/java/me/kavin/piped/utils/obj/MetaInfo.java new file mode 100644 index 0000000..53bf2eb --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/obj/MetaInfo.java @@ -0,0 +1,15 @@ +package me.kavin.piped.utils.obj; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; + +import java.net.URL; +import java.util.List; + +@NoArgsConstructor +@AllArgsConstructor +public class MetaInfo { + public String title, description; + public List urls; + public List urlTexts; +} diff --git a/src/main/java/me/kavin/piped/utils/obj/Streams.java b/src/main/java/me/kavin/piped/utils/obj/Streams.java index 2567e89..478a37b 100644 --- a/src/main/java/me/kavin/piped/utils/obj/Streams.java +++ b/src/main/java/me/kavin/piped/utils/obj/Streams.java @@ -9,7 +9,11 @@ import java.util.List; public class Streams { public String title, description, uploadDate, uploader, uploaderUrl, uploaderAvatar, thumbnailUrl, hls, dash, - lbryId, category; + lbryId, category, license, visibility; + + public List tags; + + public List metaInfo; public boolean uploaderVerified; @@ -33,7 +37,8 @@ public class Streams { String uploaderAvatar, String thumbnailUrl, long duration, long views, long likes, long dislikes, long uploaderSubscriberCount, boolean uploaderVerified, List audioStreams, List videoStreams, List relatedStreams, List subtitles, boolean livestream, String hls, String dash, - String lbryId, String category, List chapters, List previewFrames) { + String lbryId, String category, String license, String visibility, List tags, List metaInfo, + List chapters, List previewFrames) { this.title = title; this.description = description; this.uploadDate = uploadDate; @@ -58,5 +63,9 @@ public class Streams { this.chapters = chapters; this.previewFrames = previewFrames; this.category = category; + this.license = license; + this.tags = tags; + this.metaInfo = metaInfo; + this.visibility = visibility; } } diff --git a/testing/api-test.sh b/testing/api-test.sh index b0aee74..df3b9be 100755 --- a/testing/api-test.sh +++ b/testing/api-test.sh @@ -39,6 +39,9 @@ curl "${CURLOPTS[@]}" $HOST/clips/Ugkx71jS31nwsms_Cc65oi7yXF1mILflhhrO || exit 1 # Streams curl "${CURLOPTS[@]}" $HOST/streams/BtN-goy9VOY || exit 1 +# Streams with meta info +curl "${CURLOPTS[@]}" $HOST/streams/cJ9to6EmElQ || exit 1 + # Comments curl "${CURLOPTS[@]}" $HOST/comments/BtN-goy9VOY || exit 1