Merge remote-tracking branch 'origin/master' into oidc

Dockerfile

@@ -11,10 +11,12 @@ FROM eclipse-temurin:17-jre
 
 WORKDIR /app/
 
+COPY hotspot-entrypoint.sh /
+
 COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar
 
 COPY VERSION .
 
 EXPOSE 8080
 
-CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -Xshare:on -jar /app/piped.jar
+ENTRYPOINT ["/hotspot-entrypoint.sh"]

Dockerfile.azul

@@ -11,10 +11,12 @@ FROM azul/zulu-openjdk:17-jre-headless-latest
 
 WORKDIR /app/
 
+COPY hotspot-entrypoint.sh /
+
 COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar
 
 COPY VERSION .
 
 EXPOSE 8080
 
-CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar
+ENTRYPOINT ["/hotspot-entrypoint.sh"]

Dockerfile.azul.ci

@@ -2,10 +2,12 @@ FROM azul/zulu-openjdk:17-jre-headless-latest
 
 WORKDIR /app/
 
+COPY hotspot-entrypoint.sh /
+
 COPY ./piped.jar /app/piped.jar
 
 COPY VERSION .
 
 EXPOSE 8080
 
-CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar
+ENTRYPOINT ["/hotspot-entrypoint.sh"]

Dockerfile.ci

@@ -2,10 +2,12 @@ FROM eclipse-temurin:17-jre
 
 WORKDIR /app/
 
+COPY hotspot-entrypoint.sh /
+
 COPY ./piped.jar /app/piped.jar
 
 COPY VERSION .
 
 EXPOSE 8080
 
-CMD java -server -Xmx1G -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -Xshare:on -jar /app/piped.jar
+ENTRYPOINT ["/hotspot-entrypoint.sh"]

build.gradle

@@ -11,14 +11,14 @@ repositories {
 }
 
 dependencies {
-    implementation 'org.apache.commons:commons-lang3:3.12.0'
+    implementation 'org.apache.commons:commons-lang3:3.13.0'
     implementation 'org.apache.commons:commons-text:1.10.0'
     implementation 'commons-io:commons-io:2.12.0'
     implementation 'it.unimi.dsi:fastutil-core:8.5.12'
     implementation 'commons-codec:commons-codec:1.16.0'
     implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
-    implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:bdd1366285614f56a245e3baceb26fb8864bda27'
+    implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:88ceba0da4a48b5f4ffecb3b5b2f36f95ec53afe'
-    implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc'
+    implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7'
     implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2'
     implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2'
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
@@ -31,17 +31,17 @@ dependencies {
     implementation 'io.activej:activej-launchers-http:5.5'
     implementation 'org.hsqldb:hsqldb:2.7.2'
     implementation 'org.postgresql:postgresql:42.6.0'
-    implementation 'org.hibernate:hibernate-core:6.2.5.Final'
+    implementation 'org.hibernate:hibernate-core:6.2.7.Final'
-    implementation 'org.hibernate:hibernate-hikaricp:6.2.5.Final'
+    implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final'
     implementation 'com.zaxxer:HikariCP:5.0.1'
-    implementation 'org.springframework.security:spring-security-crypto:6.1.1'
+    implementation 'org.springframework.security:spring-security-crypto:6.1.2'
     implementation 'commons-logging:commons-logging:1.2'
     implementation(platform("com.squareup.okhttp3:okhttp-bom:4.11.0"))
     implementation 'com.squareup.okhttp3:okhttp'
     implementation 'com.squareup.okhttp3:okhttp-brotli'
     implementation 'com.nimbusds:oauth2-oidc-sdk:10.9.1'
-    implementation 'io.sentry:sentry:6.25.0'
+    implementation 'io.sentry:sentry:6.28.0'
-    implementation 'rocks.kavin:reqwest4j:1.0.6'
+    implementation 'rocks.kavin:reqwest4j:1.0.7'
     implementation 'io.minio:minio:8.5.4'
 }
 

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2-bin.zip
+distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2.1-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

hotspot-entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+MAX_MEMORY=${MAX_MEMORY:-1G}
+
+java -server -Xmx"$MAX_MEMORY" -XX:+UnlockExperimentalVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar

src/main/java/me/kavin/piped/Main.java

@@ -23,6 +23,8 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExt
 import rocks.kavin.reqwest4j.ReqwestUtils;
 
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -36,8 +38,6 @@ public class Main {
         YoutubeStreamExtractor.forceFetchAndroidClient(true);
         YoutubeStreamExtractor.forceFetchIosClient(true);
 
-        ReqwestUtils.init(Constants.REQWEST_PROXY);
-
         Sentry.init(options -> {
             options.setDsn(Constants.SENTRY_DSN);
             options.setRelease(Constants.VERSION);
@@ -102,15 +102,39 @@ public class Main {
 
                     Collections.shuffle(channelIds);
 
-                    channelIds.stream()
+                    var queue = new ConcurrentLinkedQueue<>(channelIds);
-                            .parallel()
+
-                            .forEach(id -> Multithreading.runAsyncLimitedPubSub(() -> {
+                    System.out.println("PubSub: queue size - " + queue.size() + " channels");
+
+                    for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) {
+                        new Thread(() -> {
+
+                            Object o = new Object();
+
+                            String channelId;
+                            while ((channelId = queue.poll()) != null) {
                                 try {
-                                    PubSubHelper.subscribePubSub(id);
+                                    CompletableFuture<?> future = PubSubHelper.subscribePubSub(channelId);
+
+                                    if (future == null)
+                                        continue;
+
+                                    future.whenComplete((resp, throwable) -> {
+                                        synchronized (o) {
+                                            o.notify();
+                                        }
+                                    });
+
+                                    synchronized (o) {
+                                        o.wait();
+                                    }
+
                                 } catch (Exception e) {
                                     ExceptionHandler.handle(e);
                                 }
-                            }));
+                            }
+                        }, "PubSub-" + i).start();
+                    }
 
                 } catch (Exception e) {
                     e.printStackTrace();

src/main/java/me/kavin/piped/consts/Constants.java

@@ -21,6 +21,7 @@ import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
+import rocks.kavin.reqwest4j.ReqwestUtils;
 
 import java.io.File;
 import java.io.FileReader;
@@ -134,6 +135,7 @@ public class Constants {
             PUBSUB_URL = getProperty(prop, "PUBSUB_URL", PUBLIC_URL);
             PUBSUB_HUB_URL = getProperty(prop, "PUBSUB_HUB_URL", "https://pubsubhubbub.appspot.com/subscribe");
             REQWEST_PROXY = getProperty(prop, "REQWEST_PROXY");
+            ReqwestUtils.init(REQWEST_PROXY);
             FRONTEND_URL = getProperty(prop, "FRONTEND_URL", "https://piped.video");
             COMPROMISED_PASSWORD_CHECK = Boolean.parseBoolean(getProperty(prop, "COMPROMISED_PASSWORD_CHECK", "true"));
             DISABLE_REGISTRATION = Boolean.parseBoolean(getProperty(prop, "DISABLE_REGISTRATION", "false"));
@@ -195,6 +197,7 @@ public class Constants {
                             .map(JsonNodeFactory.instance::textNode).toList()
             );
             frontendProperties.put("s3Enabled", S3_CLIENT != null);
+            frontendProperties.put("registrationDisabled", DISABLE_REGISTRATION);
 
             // transform hibernate properties for legacy configurations
             hibernateProperties.replace("hibernate.dialect",

src/main/java/me/kavin/piped/server/ServerLauncher.java

@@ -42,6 +42,7 @@ import com.nimbusds.oauth2.sdk.id.*;
 import java.io.ByteArrayInputStream;
 import java.net.InetSocketAddress;
 import java.net.URI;
+import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
@@ -85,18 +86,21 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
                 .map(GET, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> {
                     var topic = request.getQueryParameter("hub.topic");
                     if (topic != null)
-                        Multithreading.runAsync(() -> {
+                        Multithreading.runAsyncLimited(() -> {
                             String channelId = StringUtils.substringAfter(topic, "channel_id=");
                             PubSubHelper.updatePubSub(channelId);
                         });
-                    return HttpResponse.ok200().withPlainText(Objects.requireNonNull(request.getQueryParameter("hub.challenge")));
+
+                    var challenge = request.getQueryParameter("hub.challenge");
+                    return HttpResponse.ok200()
+                            .withPlainText(Objects.requireNonNullElse(challenge, "ok"));
                 })).map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> {
                     try {
 
                         SyndFeed feed = new SyndFeedInput().build(
                                 new InputSource(new ByteArrayInputStream(request.loadBody().getResult().asArray())));
 
-                        Multithreading.runAsync(() -> {
+                        Multithreading.runAsyncLimited(() -> {
                             for (var entry : feed.getEntries()) {
                                 String url = entry.getLinks().get(0).getHref();
                                 String videoId = StringUtils.substring(url, -11);
@@ -104,7 +108,7 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
                                     if (DatabaseHelper.doesVideoExist(s, videoId))
                                         continue;
                                 }
-                                Multithreading.runAsync(() -> {
+                                Multithreading.runAsyncLimited(() -> {
                                     try {
                                         Sentry.setExtra("videoId", videoId);
                                         var extractor = YOUTUBE_SERVICE.getStreamExtractor("https://youtube.com/watch?v=" + videoId);
@@ -155,6 +159,23 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
                     } catch (Exception e) {
                         return getErrorResponse(e, request.getPath());
                     }
+                })).map(GET, "/dearrow", AsyncServlet.ofBlocking(executor, request -> {
+                    try {
+                        var videoIds = getArray(request.getQueryParameter("videoIds"));
+
+                        return getJsonResponse(
+                                SponsorBlockUtils.getDeArrowedInfo(videoIds)
+                                        .thenApplyAsync(json -> {
+                                            try {
+                                                return mapper.writeValueAsBytes(json);
+                                            } catch (JsonProcessingException e) {
+                                                throw new RuntimeException(e);
+                                            }
+                                        }).get(),
+                                "public, max-age=3600");
+                    } catch (Exception e) {
+                        return getErrorResponse(e, request.getPath());
+                    }
                 })).map(GET, "/streams/:videoId", AsyncServlet.ofBlocking(executor, request -> {
                     try {
                         return getJsonResponse(StreamHandlers.streamsResponse(request.getPathParameter("videoId")),

src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java

@@ -15,10 +15,10 @@ import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.channel.ChannelInfo;
-import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 

src/main/java/me/kavin/piped/utils/Alea.java

@@ -0,0 +1,50 @@
+package me.kavin.piped.utils;
+
+public class Alea {
+
+    private static final double NORM32 = 2.3283064365386963e-10; // 2^-32
+    private double s0, s1, s2;
+    private int c = 1;
+
+    public double next() {
+        double t = 2091639.0 * s0 + c * NORM32; // 2^-32
+        s0 = s1;
+        s1 = s2;
+        return s2 = t - (c = (int) t);
+    }
+
+    public Alea(String seed) {
+        s0 = mash(" ");
+        s1 = mash(" ");
+        s2 = mash(" ");
+
+        s0 -= mash(seed);
+
+        if (s0 < 0)
+            s0 += 1;
+        s1 -= mash(seed);
+        if (s1 < 0)
+            s1 += 1;
+        s2 -= mash(seed);
+        if (s2 < 0)
+            s2 += 1;
+    }
+
+    private long n = 0xefc8249dL;
+
+    public double mash(String x) {
+        double h;
+
+        for (char c : x.toCharArray()) {
+            n += c;
+            h = 0.02519603282416938 * n;
+            n = (long) h;
+            h -= n;
+            h *= n;
+            n = (long) h;
+            h -= n;
+            n += h * 0x100000000L;
+        }
+        return n * 2.3283064365386963e-10; // 2^-32
+    }
+}

src/main/java/me/kavin/piped/utils/CollectionUtils.java

@@ -4,7 +4,6 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList;
 import me.kavin.piped.utils.obj.*;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
-import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
@@ -16,7 +15,6 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
 
-import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
 import static me.kavin.piped.utils.URLUtils.*;
 
 public class CollectionUtils {
@@ -66,12 +64,18 @@ public class CollectionUtils {
 
         final List<ContentItem> relatedStreams = collectRelatedItems(info.getRelatedItems());
 
+        final List<MetaInfo> metaInfo = new ObjectArrayList<>();
+        info.getMetaInfo().forEach(metaInfoItem -> metaInfo.add(new MetaInfo(metaInfoItem.getTitle(), metaInfoItem.getContent().getContent(),
+                metaInfoItem.getUrls(), metaInfoItem.getUrlTexts()
+        )));
+
         return new Streams(info.getName(), info.getDescription().getContent(),
                 info.getTextualUploadDate(), info.getUploaderName(), substringYouTube(info.getUploaderUrl()),
                 rewriteURL(info.getUploaderAvatarUrl()), rewriteURL(info.getThumbnailUrl()), info.getDuration(),
                 info.getViewCount(), info.getLikeCount(), info.getDislikeCount(), info.getUploaderSubscriberCount(), info.isUploaderVerified(),
                 audioStreams, videoStreams, relatedStreams, subtitles, livestream, rewriteVideoURL(info.getHlsUrl()),
-                rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), chapters, previewFrames);
+                rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), info.getLicence(),
+                info.getPrivacy().name().toLowerCase(), info.getTags(), metaInfo, chapters, previewFrames);
     }
 
     public static List<ContentItem> collectRelatedItems(List<? extends InfoItem> items) {

src/main/java/me/kavin/piped/utils/DatabaseHelper.java

@@ -10,9 +10,8 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.hibernate.SharedSessionContract;
 import org.hibernate.StatelessSession;
-import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.channel.ChannelInfo;
-import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 

src/main/java/me/kavin/piped/utils/Multithreading.java

@@ -1,8 +1,6 @@
 package me.kavin.piped.utils;
 
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 import java.util.function.Supplier;
 
 public class Multithreading {
@@ -12,11 +10,16 @@ public class Multithreading {
             .newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 8);
     private static final ExecutorService esLimitedPubSub = Executors
             .newFixedThreadPool(Runtime.getRuntime().availableProcessors());
+    private static final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors());
 
     public static void runAsync(final Runnable runnable) {
         es.submit(runnable);
     }
 
+    public static void runAsyncTask(final ForkJoinTask<?> task) {
+        forkJoinPool.submit(task);
+    }
+
     public static void runAsyncLimited(final Runnable runnable) {
         esLimited.submit(runnable);
     }

src/main/java/me/kavin/piped/utils/PubSubHelper.java

@@ -6,16 +6,21 @@ import okhttp3.FormBody;
 import okio.Buffer;
 import org.hibernate.StatelessSession;
 import rocks.kavin.reqwest4j.ReqwestUtils;
+import rocks.kavin.reqwest4j.Response;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 
 public class PubSubHelper {
-    public static void subscribePubSub(String channelId) throws IOException {
+
+    @Nullable
+    public static CompletableFuture<Response> subscribePubSub(String channelId) throws IOException {
 
         if (!ChannelHelpers.isValidId(channelId))
-            return;
+            return null;
 
         PubSub pubsub = DatabaseHelper.getPubSubFromId(channelId);
 
@@ -44,30 +49,32 @@ public class PubSubHelper {
             var buffer = new Buffer();
             formBuilder.build().writeTo(buffer);
 
-            ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of())
+            var completableFuture = ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of());
-                    .thenAccept(resp -> {
+
-                        if (resp.status() != 202)
+            completableFuture
+                    .whenComplete((resp, e) -> {
+                        if (e != null) {
+                            ExceptionHandler.handle((Exception) e);
+                            return;
+                        }
+                        if (resp != null && resp.status() != 202)
                             System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
-                    })
-                    .exceptionally(e -> {
-                        ExceptionHandler.handle((Exception) e);
-                        return null;
                     });
+
+            return completableFuture;
         }
+        return null;
     }
 
     public static void updatePubSub(String channelId) {
-        var pubsub = DatabaseHelper.getPubSubFromId(channelId);
         try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) {
-            s.beginTransaction();
+            var tr = s.beginTransaction();
-            if (pubsub == null) {
+            s.createNativeMutationQuery("INSERT INTO pubsub (id, subbed_at) VALUES (?, ?) " +
-                pubsub = new PubSub(channelId, System.currentTimeMillis());
+                            "ON CONFLICT (id) DO UPDATE SET subbed_at = excluded.subbed_at")
-                s.insert(pubsub);
+                    .setParameter(1, channelId)
-            } else {
+                    .setParameter(2, System.currentTimeMillis())
-                pubsub.setSubbedAt(System.currentTimeMillis());
+                    .executeUpdate();
-                s.update(pubsub);
+            tr.commit();
-            }
-            s.getTransaction().commit();
         }
     }
 }

src/main/java/me/kavin/piped/utils/RequestUtils.java

@@ -42,13 +42,13 @@ public class RequestUtils {
         }
     }
 
-    public static CompletableFuture<JsonNode> sendGetJson(String url) throws Exception {
+    public static CompletableFuture<JsonNode> sendGetJson(String url) {
         return ReqwestUtils.fetch(url, "GET", null, Map.of()).thenApply(Response::body).thenApplyAsync(resp -> {
             try {
                 return mapper.readTree(resp);
             } catch (Exception e) {
                 throw new RuntimeException("Failed to parse JSON", e);
             }
-        }, Multithreading.getCachedExecutor());
+        });
     }
 }

src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java

@@ -1,5 +1,10 @@
 package me.kavin.piped.utils;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
 import me.kavin.piped.consts.Constants;
 import me.kavin.piped.utils.resp.InvalidRequestResponse;
 import me.kavin.piped.utils.resp.SimpleErrorMessage;
@@ -7,7 +12,12 @@ import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ForkJoinTask;
 
+import static me.kavin.piped.consts.Constants.SPONSORBLOCK_SERVERS;
 import static me.kavin.piped.consts.Constants.mapper;
 
 public class SponsorBlockUtils {
@@ -46,4 +56,76 @@ public class SponsorBlockUtils {
         return null;
     }
 
+    public static CompletableFuture<ObjectNode> getDeArrowedInfo(String[] videoIds) {
+        ObjectNode objectNode = mapper.createObjectNode();
+
+        var futures = Arrays.stream(videoIds)
+                .map(id -> getDeArrowedInfo(id, SPONSORBLOCK_SERVERS.toArray(new String[0]))
+                        .thenAcceptAsync(jsonNode -> objectNode.set(id, jsonNode.orElse(NullNode.getInstance())))
+                )
+                .toArray(CompletableFuture[]::new);
+
+        return CompletableFuture.allOf(futures)
+                .thenApplyAsync(v -> objectNode, Multithreading.getCachedExecutor());
+    }
+
+    private static CompletableFuture<Optional<JsonNode>> getDeArrowedInfo(String videoId, String[] servers) {
+
+        String hash = DigestUtils.sha256Hex(videoId);
+
+        CompletableFuture<Optional<JsonNode>> future = new CompletableFuture<>();
+
+        var task = ForkJoinTask.adapt(() -> {
+            fetchDeArrowedCf(future, videoId, hash, servers);
+        });
+
+        Multithreading.runAsyncTask(task);
+
+        return future;
+
+    }
+
+    private static final ObjectNode EMPTY_DEARROWED_INFO;
+
+    static {
+        EMPTY_DEARROWED_INFO = mapper.createObjectNode();
+        EMPTY_DEARROWED_INFO.putArray("titles");
+        EMPTY_DEARROWED_INFO.putArray("thumbnails");
+        EMPTY_DEARROWED_INFO.set("videoDuration", NullNode.getInstance());
+    }
+
+    private static void fetchDeArrowedCf(CompletableFuture<Optional<JsonNode>> future, String videoId, String hash, String[] servers) {
+
+        var completableFuture = RequestUtils.sendGetJson(servers[0] + "/api/branding/" + URLUtils.silentEncode(hash.substring(0, 4)))
+                .thenApplyAsync(json -> json.has(videoId) ? Optional.of(json.get(videoId)) : Optional.<JsonNode>empty());
+
+        completableFuture.thenAcceptAsync(optional -> optional.ifPresent(jsonNode -> {
+            ArrayNode nodes = (ArrayNode) jsonNode.get("thumbnails");
+            for (JsonNode node : nodes) {
+                if (!node.get("original").booleanValue())
+                    ((ObjectNode) node).set("thumbnail", new TextNode(URLUtils.rewriteURL("https://dearrow-thumb.ajay.app/api/v1/getThumbnail?videoID=" + videoId + "&time=" + node.get("timestamp").asText())));
+            }
+        }));
+
+        completableFuture = completableFuture.thenApplyAsync(optional -> {
+            if (optional.isEmpty()) {
+                var clone = EMPTY_DEARROWED_INFO.deepCopy();
+                clone.put("randomTime", new Alea(videoId).next());
+                return Optional.of(clone);
+            } else
+                return optional;
+        });
+
+
+        completableFuture.whenComplete((optional, throwable) -> {
+            if (throwable == null)
+                future.complete(optional);
+            else {
+                if (servers.length == 1)
+                    future.completeExceptionally(new Exception("All SponsorBlock servers are down"));
+                else
+                    fetchDeArrowedCf(future, videoId, hash, Arrays.copyOfRange(servers, 1, servers.length));
+            }
+        });
+    }
 }

src/main/java/me/kavin/piped/utils/VideoHelpers.java

@@ -6,7 +6,6 @@ import me.kavin.piped.consts.Constants;
 import me.kavin.piped.utils.obj.db.Video;
 import org.apache.commons.lang3.StringUtils;
 import org.hibernate.StatelessSession;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfo;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@@ -49,14 +48,7 @@ public class VideoHelpers {
                     Video video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(),
                             Math.max(infoTime, time), info.getThumbnailUrl(), info.isShortFormContent(), channel);
 
-                    var tr = s.beginTransaction();
+                    insertVideo(video);
-                    try {
-                        s.insert(video);
-                        tr.commit();
-                    } catch (Exception e) {
-                        tr.rollback();
-                        ExceptionHandler.handle(e);
-                    }
                     return;
                 }
             }
@@ -87,14 +79,7 @@ public class VideoHelpers {
                     Video video = new Video(extractor.getId(), extractor.getName(), extractor.getViewCount(), extractor.getLength(),
                             Math.max(infoTime, time), extractor.getThumbnailUrl(), isShort, channel);
 
-                    var tr = s.beginTransaction();
+                    insertVideo(video);
-                    try {
-                        s.insert(video);
-                        tr.commit();
-                    } catch (Exception e) {
-                        tr.rollback();
-                        ExceptionHandler.handle(e);
-                    }
 
                 }
             }
@@ -169,4 +154,30 @@ public class VideoHelpers {
             return updated > 0;
         }
     }
+
+    public static void insertVideo(Video video) {
+        try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) {
+            var tr = s.beginTransaction();
+            try {
+                s.createNativeMutationQuery(
+                                "INSERT INTO videos (uploader_id,duration,is_short,thumbnail,title,uploaded,views,id) values " +
+                                        "(:uploader_id,:duration,:is_short,:thumbnail,:title,:uploaded,:views,:id) ON CONFLICT (id) DO UPDATE SET " +
+                                        "duration = excluded.duration, title = excluded.title, views = excluded.views"
+                        )
+                        .setParameter("uploader_id", video.getChannel().getUploaderId())
+                        .setParameter("duration", video.getDuration())
+                        .setParameter("is_short", video.isShort())
+                        .setParameter("thumbnail", video.getThumbnail())
+                        .setParameter("title", video.getTitle())
+                        .setParameter("uploaded", video.getUploaded())
+                        .setParameter("views", video.getViews())
+                        .setParameter("id", video.getId())
+                        .executeUpdate();
+                tr.commit();
+            } catch (Exception e) {
+                tr.rollback();
+                ExceptionHandler.handle(e);
+            }
+        }
+    }
 }

src/main/java/me/kavin/piped/utils/obj/MetaInfo.java

@@ -0,0 +1,15 @@
+package me.kavin.piped.utils.obj;
+
+import lombok.AllArgsConstructor;
+import lombok.NoArgsConstructor;
+
+import java.net.URL;
+import java.util.List;
+
+@NoArgsConstructor
+@AllArgsConstructor
+public class MetaInfo {
+    public String title, description;
+    public List<URL> urls;
+    public List<String> urlTexts;
+}

src/main/java/me/kavin/piped/utils/obj/Streams.java

@@ -9,7 +9,11 @@ import java.util.List;
 public class Streams {
 
     public String title, description, uploadDate, uploader, uploaderUrl, uploaderAvatar, thumbnailUrl, hls, dash,
-            lbryId, category;
+            lbryId, category, license, visibility;
+
+    public List<String> tags;
+
+    public List<MetaInfo> metaInfo;
 
     public boolean uploaderVerified;
 
@@ -33,7 +37,8 @@ public class Streams {
                    String uploaderAvatar, String thumbnailUrl, long duration, long views, long likes, long dislikes, long uploaderSubscriberCount,
                    boolean uploaderVerified, List<PipedStream> audioStreams, List<PipedStream> videoStreams,
                    List<ContentItem> relatedStreams, List<Subtitle> subtitles, boolean livestream, String hls, String dash,
-                   String lbryId, String category, List<ChapterSegment> chapters, List<PreviewFrames> previewFrames) {
+                   String lbryId, String category, String license, String visibility, List<String> tags, List<MetaInfo> metaInfo,
+                   List<ChapterSegment> chapters, List<PreviewFrames> previewFrames) {
         this.title = title;
         this.description = description;
         this.uploadDate = uploadDate;
@@ -58,5 +63,9 @@ public class Streams {
         this.chapters = chapters;
         this.previewFrames = previewFrames;
         this.category = category;
+        this.license = license;
+        this.tags = tags;
+        this.metaInfo = metaInfo;
+        this.visibility = visibility;
     }
 }

testing/api-test.sh

@@ -39,6 +39,9 @@ curl "${CURLOPTS[@]}" $HOST/clips/Ugkx71jS31nwsms_Cc65oi7yXF1mILflhhrO || exit 1
 # Streams
 curl "${CURLOPTS[@]}" $HOST/streams/BtN-goy9VOY || exit 1
 
+# Streams with meta info
+curl "${CURLOPTS[@]}" $HOST/streams/cJ9to6EmElQ || exit 1
+
 # Comments
 curl "${CURLOPTS[@]}" $HOST/comments/BtN-goy9VOY || exit 1