refactor: optimize extraction performance & simplify code

+ Update NewPipeExtractor
+ Simplify NewPipeDownloaderImpl.kt implementation
+ Add parallel coroutines and avatar caching to StreamsExtractor.kt
+ Add batching to DeArrow API calls.

feat: optimize extraction performance & simplify code

+ fixes

spacing

app/src/main/java/com/github/libretube/api/StreamsExtractor.kt

@@ -13,17 +13,22 @@ import com.github.libretube.api.obj.Subtitle
 import com.github.libretube.extensions.toID
 import com.github.libretube.helpers.PlayerHelper
 import com.github.libretube.ui.dialogs.ShareDialog.Companion.YOUTUBE_FRONTEND_URL
+import com.github.libretube.util.deArrow
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.withContext
 import kotlinx.datetime.toKotlinInstant
+import org.schabi.newpipe.extractor.stream.AudioStream
 import org.schabi.newpipe.extractor.stream.StreamInfo
 import org.schabi.newpipe.extractor.stream.StreamInfoItem
 import org.schabi.newpipe.extractor.stream.VideoStream
 import retrofit2.HttpException
 import java.io.IOException
 
-fun VideoStream.toPipedStream(): PipedStream = PipedStream(
+fun VideoStream.toPipedStream() = PipedStream(
     url = content,
     codec = codec,
-    format = format.toString(),
+    format = format?.toString(),
     height = height,
     width = width,
     quality = getResolution(),
@@ -37,14 +42,34 @@ fun VideoStream.toPipedStream(): PipedStream = PipedStream(
     contentLength = itagItem?.contentLength ?: 0L
 )
 
+fun AudioStream.toPipedStream() = PipedStream(
+    url = content,
+    format = format?.toString(),
+    quality = "$averageBitrate bits",
+    bitrate = bitrate,
+    mimeType = format?.mimeType,
+    initStart = initStart,
+    initEnd = initEnd,
+    indexStart = indexStart,
+    indexEnd = indexEnd,
+    contentLength = itagItem?.contentLength ?: 0L,
+    codec = codec,
+    audioTrackId = audioTrackId,
+    audioTrackName = audioTrackName,
+    audioTrackLocale = audioLocale?.toLanguageTag(),
+    audioTrackType = audioTrackType?.name,
+    videoOnly = false
+)
+
 fun StreamInfoItem.toStreamItem(
     uploaderAvatarUrl: String? = null
-): StreamItem = StreamItem(
+) = StreamItem(
     type = StreamItem.TYPE_STREAM,
     url = url.toID(),
     title = name,
     uploaded = uploadDate?.offsetDateTime()?.toEpochSecond()?.times(1000) ?: -1,
-    uploadedDate = textualUploadDate ?: uploadDate?.offsetDateTime()?.toLocalDateTime()?.toLocalDate()
+    uploadedDate = textualUploadDate ?: uploadDate?.offsetDateTime()?.toLocalDateTime()
+        ?.toLocalDate()
         ?.toString(),
     uploaderName = uploaderName,
     uploaderUrl = uploaderUrl.toID(),
@@ -58,13 +83,22 @@ fun StreamInfoItem.toStreamItem(
 )
 
 object StreamsExtractor {
-    suspend fun extractStreams(videoId: String): Streams {
+    suspend fun extractStreams(videoId: String): Streams = withContext(Dispatchers.IO) {
         if (!PlayerHelper.disablePipedProxy || !PlayerHelper.localStreamExtraction) {
-            return RetrofitInstance.api.getStreams(videoId)
+            return@withContext RetrofitInstance.api.getStreams(videoId).deArrow(videoId)
         }
 
-        val resp = StreamInfo.getInfo("${YOUTUBE_FRONTEND_URL}/watch?v=$videoId")
+        val respAsync = async {
-        return Streams(
+            StreamInfo.getInfo("$YOUTUBE_FRONTEND_URL/watch?v=$videoId")
+        }
+        val dislikesAsync = async {
+            if (PlayerHelper.localRYD) runCatching {
+                RetrofitInstance.externalApi.getVotes(videoId).dislikes
+            }.getOrElse { -1 } else -1
+        }
+        val (resp, dislikes) = Pair(respAsync.await(), dislikesAsync.await())
+
+        Streams(
             title = resp.name,
             description = resp.description.content,
             uploader = resp.uploaderName,
@@ -75,9 +109,7 @@ object StreamsExtractor {
             category = resp.category,
             views = resp.viewCount,
             likes = resp.likeCount,
-            dislikes = if (PlayerHelper.localRYD) runCatching {
+            dislikes = dislikes,
-                RetrofitInstance.externalApi.getVotes(videoId).dislikes
-            }.getOrElse { -1 } else -1,
             license = resp.licence,
             hls = resp.hlsUrl,
             dash = resp.dashMpdUrl,
@@ -95,7 +127,9 @@ object StreamsExtractor {
             uploadTimestamp = resp.uploadDate.offsetDateTime().toInstant().toKotlinInstant(),
             uploaded = resp.uploadDate.offsetDateTime().toEpochSecond() * 1000,
             thumbnailUrl = resp.thumbnails.maxBy { it.height }.url,
-            relatedStreams = resp.relatedItems.filterIsInstance<StreamInfoItem>().map(StreamInfoItem::toStreamItem),
+            relatedStreams = resp.relatedItems
+                .filterIsInstance<StreamInfoItem>()
+                .map { item -> item.toStreamItem() },
             chapters = resp.streamSegments.map {
                 ChapterSegment(
                     title = it.title,
@@ -103,31 +137,9 @@ object StreamsExtractor {
                     start = it.startTimeSeconds.toLong()
                 )
             },
-            audioStreams = resp.audioStreams.map {
+            audioStreams = resp.audioStreams.map { it.toPipedStream() },
-                PipedStream(
+            videoStreams = resp.videoOnlyStreams.map { it.toPipedStream().copy(videoOnly = true) } +
-                    url = it.content,
+                    resp.videoStreams.map { it.toPipedStream().copy(videoOnly = false) },
-                    format = it.format?.toString(),
-                    quality = "${it.averageBitrate} bits",
-                    bitrate = it.bitrate,
-                    mimeType = it.format?.mimeType,
-                    initStart = it.initStart,
-                    initEnd = it.initEnd,
-                    indexStart = it.indexStart,
-                    indexEnd = it.indexEnd,
-                    contentLength = it.itagItem?.contentLength ?: 0L,
-                    codec = it.codec,
-                    audioTrackId = it.audioTrackId,
-                    audioTrackName = it.audioTrackName,
-                    audioTrackLocale = it.audioLocale?.toLanguageTag(),
-                    audioTrackType = it.audioTrackType?.name,
-                    videoOnly = false
-                )
-            },
-            videoStreams = resp.videoOnlyStreams.map {
-                it.toPipedStream().copy(videoOnly = true)
-            } + resp.videoStreams.map {
-                it.toPipedStream().copy(videoOnly = false)
-            },
             previewFrames = resp.previewFrames.map {
                 PreviewFrames(
                     it.urls,
@@ -148,7 +160,7 @@ object StreamsExtractor {
                     it.isAutoGenerated
                 )
             }
-        )
+        ).deArrow(videoId)
     }
 
     fun getExtractorErrorMessageString(context: Context, exception: Exception): String {
@@ -157,6 +169,7 @@ object StreamsExtractor {
             is HttpException -> exception.response()?.errorBody()?.string()?.runCatching {
                 JsonHelper.json.decodeFromString<Message>(this).message
             }?.getOrNull() ?: context.getString(R.string.server_error)
+
             else -> exception.localizedMessage.orEmpty()
         }
     }

app/src/main/java/com/github/libretube/util/DeArrowUtil.kt

@@ -13,20 +13,22 @@ import java.util.TreeSet
 
 object DeArrowUtil {
     private fun extractTitleAndThumbnail(content: DeArrowContent): Pair<String?, String?> {
-        val newTitle = content.titles.firstOrNull { it.votes >= 0 || it.locked }?.title
+        val title = content.titles.firstOrNull { it.votes >= 0 || it.locked }?.title
-        val newThumbnail = content.thumbnails.firstOrNull {
+        val thumbnail = content.thumbnails.firstOrNull {
             it.thumbnail != null && !it.original && (it.votes >= 0 || it.locked)
         }?.thumbnail
-        return newTitle to newThumbnail
+
+        return title to thumbnail
     }
 
+
     private suspend fun fetchDeArrowContent(videoIds: List<String>): Map<String, DeArrowContent>? {
         val videoIdsString = videoIds.mapTo(TreeSet()) { it }.joinToString(",")
 
         return try {
             RetrofitInstance.api.getDeArrowContent(videoIdsString)
         } catch (e: Exception) {
-            Log.e(this::class.java.name, e.toString())
+            Log.e(this::class.java.name, "Failed to fetch DeArrow content: ${e.message}")
             null
         }
     }
@@ -37,19 +39,19 @@ object DeArrowUtil {
     suspend fun deArrowStreams(streams: Streams, vidId: String): Streams {
         if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return streams
 
-        val videoIds = listOf(vidId) + streams.relatedStreams.map { it.url!!.toID() }
+        val videoIds = listOf(vidId) + streams.relatedStreams.mapNotNull { it.url?.toID() }
         val response = fetchDeArrowContent(videoIds) ?: return streams
 
-        for ((videoId, data) in response.entries) {
+        response[vidId]?.let { data ->
             val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
-
-            if (videoId == vidId) {
             if (newTitle != null) streams.title = newTitle
             if (newThumbnail != null) streams.thumbnailUrl = newThumbnail
-            } else {
+        }
-                val streamItem = streams.relatedStreams
-                    .firstOrNull { it.url?.toID() == videoId } ?: continue
 
+        streams.relatedStreams.forEach { streamItem ->
+            val videoId = streamItem.url?.toID() ?: return@forEach
+            response[videoId]?.let { data ->
+                val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
                 if (newTitle != null) streamItem.title = newTitle
                 if (newThumbnail != null) streamItem.thumbnail = newThumbnail
             }
@@ -63,54 +65,53 @@ object DeArrowUtil {
      */
     suspend fun deArrowStreamItems(streamItems: List<StreamItem>): List<StreamItem> {
         if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return streamItems
+        if (streamItems.isEmpty()) return streamItems
 
-        val response = fetchDeArrowContent(streamItems.map{ it.url!!.toID() }) ?: return streamItems
+        val videoIds = streamItems.mapNotNull { it.url?.toID() }
+        val response = fetchDeArrowContent(videoIds) ?: return streamItems
 
-        for ((videoId, data) in response.entries) {
+        streamItems.forEach { streamItem ->
+            val videoId = streamItem.url?.toID() ?: return@forEach
+            response[videoId]?.let { data ->
                 val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
-            val streamItem = streamItems.firstOrNull { it.url?.toID() == videoId } ?: continue
-
                 if (newTitle != null) streamItem.title = newTitle
                 if (newThumbnail != null) streamItem.thumbnail = newThumbnail
             }
+        }
+
         return streamItems
     }
 
     /**
-     * Apply the new titles and thumbnails generated by DeArrow to the stream items
+     * Apply the new titles and thumbnails generated by DeArrow to the content items
      */
     suspend fun deArrowContentItems(contentItems: List<ContentItem>): List<ContentItem> {
         if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return contentItems
 
-        val videoIds = contentItems.filter { it.type == "stream" }
+        val videoIds = contentItems
+            .filter { it.type == "stream" }
             .map { it.url.toID() }
 
         if (videoIds.isEmpty()) return contentItems
 
         val response = fetchDeArrowContent(videoIds) ?: return contentItems
 
-        for ((videoId, data) in response.entries) {
+        contentItems.forEach { contentItem ->
+            val videoId = contentItem.url.toID()
+            response[videoId]?.let { data ->
                 val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
-            val contentItem = contentItems.firstOrNull { it.url.toID() == videoId } ?: continue
+                if (newTitle != null) contentItem.title = newTitle
-
+                if (newThumbnail != null) contentItem.thumbnail = newThumbnail
-            if (newTitle != null) { contentItem.title = newTitle }
-            if (newThumbnail != null) { contentItem.thumbnail = newThumbnail }
             }
+        }
+
         return contentItems
     }
 }
 
-/**
- * If enabled in the preferences, this overrides the video's thumbnail and title with the one
- * provided by the DeArrow project
- */
 @JvmName("deArrowStreamItems")
 suspend fun List<StreamItem>.deArrow() = DeArrowUtil.deArrowStreamItems(this)
 
-/**
- * If enabled in the preferences, this overrides the video's thumbnail and title with the one
- * provided by the DeArrow project
- */
 @JvmName("deArrowContentItems")
 suspend fun List<ContentItem>.deArrow() = DeArrowUtil.deArrowContentItems(this)
 

app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt

@@ -1,60 +1,58 @@
 package com.github.libretube.util
 
-import java.io.IOException
-import java.util.concurrent.TimeUnit
-import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
 import okhttp3.RequestBody.Companion.toRequestBody
 import org.schabi.newpipe.extractor.downloader.Downloader
 import org.schabi.newpipe.extractor.downloader.Request
 import org.schabi.newpipe.extractor.downloader.Response
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException
+import java.io.IOException
 
 class NewPipeDownloaderImpl : Downloader() {
-    private val client: OkHttpClient = OkHttpClient.Builder()
+    private val client = OkHttpClient.Builder()
-        .readTimeout(READ_TIMEOUT_SECONDS, TimeUnit.SECONDS)
         .build()
 
     @Throws(IOException::class, ReCaptchaException::class)
     override fun execute(request: Request): Response {
+        val httpMethod = request.httpMethod()
         val url = request.url()
-
+        val headers = request.headers()
-        val requestBody = request.dataToSend()?.let {
+        val dataToSend = request.dataToSend()
-            it.toRequestBody(APPLICATION_JSON, 0, it.size)
-        }
 
         val requestBuilder = okhttp3.Request.Builder()
-            .method(request.httpMethod(), requestBody)
+            .method(httpMethod, dataToSend?.toRequestBody())
             .url(url)
-            .addHeader(USER_AGENT_HEADER_NAME, USER_AGENT)
+            .addHeader("User-Agent", USER_AGENT)
 
-        for ((headerName, headerValueList) in request.headers()) {
+        for ((headerKey, headerValues) in headers) {
-            requestBuilder.removeHeader(headerName)
+            requestBuilder.removeHeader(headerKey)
-            for (headerValue in headerValueList) {
+            for (headerValue in headerValues) {
-                requestBuilder.addHeader(headerName, headerValue)
+                requestBuilder.addHeader(headerKey, headerValue)
             }
         }
-
         val response = client.newCall(requestBuilder.build()).execute()
-        if (response.code == CAPTCHA_STATUS_CODE) {
+
+        return when (response.code) {
+            429 -> {
                 response.close()
                 throw ReCaptchaException("reCaptcha Challenge requested", url)
             }
 
-        return Response(
+            else -> {
+                val responseBodyToReturn = response.body?.string()
+                Response(
                     response.code,
                     response.message,
                     response.headers.toMultimap(),
-            response.body?.string(),
+                    responseBodyToReturn,
                     response.request.url.toString()
                 )
             }
+        }
+    }
 
     companion object {
-        private const val USER_AGENT_HEADER_NAME = "User-Agent"
+        private const val USER_AGENT =
-        private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0"
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0"
-        private const val CAPTCHA_STATUS_CODE = 429
-        private val APPLICATION_JSON = "application/json".toMediaType()
-        private const val READ_TIMEOUT_SECONDS = 30L
     }
 }