diff --git a/app/src/main/java/com/github/libretube/api/StreamsExtractor.kt b/app/src/main/java/com/github/libretube/api/StreamsExtractor.kt index 5320de268..955da92d9 100644 --- a/app/src/main/java/com/github/libretube/api/StreamsExtractor.kt +++ b/app/src/main/java/com/github/libretube/api/StreamsExtractor.kt @@ -13,17 +13,22 @@ import com.github.libretube.api.obj.Subtitle import com.github.libretube.extensions.toID import com.github.libretube.helpers.PlayerHelper import com.github.libretube.ui.dialogs.ShareDialog.Companion.YOUTUBE_FRONTEND_URL +import com.github.libretube.util.deArrow +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.withContext import kotlinx.datetime.toKotlinInstant +import org.schabi.newpipe.extractor.stream.AudioStream import org.schabi.newpipe.extractor.stream.StreamInfo import org.schabi.newpipe.extractor.stream.StreamInfoItem import org.schabi.newpipe.extractor.stream.VideoStream import retrofit2.HttpException import java.io.IOException -fun VideoStream.toPipedStream(): PipedStream = PipedStream( +fun VideoStream.toPipedStream() = PipedStream( url = content, codec = codec, - format = format.toString(), + format = format?.toString(), height = height, width = width, quality = getResolution(), @@ -37,14 +42,34 @@ fun VideoStream.toPipedStream(): PipedStream = PipedStream( contentLength = itagItem?.contentLength ?: 0L ) +fun AudioStream.toPipedStream() = PipedStream( + url = content, + format = format?.toString(), + quality = "$averageBitrate bits", + bitrate = bitrate, + mimeType = format?.mimeType, + initStart = initStart, + initEnd = initEnd, + indexStart = indexStart, + indexEnd = indexEnd, + contentLength = itagItem?.contentLength ?: 0L, + codec = codec, + audioTrackId = audioTrackId, + audioTrackName = audioTrackName, + audioTrackLocale = audioLocale?.toLanguageTag(), + audioTrackType = audioTrackType?.name, + videoOnly = false +) + fun StreamInfoItem.toStreamItem( uploaderAvatarUrl: String? = null -): StreamItem = StreamItem( +) = StreamItem( type = StreamItem.TYPE_STREAM, url = url.toID(), title = name, uploaded = uploadDate?.offsetDateTime()?.toEpochSecond()?.times(1000) ?: -1, - uploadedDate = textualUploadDate ?: uploadDate?.offsetDateTime()?.toLocalDateTime()?.toLocalDate() + uploadedDate = textualUploadDate ?: uploadDate?.offsetDateTime()?.toLocalDateTime() + ?.toLocalDate() ?.toString(), uploaderName = uploaderName, uploaderUrl = uploaderUrl.toID(), @@ -58,13 +83,22 @@ fun StreamInfoItem.toStreamItem( ) object StreamsExtractor { - suspend fun extractStreams(videoId: String): Streams { + suspend fun extractStreams(videoId: String): Streams = withContext(Dispatchers.IO) { if (!PlayerHelper.disablePipedProxy || !PlayerHelper.localStreamExtraction) { - return RetrofitInstance.api.getStreams(videoId) + return@withContext RetrofitInstance.api.getStreams(videoId).deArrow(videoId) } - val resp = StreamInfo.getInfo("${YOUTUBE_FRONTEND_URL}/watch?v=$videoId") - return Streams( + val respAsync = async { + StreamInfo.getInfo("$YOUTUBE_FRONTEND_URL/watch?v=$videoId") + } + val dislikesAsync = async { + if (PlayerHelper.localRYD) runCatching { + RetrofitInstance.externalApi.getVotes(videoId).dislikes + }.getOrElse { -1 } else -1 + } + val (resp, dislikes) = Pair(respAsync.await(), dislikesAsync.await()) + + Streams( title = resp.name, description = resp.description.content, uploader = resp.uploaderName, @@ -75,9 +109,7 @@ object StreamsExtractor { category = resp.category, views = resp.viewCount, likes = resp.likeCount, - dislikes = if (PlayerHelper.localRYD) runCatching { - RetrofitInstance.externalApi.getVotes(videoId).dislikes - }.getOrElse { -1 } else -1, + dislikes = dislikes, license = resp.licence, hls = resp.hlsUrl, dash = resp.dashMpdUrl, @@ -95,7 +127,9 @@ object StreamsExtractor { uploadTimestamp = resp.uploadDate.offsetDateTime().toInstant().toKotlinInstant(), uploaded = resp.uploadDate.offsetDateTime().toEpochSecond() * 1000, thumbnailUrl = resp.thumbnails.maxBy { it.height }.url, - relatedStreams = resp.relatedItems.filterIsInstance().map(StreamInfoItem::toStreamItem), + relatedStreams = resp.relatedItems + .filterIsInstance() + .map { item -> item.toStreamItem() }, chapters = resp.streamSegments.map { ChapterSegment( title = it.title, @@ -103,31 +137,9 @@ object StreamsExtractor { start = it.startTimeSeconds.toLong() ) }, - audioStreams = resp.audioStreams.map { - PipedStream( - url = it.content, - format = it.format?.toString(), - quality = "${it.averageBitrate} bits", - bitrate = it.bitrate, - mimeType = it.format?.mimeType, - initStart = it.initStart, - initEnd = it.initEnd, - indexStart = it.indexStart, - indexEnd = it.indexEnd, - contentLength = it.itagItem?.contentLength ?: 0L, - codec = it.codec, - audioTrackId = it.audioTrackId, - audioTrackName = it.audioTrackName, - audioTrackLocale = it.audioLocale?.toLanguageTag(), - audioTrackType = it.audioTrackType?.name, - videoOnly = false - ) - }, - videoStreams = resp.videoOnlyStreams.map { - it.toPipedStream().copy(videoOnly = true) - } + resp.videoStreams.map { - it.toPipedStream().copy(videoOnly = false) - }, + audioStreams = resp.audioStreams.map { it.toPipedStream() }, + videoStreams = resp.videoOnlyStreams.map { it.toPipedStream().copy(videoOnly = true) } + + resp.videoStreams.map { it.toPipedStream().copy(videoOnly = false) }, previewFrames = resp.previewFrames.map { PreviewFrames( it.urls, @@ -148,7 +160,7 @@ object StreamsExtractor { it.isAutoGenerated ) } - ) + ).deArrow(videoId) } fun getExtractorErrorMessageString(context: Context, exception: Exception): String { @@ -157,6 +169,7 @@ object StreamsExtractor { is HttpException -> exception.response()?.errorBody()?.string()?.runCatching { JsonHelper.json.decodeFromString(this).message }?.getOrNull() ?: context.getString(R.string.server_error) + else -> exception.localizedMessage.orEmpty() } } diff --git a/app/src/main/java/com/github/libretube/util/DeArrowUtil.kt b/app/src/main/java/com/github/libretube/util/DeArrowUtil.kt index b4052aceb..3677d38e6 100644 --- a/app/src/main/java/com/github/libretube/util/DeArrowUtil.kt +++ b/app/src/main/java/com/github/libretube/util/DeArrowUtil.kt @@ -13,20 +13,22 @@ import java.util.TreeSet object DeArrowUtil { private fun extractTitleAndThumbnail(content: DeArrowContent): Pair { - val newTitle = content.titles.firstOrNull { it.votes >= 0 || it.locked }?.title - val newThumbnail = content.thumbnails.firstOrNull { + val title = content.titles.firstOrNull { it.votes >= 0 || it.locked }?.title + val thumbnail = content.thumbnails.firstOrNull { it.thumbnail != null && !it.original && (it.votes >= 0 || it.locked) }?.thumbnail - return newTitle to newThumbnail + + return title to thumbnail } + private suspend fun fetchDeArrowContent(videoIds: List): Map? { val videoIdsString = videoIds.mapTo(TreeSet()) { it }.joinToString(",") return try { RetrofitInstance.api.getDeArrowContent(videoIdsString) } catch (e: Exception) { - Log.e(this::class.java.name, e.toString()) + Log.e(this::class.java.name, "Failed to fetch DeArrow content: ${e.message}") null } } @@ -37,19 +39,19 @@ object DeArrowUtil { suspend fun deArrowStreams(streams: Streams, vidId: String): Streams { if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return streams - val videoIds = listOf(vidId) + streams.relatedStreams.map { it.url!!.toID() } + val videoIds = listOf(vidId) + streams.relatedStreams.mapNotNull { it.url?.toID() } val response = fetchDeArrowContent(videoIds) ?: return streams - for ((videoId, data) in response.entries) { + response[vidId]?.let { data -> val (newTitle, newThumbnail) = extractTitleAndThumbnail(data) + if (newTitle != null) streams.title = newTitle + if (newThumbnail != null) streams.thumbnailUrl = newThumbnail + } - if (videoId == vidId) { - if (newTitle != null) streams.title = newTitle - if (newThumbnail != null) streams.thumbnailUrl = newThumbnail - } else { - val streamItem = streams.relatedStreams - .firstOrNull { it.url?.toID() == videoId } ?: continue - + streams.relatedStreams.forEach { streamItem -> + val videoId = streamItem.url?.toID() ?: return@forEach + response[videoId]?.let { data -> + val (newTitle, newThumbnail) = extractTitleAndThumbnail(data) if (newTitle != null) streamItem.title = newTitle if (newThumbnail != null) streamItem.thumbnail = newThumbnail } @@ -63,54 +65,53 @@ object DeArrowUtil { */ suspend fun deArrowStreamItems(streamItems: List): List { if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return streamItems + if (streamItems.isEmpty()) return streamItems - val response = fetchDeArrowContent(streamItems.map{ it.url!!.toID() }) ?: return streamItems + val videoIds = streamItems.mapNotNull { it.url?.toID() } + val response = fetchDeArrowContent(videoIds) ?: return streamItems - for ((videoId, data) in response.entries) { - val (newTitle, newThumbnail) = extractTitleAndThumbnail(data) - val streamItem = streamItems.firstOrNull { it.url?.toID() == videoId } ?: continue - - if (newTitle != null) streamItem.title = newTitle - if (newThumbnail != null) streamItem.thumbnail = newThumbnail + streamItems.forEach { streamItem -> + val videoId = streamItem.url?.toID() ?: return@forEach + response[videoId]?.let { data -> + val (newTitle, newThumbnail) = extractTitleAndThumbnail(data) + if (newTitle != null) streamItem.title = newTitle + if (newThumbnail != null) streamItem.thumbnail = newThumbnail + } } + return streamItems } /** - * Apply the new titles and thumbnails generated by DeArrow to the stream items + * Apply the new titles and thumbnails generated by DeArrow to the content items */ suspend fun deArrowContentItems(contentItems: List): List { if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return contentItems - val videoIds = contentItems.filter { it.type == "stream" } + val videoIds = contentItems + .filter { it.type == "stream" } .map { it.url.toID() } if (videoIds.isEmpty()) return contentItems val response = fetchDeArrowContent(videoIds) ?: return contentItems - for ((videoId, data) in response.entries) { - val (newTitle, newThumbnail) = extractTitleAndThumbnail(data) - val contentItem = contentItems.firstOrNull { it.url.toID() == videoId } ?: continue - - if (newTitle != null) { contentItem.title = newTitle } - if (newThumbnail != null) { contentItem.thumbnail = newThumbnail } + contentItems.forEach { contentItem -> + val videoId = contentItem.url.toID() + response[videoId]?.let { data -> + val (newTitle, newThumbnail) = extractTitleAndThumbnail(data) + if (newTitle != null) contentItem.title = newTitle + if (newThumbnail != null) contentItem.thumbnail = newThumbnail + } } + return contentItems } } -/** - * If enabled in the preferences, this overrides the video's thumbnail and title with the one - * provided by the DeArrow project - */ @JvmName("deArrowStreamItems") suspend fun List.deArrow() = DeArrowUtil.deArrowStreamItems(this) -/** - * If enabled in the preferences, this overrides the video's thumbnail and title with the one - * provided by the DeArrow project - */ @JvmName("deArrowContentItems") suspend fun List.deArrow() = DeArrowUtil.deArrowContentItems(this) diff --git a/app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt b/app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt index b648f831b..e432cf42e 100644 --- a/app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt +++ b/app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt @@ -1,60 +1,58 @@ package com.github.libretube.util -import java.io.IOException -import java.util.concurrent.TimeUnit -import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.RequestBody.Companion.toRequestBody import org.schabi.newpipe.extractor.downloader.Downloader import org.schabi.newpipe.extractor.downloader.Request import org.schabi.newpipe.extractor.downloader.Response import org.schabi.newpipe.extractor.exceptions.ReCaptchaException +import java.io.IOException class NewPipeDownloaderImpl : Downloader() { - private val client: OkHttpClient = OkHttpClient.Builder() - .readTimeout(READ_TIMEOUT_SECONDS, TimeUnit.SECONDS) + private val client = OkHttpClient.Builder() .build() @Throws(IOException::class, ReCaptchaException::class) override fun execute(request: Request): Response { + val httpMethod = request.httpMethod() val url = request.url() - - val requestBody = request.dataToSend()?.let { - it.toRequestBody(APPLICATION_JSON, 0, it.size) - } + val headers = request.headers() + val dataToSend = request.dataToSend() val requestBuilder = okhttp3.Request.Builder() - .method(request.httpMethod(), requestBody) + .method(httpMethod, dataToSend?.toRequestBody()) .url(url) - .addHeader(USER_AGENT_HEADER_NAME, USER_AGENT) + .addHeader("User-Agent", USER_AGENT) - for ((headerName, headerValueList) in request.headers()) { - requestBuilder.removeHeader(headerName) - for (headerValue in headerValueList) { - requestBuilder.addHeader(headerName, headerValue) + for ((headerKey, headerValues) in headers) { + requestBuilder.removeHeader(headerKey) + for (headerValue in headerValues) { + requestBuilder.addHeader(headerKey, headerValue) } } - val response = client.newCall(requestBuilder.build()).execute() - if (response.code == CAPTCHA_STATUS_CODE) { - response.close() - throw ReCaptchaException("reCaptcha Challenge requested", url) - } - return Response( - response.code, - response.message, - response.headers.toMultimap(), - response.body?.string(), - response.request.url.toString() - ) + return when (response.code) { + 429 -> { + response.close() + throw ReCaptchaException("reCaptcha Challenge requested", url) + } + + else -> { + val responseBodyToReturn = response.body?.string() + Response( + response.code, + response.message, + response.headers.toMultimap(), + responseBodyToReturn, + response.request.url.toString() + ) + } + } } companion object { - private const val USER_AGENT_HEADER_NAME = "User-Agent" - private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0" - private const val CAPTCHA_STATUS_CODE = 429 - private val APPLICATION_JSON = "application/json".toMediaType() - private const val READ_TIMEOUT_SECONDS = 30L + private const val USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0" } } \ No newline at end of file