kavin/LibreTube
1
0
Fork 0
mirror of https://github.com/libre-tube/LibreTube.git synced 2025-04-29 08:20:32 +05:30
Code Issues Projects Releases Wiki Activity

feat(LocalFeedRepository): throttle feed extraction

Browse Source 
Throttles the local feed extration in-order to avoid rate-limiting.
This is done, similary to NewPipe, by introducting an articificial
delay every 50 fetched feeds of on average 1 second. These values
may have to be changed in the future.

Ref: https://github.com/libre-tube/LibreTube/pull/6941#issuecomment-2585206589
This commit is contained in:
FineFindus 2025-01-11 12:55:52 +01:00 committed by Bnyro
parent 83d257d513
commit 4b049320c1
1 changed files with 21 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
app/src/main/java/com/github/libretube/repo/LocalFeedRepository.kt
View File

@ -13,6 +13,7 @@ import com.github.libretube.extensions.toID
import com.github.libretube.helpers.NewPipeExtractorInstance import com.github.libretube.helpers.NewPipeExtractorInstance
import com.github.libretube.helpers.PreferenceHelper import com.github.libretube.helpers.PreferenceHelper
import com.github.libretube.ui.dialogs.ShareDialog.Companion.YOUTUBE_FRONTEND_URL import com.github.libretube.ui.dialogs.ShareDialog.Companion.YOUTUBE_FRONTEND_URL
import kotlinx.coroutines.delay
import org.schabi.newpipe.extractor.channel.ChannelInfo import org.schabi.newpipe.extractor.channel.ChannelInfo
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs
@ -20,6 +21,7 @@ import org.schabi.newpipe.extractor.feed.FeedInfo
import org.schabi.newpipe.extractor.stream.StreamInfoItem import org.schabi.newpipe.extractor.stream.StreamInfoItem
import java.time.Duration import java.time.Duration
import java.time.Instant import java.time.Instant
import java.util.concurrent.atomic.AtomicInteger
class LocalFeedRepository : FeedRepository { class LocalFeedRepository : FeedRepository {
private val relevantTabs = private val relevantTabs =
@ -60,13 +62,24 @@ class LocalFeedRepository : FeedRepository {
} }
private suspend fun refreshFeed(channelIds: List<String>, minimumDateMillis: Long) { private suspend fun refreshFeed(channelIds: List<String>, minimumDateMillis: Long) {
val extractionCount = AtomicInteger()
for (channelIdChunk in channelIds.chunked(CHUNK_SIZE)) { for (channelIdChunk in channelIds.chunked(CHUNK_SIZE)) {
// add a delay after each BATCH_SIZE amount of visited channels
val count = extractionCount.get();
if (count >= BATCH_SIZE) {
delay(BATCH_DELAY.random())
extractionCount.set(0)
}
val collectedFeedItems = channelIdChunk.parallelMap { channelId -> val collectedFeedItems = channelIdChunk.parallelMap { channelId ->
try { try {
getRelatedStreams(channelId, minimumDateMillis) getRelatedStreams(channelId, minimumDateMillis)
} catch (e: Exception) { } catch (e: Exception) {
Log.e(channelId, e.stackTraceToString()) Log.e(channelId, e.stackTraceToString())
null null
} finally {
extractionCount.incrementAndGet();
} }
}.filterNotNull().flatten().map(StreamItem::toFeedItem) }.filterNotNull().flatten().map(StreamItem::toFeedItem)
@ -114,6 +127,14 @@ class LocalFeedRepository : FeedRepository {
companion object { companion object {
private const val CHUNK_SIZE = 2 private const val CHUNK_SIZE = 2
/**
* Maximum amount of feeds that should be fetched together, before a delay should be applied.
*/
private const val BATCH_SIZE = 50
/**
* Millisecond delay between two consecutive batches to avoid throttling.
*/
private val BATCH_DELAY = (500L..1500L)
private const val MAX_FEED_AGE_DAYS = 30L // 30 days private const val MAX_FEED_AGE_DAYS = 30L // 30 days
} }
} }