feat: support for local feed extraction

This commit is contained in:
Bnyro 2025-01-10 17:32:15 +01:00
parent 2698368aee
commit 579fae287c
32 changed files with 948 additions and 71 deletions

View File

@ -0,0 +1,663 @@
{
"formatVersion": 1,
"database": {
"version": 19,
"identityHash": "74afbaf921a81ccd97447002122d5077",
"entities": [
{
"tableName": "watchHistoryItem",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`videoId` TEXT NOT NULL, `title` TEXT, `uploadDate` TEXT, `uploader` TEXT, `uploaderUrl` TEXT, `uploaderAvatar` TEXT, `thumbnailUrl` TEXT, `duration` INTEGER, `isShort` INTEGER NOT NULL, PRIMARY KEY(`videoId`))",
"fields": [
{
"fieldPath": "videoId",
"columnName": "videoId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "title",
"columnName": "title",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploadDate",
"columnName": "uploadDate",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploader",
"columnName": "uploader",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderUrl",
"columnName": "uploaderUrl",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderAvatar",
"columnName": "uploaderAvatar",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "thumbnailUrl",
"columnName": "thumbnailUrl",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "duration",
"columnName": "duration",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "isShort",
"columnName": "isShort",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"videoId"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "watchPosition",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`videoId` TEXT NOT NULL, `position` INTEGER NOT NULL, PRIMARY KEY(`videoId`))",
"fields": [
{
"fieldPath": "videoId",
"columnName": "videoId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "position",
"columnName": "position",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"videoId"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "searchHistoryItem",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`query` TEXT NOT NULL, PRIMARY KEY(`query`))",
"fields": [
{
"fieldPath": "query",
"columnName": "query",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"query"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "customInstance",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `apiUrl` TEXT NOT NULL, `frontendUrl` TEXT NOT NULL, PRIMARY KEY(`name`))",
"fields": [
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "apiUrl",
"columnName": "apiUrl",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "frontendUrl",
"columnName": "frontendUrl",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"name"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "localSubscription",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`channelId` TEXT NOT NULL, PRIMARY KEY(`channelId`))",
"fields": [
{
"fieldPath": "channelId",
"columnName": "channelId",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"channelId"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "playlistBookmark",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`playlistId` TEXT NOT NULL, `playlistName` TEXT, `thumbnailUrl` TEXT, `uploader` TEXT, `uploaderUrl` TEXT, `uploaderAvatar` TEXT, `videos` INTEGER NOT NULL, PRIMARY KEY(`playlistId`))",
"fields": [
{
"fieldPath": "playlistId",
"columnName": "playlistId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "playlistName",
"columnName": "playlistName",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "thumbnailUrl",
"columnName": "thumbnailUrl",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploader",
"columnName": "uploader",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderUrl",
"columnName": "uploaderUrl",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderAvatar",
"columnName": "uploaderAvatar",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "videos",
"columnName": "videos",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"playlistId"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "LocalPlaylist",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `thumbnailUrl` TEXT NOT NULL, `description` TEXT)",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "thumbnailUrl",
"columnName": "thumbnailUrl",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "description",
"columnName": "description",
"affinity": "TEXT",
"notNull": false
}
],
"primaryKey": {
"autoGenerate": true,
"columnNames": [
"id"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "LocalPlaylistItem",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `playlistId` INTEGER NOT NULL, `videoId` TEXT NOT NULL, `title` TEXT, `uploadDate` TEXT, `uploader` TEXT, `uploaderUrl` TEXT, `uploaderAvatar` TEXT, `thumbnailUrl` TEXT, `duration` INTEGER)",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "playlistId",
"columnName": "playlistId",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "videoId",
"columnName": "videoId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "title",
"columnName": "title",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploadDate",
"columnName": "uploadDate",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploader",
"columnName": "uploader",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderUrl",
"columnName": "uploaderUrl",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderAvatar",
"columnName": "uploaderAvatar",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "thumbnailUrl",
"columnName": "thumbnailUrl",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "duration",
"columnName": "duration",
"affinity": "INTEGER",
"notNull": false
}
],
"primaryKey": {
"autoGenerate": true,
"columnNames": [
"id"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "download",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`videoId` TEXT NOT NULL, `title` TEXT NOT NULL, `description` TEXT NOT NULL, `uploader` TEXT NOT NULL, `duration` INTEGER DEFAULT NULL, `uploadDate` TEXT, `thumbnailPath` TEXT, PRIMARY KEY(`videoId`))",
"fields": [
{
"fieldPath": "videoId",
"columnName": "videoId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "title",
"columnName": "title",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "description",
"columnName": "description",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "uploader",
"columnName": "uploader",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "duration",
"columnName": "duration",
"affinity": "INTEGER",
"notNull": false,
"defaultValue": "NULL"
},
{
"fieldPath": "uploadDate",
"columnName": "uploadDate",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "thumbnailPath",
"columnName": "thumbnailPath",
"affinity": "TEXT",
"notNull": false
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"videoId"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "downloadItem",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `type` TEXT NOT NULL, `videoId` TEXT NOT NULL, `fileName` TEXT NOT NULL, `path` TEXT NOT NULL, `url` TEXT, `format` TEXT, `quality` TEXT, `language` TEXT, `downloadSize` INTEGER NOT NULL, FOREIGN KEY(`videoId`) REFERENCES `download`(`videoId`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "videoId",
"columnName": "videoId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "fileName",
"columnName": "fileName",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "path",
"columnName": "path",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "url",
"columnName": "url",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "format",
"columnName": "format",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "quality",
"columnName": "quality",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "language",
"columnName": "language",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "downloadSize",
"columnName": "downloadSize",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": true,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_downloadItem_path",
"unique": true,
"columnNames": [
"path"
],
"orders": [],
"createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_downloadItem_path` ON `${TABLE_NAME}` (`path`)"
}
],
"foreignKeys": [
{
"table": "download",
"onDelete": "CASCADE",
"onUpdate": "NO ACTION",
"columns": [
"videoId"
],
"referencedColumns": [
"videoId"
]
}
]
},
{
"tableName": "downloadChapters",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `videoId` TEXT NOT NULL, `name` TEXT NOT NULL, `start` INTEGER NOT NULL, `thumbnailUrl` TEXT NOT NULL)",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "videoId",
"columnName": "videoId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "start",
"columnName": "start",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "thumbnailUrl",
"columnName": "thumbnailUrl",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": true,
"columnNames": [
"id"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "subscriptionGroups",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `channels` TEXT NOT NULL, `index` INTEGER NOT NULL, PRIMARY KEY(`name`))",
"fields": [
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "channels",
"columnName": "channels",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "index",
"columnName": "index",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"name"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "feedItem",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`videoId` TEXT NOT NULL, `title` TEXT, `thumbnail` TEXT, `uploaderName` TEXT, `uploaderUrl` TEXT, `uploaderAvatar` TEXT, `duration` INTEGER, `views` INTEGER, `uploaderVerified` INTEGER NOT NULL, `uploaded` INTEGER NOT NULL, `shortDescription` TEXT, `isShort` INTEGER NOT NULL, PRIMARY KEY(`videoId`))",
"fields": [
{
"fieldPath": "videoId",
"columnName": "videoId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "title",
"columnName": "title",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "thumbnail",
"columnName": "thumbnail",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderName",
"columnName": "uploaderName",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderUrl",
"columnName": "uploaderUrl",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "uploaderAvatar",
"columnName": "uploaderAvatar",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "duration",
"columnName": "duration",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "views",
"columnName": "views",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "uploaderVerified",
"columnName": "uploaderVerified",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "uploaded",
"columnName": "uploaded",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "shortDescription",
"columnName": "shortDescription",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "isShort",
"columnName": "isShort",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"videoId"
]
},
"indices": [],
"foreignKeys": []
}
],
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '74afbaf921a81ccd97447002122d5077')"
]
}
}

View File

@ -5,6 +5,7 @@ import androidx.core.app.NotificationChannelCompat
import androidx.core.app.NotificationManagerCompat
import androidx.work.ExistingPeriodicWorkPolicy
import com.github.libretube.helpers.ImageHelper
import com.github.libretube.helpers.NewPipeExtractorInstance
import com.github.libretube.helpers.NotificationHelper
import com.github.libretube.helpers.PreferenceHelper
import com.github.libretube.helpers.ProxyHelper
@ -55,6 +56,8 @@ class LibreTubeApp : Application() {
* Dynamically create App Shortcuts
*/
ShortcutHelper.createShortcuts(this)
NewPipeExtractorInstance.init()
}
/**

View File

@ -11,15 +11,13 @@ import com.github.libretube.api.obj.StreamItem
import com.github.libretube.api.obj.Streams
import com.github.libretube.api.obj.Subtitle
import com.github.libretube.helpers.PlayerHelper
import com.github.libretube.util.NewPipeDownloaderImpl
import com.github.libretube.ui.dialogs.ShareDialog.Companion.YOUTUBE_FRONTEND_URL
import kotlinx.datetime.toKotlinInstant
import org.schabi.newpipe.extractor.NewPipe
import org.schabi.newpipe.extractor.stream.StreamInfo
import org.schabi.newpipe.extractor.stream.StreamInfoItem
import org.schabi.newpipe.extractor.stream.VideoStream
import retrofit2.HttpException
import java.io.IOException
import java.lang.Exception
fun VideoStream.toPipedStream(): PipedStream = PipedStream(
url = content,
@ -39,26 +37,18 @@ fun VideoStream.toPipedStream(): PipedStream = PipedStream(
)
object StreamsExtractor {
// val npe by lazy {
// NewPipe.getService(ServiceList.YouTube.serviceId)
// }
init {
NewPipe.init(NewPipeDownloaderImpl())
}
suspend fun extractStreams(videoId: String): Streams {
if (!PlayerHelper.disablePipedProxy || !PlayerHelper.localStreamExtraction) {
return RetrofitInstance.api.getStreams(videoId)
}
val resp = StreamInfo.getInfo("https://www.youtube.com/watch?v=$videoId")
val resp = StreamInfo.getInfo("${YOUTUBE_FRONTEND_URL}/watch?v=$videoId")
return Streams(
title = resp.name,
description = resp.description.content,
uploader = resp.uploaderName,
uploaderAvatar = resp.uploaderAvatars.maxBy { it.height }.url,
uploaderUrl = resp.uploaderUrl.replace("https://www.youtube.com", ""),
uploaderUrl = resp.uploaderUrl.replace(YOUTUBE_FRONTEND_URL, ""),
uploaderVerified = resp.isUploaderVerified,
uploaderSubscriberCount = resp.uploaderSubscriberCount,
category = resp.category,
@ -86,12 +76,12 @@ object StreamsExtractor {
thumbnailUrl = resp.thumbnails.maxBy { it.height }.url,
relatedStreams = resp.relatedItems.filterIsInstance<StreamInfoItem>().map {
StreamItem(
it.url.replace("https://www.youtube.com", ""),
it.url.replace(YOUTUBE_FRONTEND_URL, ""),
StreamItem.TYPE_STREAM,
it.name,
it.thumbnails.maxBy { image -> image.height }.url,
it.uploaderName,
it.uploaderUrl.replace("https://www.youtube.com", ""),
it.uploaderUrl.replace(YOUTUBE_FRONTEND_URL, ""),
it.uploaderAvatars.maxBy { image -> image.height }.url,
it.textualUploadDate,
it.duration,

View File

@ -6,6 +6,7 @@ import com.github.libretube.constants.PreferenceKeys
import com.github.libretube.helpers.PreferenceHelper
import com.github.libretube.repo.AccountSubscriptionsRepository
import com.github.libretube.repo.FeedRepository
import com.github.libretube.repo.LocalFeedRepository
import com.github.libretube.repo.LocalSubscriptionsRepository
import com.github.libretube.repo.PipedAccountFeedRepository
import com.github.libretube.repo.PipedNoAccountFeedRepository
@ -26,6 +27,7 @@ object SubscriptionHelper {
else -> LocalSubscriptionsRepository()
}
private val feedRepository: FeedRepository get() = when {
PreferenceHelper.getBoolean(PreferenceKeys.LOCAL_FEED_EXTRACTION, false) -> LocalFeedRepository()
token.isNotEmpty() -> PipedAccountFeedRepository()
else -> PipedNoAccountFeedRepository()
}
@ -35,7 +37,8 @@ object SubscriptionHelper {
suspend fun isSubscribed(channelId: String) = subscriptionsRepository.isSubscribed(channelId)
suspend fun importSubscriptions(newChannels: List<String>) = subscriptionsRepository.importSubscriptions(newChannels)
suspend fun getSubscriptions() = subscriptionsRepository.getSubscriptions()
suspend fun getFeed() = feedRepository.getFeed(false)
suspend fun getSubscriptionChannelIds() = subscriptionsRepository.getSubscriptionChannelIds()
suspend fun getFeed(forceRefresh: Boolean) = feedRepository.getFeed(forceRefresh)
fun handleUnsubscribe(
context: Context,

View File

@ -2,6 +2,7 @@ package com.github.libretube.api.obj
import android.os.Parcelable
import com.github.libretube.db.obj.LocalPlaylistItem
import com.github.libretube.db.obj.SubscriptionsFeedItem
import com.github.libretube.extensions.toID
import kotlinx.parcelize.Parcelize
import kotlinx.serialization.Serializable
@ -40,6 +41,21 @@ data class StreamItem(
)
}
fun toFeedItem() = SubscriptionsFeedItem(
videoId = url!!.toID(),
title = title,
thumbnail = thumbnail,
uploaderName = uploaderName,
uploaded = uploaded,
uploaderAvatar = uploaderAvatar,
uploaderUrl = uploaderUrl,
duration = duration,
uploaderVerified = uploaderVerified ?: false,
shortDescription = shortDescription,
views = views,
isShort = isShort
)
companion object {
const val TYPE_STREAM = "stream"
const val TYPE_CHANNEL = "channel"

View File

@ -3,12 +3,11 @@ package com.github.libretube.api.obj
import android.os.Parcelable
import com.github.libretube.db.obj.DownloadItem
import com.github.libretube.enums.FileType
import com.github.libretube.extensions.toLocalDate
import com.github.libretube.helpers.ProxyHelper
import com.github.libretube.json.SafeInstantSerializer
import com.github.libretube.parcelable.DownloadData
import kotlinx.datetime.Instant
import kotlinx.datetime.TimeZone
import kotlinx.datetime.toLocalDateTime
import kotlinx.parcelize.IgnoredOnParcel
import kotlinx.parcelize.Parcelize
import kotlinx.serialization.SerialName
@ -100,8 +99,7 @@ data class Streams(
uploaderName = uploader,
uploaderUrl = uploaderUrl,
uploaderAvatar = uploaderAvatar,
uploadedDate = uploadTimestamp?.toLocalDateTime(TimeZone.currentSystemDefault())?.date
?.toString(),
uploadedDate = uploadTimestamp?.toLocalDate()?.toString(),
uploaded = uploaded ?: uploadTimestamp?.toEpochMilliseconds() ?: 0,
duration = duration,
views = views,
@ -111,6 +109,6 @@ data class Streams(
}
companion object {
const val categoryMusic = "Music"
const val CATEGORY_MUSIC = "Music"
}
}

View File

@ -113,6 +113,8 @@ object PreferenceKeys {
const val HIDE_WATCHED_FROM_FEED = "hide_watched_from_feed"
const val SELECTED_FEED_FILTERS = "filter_feed"
const val FEED_SORT_ORDER = "sort_oder_feed"
const val LOCAL_FEED_EXTRACTION = "local_feed_extraction"
const val LAST_FEED_REFRESH_TIMESTAMP_MILLIS = "last_feed_refresh_timestamp_millis"
// Advanced
const val AUTOMATIC_UPDATE_CHECKS = "automatic_update_checks"

View File

@ -11,6 +11,7 @@ import com.github.libretube.db.dao.LocalSubscriptionDao
import com.github.libretube.db.dao.PlaylistBookmarkDao
import com.github.libretube.db.dao.SearchHistoryDao
import com.github.libretube.db.dao.SubscriptionGroupsDao
import com.github.libretube.db.dao.SubscriptionsFeedDao
import com.github.libretube.db.dao.WatchHistoryDao
import com.github.libretube.db.dao.WatchPositionDao
import com.github.libretube.db.obj.CustomInstance
@ -23,6 +24,7 @@ import com.github.libretube.db.obj.LocalSubscription
import com.github.libretube.db.obj.PlaylistBookmark
import com.github.libretube.db.obj.SearchHistoryItem
import com.github.libretube.db.obj.SubscriptionGroup
import com.github.libretube.db.obj.SubscriptionsFeedItem
import com.github.libretube.db.obj.WatchHistoryItem
import com.github.libretube.db.obj.WatchPosition
@ -39,15 +41,17 @@ import com.github.libretube.db.obj.WatchPosition
Download::class,
DownloadItem::class,
DownloadChapter::class,
SubscriptionGroup::class
SubscriptionGroup::class,
SubscriptionsFeedItem::class
],
version = 18,
version = 19,
autoMigrations = [
AutoMigration(from = 7, to = 8),
AutoMigration(from = 8, to = 9),
AutoMigration(from = 9, to = 10),
AutoMigration(from = 10, to = 11),
AutoMigration(from = 16, to = 17)
AutoMigration(from = 16, to = 17),
AutoMigration(from = 18, to = 19)
]
)
@TypeConverters(Converters::class)
@ -96,4 +100,6 @@ abstract class AppDatabase : RoomDatabase() {
* Subscription groups
*/
abstract fun subscriptionGroupsDao(): SubscriptionGroupsDao
abstract fun feedDao(): SubscriptionsFeedDao
}

View File

@ -3,7 +3,6 @@ package com.github.libretube.db
import androidx.room.TypeConverter
import com.github.libretube.api.JsonHelper
import kotlinx.datetime.LocalDate
import kotlinx.datetime.toLocalDate
import kotlinx.serialization.encodeToString
import java.nio.file.Path
import kotlin.io.path.Path
@ -13,7 +12,7 @@ object Converters {
fun localDateToString(localDate: LocalDate?) = localDate?.toString()
@TypeConverter
fun stringToLocalDate(string: String?) = string?.toLocalDate()
fun stringToLocalDate(string: String?) = string?.let { LocalDate.parse(it) }
@TypeConverter
fun pathToString(path: Path?) = path?.toString()

View File

@ -8,13 +8,11 @@ import com.github.libretube.db.obj.SearchHistoryItem
import com.github.libretube.db.obj.WatchHistoryItem
import com.github.libretube.enums.ContentFilter
import com.github.libretube.extensions.toID
import com.github.libretube.extensions.toLocalDate
import com.github.libretube.helpers.PreferenceHelper
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import kotlinx.datetime.Instant
import kotlinx.datetime.TimeZone
import kotlinx.datetime.toLocalDateTime
object DatabaseHelper {
private const val MAX_SEARCH_HISTORY_SIZE = 20
@ -28,8 +26,7 @@ object DatabaseHelper {
val watchHistoryItem = WatchHistoryItem(
videoId,
stream.title,
Instant.fromEpochMilliseconds(stream.uploaded)
.toLocalDateTime(TimeZone.currentSystemDefault()).date,
stream.uploaded.toLocalDate(),
stream.uploaderName,
stream.uploaderUrl?.toID(),
stream.uploaderAvatar,

View File

@ -66,7 +66,6 @@ object DatabaseHolder {
MIGRATION_15_16,
MIGRATION_17_18
)
.fallbackToDestructiveMigration()
.build()
}
}

View File

@ -0,0 +1,22 @@
package com.github.libretube.db.dao
import androidx.room.Dao
import androidx.room.Insert
import androidx.room.OnConflictStrategy
import androidx.room.Query
import com.github.libretube.db.obj.SubscriptionsFeedItem
@Dao
interface SubscriptionsFeedDao {
@Query("SELECT * FROM feedItem ORDER BY uploaded DESC")
suspend fun getAll(): List<SubscriptionsFeedItem>
@Insert(onConflict = OnConflictStrategy.REPLACE)
suspend fun insertAll(feedItems: List<SubscriptionsFeedItem>)
@Query("DELETE FROM feedItem WHERE uploaded < :olderThan")
suspend fun cleanUpOlderThan(olderThan: Long)
@Query("DELETE FROM feedItem")
suspend fun deleteAll()
}

View File

@ -0,0 +1,40 @@
package com.github.libretube.db.obj
import androidx.room.Entity
import androidx.room.PrimaryKey
import com.github.libretube.api.obj.StreamItem
import com.github.libretube.extensions.toLocalDate
@Entity(tableName = "feedItem")
data class SubscriptionsFeedItem(
@PrimaryKey
val videoId: String,
val title: String? = null,
val thumbnail: String? = null,
val uploaderName: String? = null,
val uploaderUrl: String? = null,
val uploaderAvatar: String? = null,
val duration: Long? = null,
val views: Long? = null,
val uploaderVerified: Boolean,
val uploaded: Long = 0,
val shortDescription: String? = null,
val isShort: Boolean = false
) {
fun toStreamItem() = StreamItem(
url = videoId,
type = StreamItem.TYPE_STREAM,
title = title,
thumbnail = thumbnail,
uploaderName = uploaderName,
uploaded = uploaded,
uploadedDate = uploaded.toLocalDate().toString(),
uploaderAvatar = uploaderAvatar,
uploaderUrl = uploaderUrl,
duration = duration,
uploaderVerified = uploaderVerified,
shortDescription = shortDescription,
views = views,
isShort = isShort
)
}

View File

@ -30,6 +30,7 @@ data class WatchHistoryItem(
thumbnail = thumbnailUrl,
uploaderName = uploader,
uploaded = uploadDate?.toMillis() ?: 0,
uploadedDate = uploadDate?.toString(),
uploaderAvatar = uploaderAvatar,
uploaderUrl = uploaderUrl,
duration = duration,

View File

@ -1,7 +1,19 @@
package com.github.libretube.extensions
import kotlinx.datetime.Instant
import kotlinx.datetime.LocalDate
import kotlinx.datetime.TimeZone
import kotlinx.datetime.atStartOfDayIn
import kotlinx.datetime.toLocalDateTime
fun LocalDate.toMillis() = this.atStartOfDayIn(TimeZone.UTC).toEpochMilliseconds()
fun Long.toLocalDateTime() =
Instant.fromEpochMilliseconds(this).toLocalDateTime()
fun Long.toLocalDate() =
Instant.fromEpochMilliseconds(this).toLocalDate()
fun Instant.toLocalDateTime() = this.toLocalDateTime(TimeZone.currentSystemDefault())
fun Instant.toLocalDate() = this.toLocalDateTime().date

View File

@ -0,0 +1,16 @@
package com.github.libretube.helpers
import com.github.libretube.util.NewPipeDownloaderImpl
import org.schabi.newpipe.extractor.NewPipe
import org.schabi.newpipe.extractor.ServiceList
import org.schabi.newpipe.extractor.StreamingService
object NewPipeExtractorInstance {
val extractor: StreamingService by lazy {
NewPipe.getService(ServiceList.YouTube.serviceId)
}
fun init() {
NewPipe.init(NewPipeDownloaderImpl())
}
}

View File

@ -3,6 +3,7 @@ package com.github.libretube.json
import android.util.Log
import com.github.libretube.extensions.TAG
import kotlinx.datetime.Instant
import kotlinx.datetime.LocalDate
import kotlinx.datetime.TimeZone
import kotlinx.datetime.atStartOfDayIn
import kotlinx.datetime.toInstant
@ -19,10 +20,10 @@ object SafeInstantSerializer : KSerializer<Instant> {
override fun deserialize(decoder: Decoder): Instant {
val string = decoder.decodeString()
return try {
string.toInstant()
Instant.parse(string)
} catch (e: IllegalArgumentException) {
Log.e(TAG(), "Error parsing date '$string'", e)
string.toLocalDate().atStartOfDayIn(TimeZone.currentSystemDefault())
LocalDate.parse(string).atStartOfDayIn(TimeZone.currentSystemDefault())
}
}

View File

@ -3,6 +3,7 @@ package com.github.libretube.repo
import com.github.libretube.api.RetrofitInstance
import com.github.libretube.api.obj.Subscribe
import com.github.libretube.api.obj.Subscription
import com.github.libretube.extensions.toID
import com.github.libretube.helpers.PreferenceHelper
class AccountSubscriptionsRepository: SubscriptionsRepository {
@ -33,4 +34,8 @@ class AccountSubscriptionsRepository: SubscriptionsRepository {
override suspend fun getSubscriptions(): List<Subscription> {
return RetrofitInstance.authApi.subscriptions(token)
}
override suspend fun getSubscriptionChannelIds(): List<String> {
return getSubscriptions().map { it.url.toID() }
}
}

View File

@ -0,0 +1,103 @@
package com.github.libretube.repo
import android.util.Log
import com.github.libretube.api.SubscriptionHelper
import com.github.libretube.api.obj.StreamItem
import com.github.libretube.constants.PreferenceKeys
import com.github.libretube.db.DatabaseHolder
import com.github.libretube.db.obj.SubscriptionsFeedItem
import com.github.libretube.extensions.parallelMap
import com.github.libretube.helpers.NewPipeExtractorInstance
import com.github.libretube.helpers.PreferenceHelper
import com.github.libretube.ui.dialogs.ShareDialog.Companion.YOUTUBE_FRONTEND_URL
import org.schabi.newpipe.extractor.channel.ChannelInfo
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs
import org.schabi.newpipe.extractor.stream.StreamInfoItem
import java.time.Duration
import java.time.Instant
class LocalFeedRepository : FeedRepository {
private val relevantTabs =
arrayOf(ChannelTabs.LIVESTREAMS, ChannelTabs.VIDEOS, ChannelTabs.SHORTS)
override suspend fun getFeed(forceRefresh: Boolean): List<StreamItem> {
val nowMillis = Instant.now().toEpochMilli()
if (!forceRefresh) {
val feed = DatabaseHolder.Database.feedDao().getAll()
val oneDayAgo = nowMillis - Duration.ofDays(1).toMillis()
// only refresh if feed is empty or last refresh was more than a day ago
val lastRefresh =
PreferenceHelper.getLong(PreferenceKeys.LAST_FEED_REFRESH_TIMESTAMP_MILLIS, 0)
if (feed.isNotEmpty() && lastRefresh > oneDayAgo) {
return DatabaseHolder.Database.feedDao().getAll()
.map(SubscriptionsFeedItem::toStreamItem)
}
}
val minimumDateMillis = nowMillis - Duration.ofDays(MAX_FEED_AGE_DAYS).toMillis()
DatabaseHolder.Database.feedDao().cleanUpOlderThan(minimumDateMillis)
refreshFeed(minimumDateMillis)
PreferenceHelper.putLong(PreferenceKeys.LAST_FEED_REFRESH_TIMESTAMP_MILLIS, nowMillis)
return DatabaseHolder.Database.feedDao().getAll().map(SubscriptionsFeedItem::toStreamItem)
}
private suspend fun refreshFeed(minimumDateMillis: Long) {
val channelIds = SubscriptionHelper.getSubscriptionChannelIds()
for (channelIdChunk in channelIds.chunked(CHUNK_SIZE)) {
val collectedFeedItems = channelIdChunk.parallelMap { channelId ->
try {
getRelatedStreams(channelId)
} catch (e: Exception) {
Log.e(channelId, e.stackTraceToString())
null
}
}.filterNotNull().flatten().map(StreamItem::toFeedItem)
.filter { it.uploaded > minimumDateMillis }
DatabaseHolder.Database.feedDao().insertAll(collectedFeedItems)
}
}
private suspend fun getRelatedStreams(channelId: String): List<StreamItem> {
val channelInfo = ChannelInfo.getInfo("$YOUTUBE_FRONTEND_URL/channel/${channelId}")
val relevantInfoTabs = channelInfo.tabs.filter { tab ->
relevantTabs.any { tab.contentFilters.contains(it) }
}
val related = relevantInfoTabs.parallelMap { tab ->
runCatching {
ChannelTabInfo.getInfo(NewPipeExtractorInstance.extractor, tab).relatedItems
}.getOrElse { emptyList() }
}.flatten().filterIsInstance<StreamInfoItem>()
return related.map { item ->
StreamItem(
type = StreamItem.TYPE_STREAM,
url = item.url.replace(YOUTUBE_FRONTEND_URL, ""),
title = item.name,
uploaded = item.uploadDate?.offsetDateTime()?.toEpochSecond()?.times(1000) ?: 0,
uploadedDate = item.uploadDate?.offsetDateTime()?.toLocalDateTime()?.toLocalDate()
?.toString(),
uploaderName = item.uploaderName,
uploaderUrl = item.uploaderUrl.replace(YOUTUBE_FRONTEND_URL, ""),
uploaderAvatar = channelInfo.avatars.maxByOrNull { it.height }?.url,
thumbnail = item.thumbnails.maxByOrNull { it.height }?.url,
duration = item.duration,
views = item.viewCount,
uploaderVerified = item.isUploaderVerified,
shortDescription = item.shortDescription
)
}
}
companion object {
private const val CHUNK_SIZE = 2
private const val MAX_FEED_AGE_DAYS = 30L // 30 days
}
}

View File

@ -24,16 +24,20 @@ class LocalSubscriptionsRepository: SubscriptionsRepository {
}
override suspend fun getSubscriptions(): List<Subscription> {
val subscriptions = Database.localSubscriptionDao().getAll().map { it.channelId }
val channelIds = getSubscriptionChannelIds()
return when {
subscriptions.size > GET_SUBSCRIPTIONS_LIMIT ->
channelIds.size > GET_SUBSCRIPTIONS_LIMIT ->
RetrofitInstance.authApi
.unauthenticatedSubscriptions(subscriptions)
.unauthenticatedSubscriptions(channelIds)
else -> RetrofitInstance.authApi.unauthenticatedSubscriptions(
subscriptions.joinToString(",")
channelIds.joinToString(",")
)
}
}
override suspend fun getSubscriptionChannelIds(): List<String> {
return Database.localSubscriptionDao().getAll().map { it.channelId }
}
}

View File

@ -4,19 +4,18 @@ import com.github.libretube.api.RetrofitInstance
import com.github.libretube.api.SubscriptionHelper
import com.github.libretube.api.SubscriptionHelper.GET_SUBSCRIPTIONS_LIMIT
import com.github.libretube.api.obj.StreamItem
import com.github.libretube.extensions.toID
class PipedNoAccountFeedRepository: FeedRepository {
override suspend fun getFeed(forceRefresh: Boolean): List<StreamItem> {
val subscriptions = SubscriptionHelper.getSubscriptions().map { it.url.toID() }
val channelIds = SubscriptionHelper.getSubscriptionChannelIds()
return when {
subscriptions.size > GET_SUBSCRIPTIONS_LIMIT ->
channelIds.size > GET_SUBSCRIPTIONS_LIMIT ->
RetrofitInstance.authApi
.getUnauthenticatedFeed(subscriptions)
.getUnauthenticatedFeed(channelIds)
else -> RetrofitInstance.authApi.getUnauthenticatedFeed(
subscriptions.joinToString(",")
channelIds.joinToString(",")
)
}
}

View File

@ -8,4 +8,5 @@ interface SubscriptionsRepository {
suspend fun isSubscribed(channelId: String): Boolean?
suspend fun importSubscriptions(newChannels: List<String>)
suspend fun getSubscriptions(): List<Subscription>
suspend fun getSubscriptionChannelIds(): List<String>
}

View File

@ -33,6 +33,7 @@ import com.github.libretube.enums.NotificationId
import com.github.libretube.extensions.formatAsFileSize
import com.github.libretube.extensions.getContentLength
import com.github.libretube.extensions.parcelableExtra
import com.github.libretube.extensions.toLocalDate
import com.github.libretube.extensions.toastFromMainDispatcher
import com.github.libretube.extensions.toastFromMainThread
import com.github.libretube.helpers.DownloadHelper
@ -56,8 +57,6 @@ import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.firstOrNull
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import kotlinx.datetime.TimeZone
import kotlinx.datetime.toLocalDateTime
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
@ -159,7 +158,7 @@ class DownloadService : LifecycleService() {
streams.description,
streams.uploader,
streams.duration,
streams.uploadTimestamp?.toLocalDateTime(TimeZone.currentSystemDefault())?.date,
streams.uploadTimestamp?.toLocalDate(),
thumbnailTargetPath
)
Database.downloadDao().insertDownload(download)

View File

@ -1059,7 +1059,7 @@ class PlayerFragment : Fragment(), OnlinePlayerOptions {
// set media source and resolution in the beginning
updateResolution(commonPlayerViewModel.isFullscreen.value == true)
if (streams.category == Streams.categoryMusic) {
if (streams.category == Streams.CATEGORY_MUSIC) {
playerController.setPlaybackSpeed(1f)
}
}

View File

@ -118,7 +118,7 @@ class SubscriptionsFragment : DynamicLayoutManagerFragment() {
binding.subProgress.isVisible = true
if (viewModel.videoFeed.value == null) {
viewModel.fetchFeed(requireContext())
viewModel.fetchFeed(requireContext(), forceRefresh = false)
}
if (viewModel.subscriptions.value == null) {
viewModel.fetchSubscriptions(requireContext())
@ -134,7 +134,7 @@ class SubscriptionsFragment : DynamicLayoutManagerFragment() {
binding.subRefresh.setOnRefreshListener {
viewModel.fetchSubscriptions(requireContext())
viewModel.fetchFeed(requireContext())
viewModel.fetchFeed(requireContext(), forceRefresh = true)
}
binding.toggleSubs.isVisible = true

View File

@ -18,7 +18,6 @@ import androidx.recyclerview.widget.GridLayoutManager
import androidx.recyclerview.widget.RecyclerView
import androidx.room.withTransaction
import com.github.libretube.R
import com.github.libretube.api.obj.StreamItem
import com.github.libretube.constants.PreferenceKeys
import com.github.libretube.databinding.FragmentWatchHistoryBinding
import com.github.libretube.db.DatabaseHelper
@ -180,18 +179,7 @@ class WatchHistoryFragment : DynamicLayoutManagerFragment() {
binding.playAll.setOnClickListener {
PlayingQueue.resetToDefaults()
PlayingQueue.add(
*watchHistory.reversed().map {
StreamItem(
url = "/watch?v=${it.videoId}",
title = it.title,
thumbnail = it.thumbnailUrl,
uploaderName = it.uploader,
uploaderUrl = it.uploaderUrl,
uploaderAvatar = it.uploaderAvatar,
uploadedDate = it.uploadDate?.toString(),
duration = it.duration
)
}.toTypedArray()
*watchHistory.reversed().map(WatchHistoryItem::toStreamItem).toTypedArray()
)
NavigationHelper.navigateVideo(
requireContext(),

View File

@ -122,7 +122,7 @@ class HomeViewModel : ViewModel() {
private suspend fun tryLoadFeed(subscriptionsViewModel: SubscriptionsViewModel): List<StreamItem> {
subscriptionsViewModel.videoFeed.value?.let { return it }
val feed = SubscriptionHelper.getFeed()
val feed = SubscriptionHelper.getFeed(forceRefresh = false)
subscriptionsViewModel.videoFeed.postValue(feed)
return if (hideWatched) feed.filterWatched() else feed

View File

@ -21,10 +21,10 @@ class SubscriptionsViewModel : ViewModel() {
var subscriptions = MutableLiveData<List<Subscription>?>()
fun fetchFeed(context: Context) {
fun fetchFeed(context: Context, forceRefresh: Boolean) {
viewModelScope.launch(Dispatchers.IO) {
val videoFeed = try {
SubscriptionHelper.getFeed()
SubscriptionHelper.getFeed(forceRefresh = forceRefresh)
} catch (e: Exception) {
context.toastFromMainDispatcher(R.string.server_error)
Log.e(TAG(), e.toString())

View File

@ -9,10 +9,9 @@ import androidx.core.text.isDigitsOnly
import com.github.libretube.BuildConfig
import com.github.libretube.R
import com.github.libretube.extensions.formatShort
import com.github.libretube.extensions.toLocalDate
import com.google.common.math.IntMath.pow
import kotlinx.datetime.TimeZone
import kotlinx.datetime.toJavaLocalDate
import kotlinx.datetime.toLocalDateTime
import java.time.Instant
import java.time.LocalDateTime
import java.time.ZoneId
@ -51,9 +50,7 @@ object TextUtils {
}
fun localizeInstant(instant: kotlinx.datetime.Instant): String {
val date = instant.toLocalDateTime(TimeZone.currentSystemDefault()).date
return localizeDate(date)
return localizeDate(instant.toLocalDate())
}
/**

View File

@ -82,7 +82,7 @@ class NotificationWorker(appContext: Context, parameters: WorkerParameters) :
// fetch the users feed
val videoFeed = try {
withContext(Dispatchers.IO) {
SubscriptionHelper.getFeed()
SubscriptionHelper.getFeed(forceRefresh = true)
}
} catch (e: Exception) {
return false

View File

@ -526,6 +526,8 @@
<string name="dialog_play_offline_body">Would you like to play the video from the download folder?</string>
<string name="view_count">%1$s views</string>
<string name="delete_only_watched_videos">Only delete already watched videos</string>
<string name="local_feed_extraction">Local feed extraction</string>
<string name="local_feed_extraction_summary">Directly fetch the feed from YouTube. This may be significantly slower.</string>
<!-- Notification channel strings -->
<string name="download_channel_name">Download Service</string>

View File

@ -91,4 +91,15 @@
</PreferenceCategory>
<PreferenceCategory app:title="@string/subscriptions">
<SwitchPreferenceCompat
android:defaultValue="false"
android:icon="@drawable/ic_region"
android:summary="@string/local_feed_extraction_summary"
android:title="@string/local_feed_extraction"
app:key="local_feed_extraction" />
</PreferenceCategory>
</PreferenceScreen>