Add Sponsorblock highlight to chapters (#4060)

Co-authored-by: Bnyro <bnyro@tutanota.com>

app/src/main/java/com/github/libretube/api/obj/ChapterSegment.kt

@@ -1,10 +1,15 @@
 package com.github.libretube.api.obj
 
+import android.graphics.drawable.Drawable
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.Transient
 
 @Serializable
 data class ChapterSegment(
     val title: String,
     val image: String,
     val start: Long,
+    // Used only for video highlights
+    @Transient var drawable: Drawable? = null
 )
+

app/src/main/java/com/github/libretube/constants/PreferenceKeys.kt

@@ -99,6 +99,7 @@ object PreferenceKeys {
     const val PLAY_AUTOMATICALLY = "play_automatically"
     const val FULLSCREEN_GESTURES = "fullscreen_gestures"
     const val UNLIMITED_SEARCH_HISTORY = "unlimited_search_history"
+    const val SB_HIGHLIGHTS = "sb_highlights"
 
     /**
      * Background mode

app/src/main/java/com/github/libretube/helpers/PlayerHelper.kt

@@ -28,11 +28,13 @@ import androidx.media3.exoplayer.LoadControl
 import androidx.media3.ui.CaptionStyleCompat
 import com.github.libretube.R
 import com.github.libretube.api.obj.ChapterSegment
+import com.github.libretube.api.obj.PreviewFrames
 import com.github.libretube.api.obj.Segment
 import com.github.libretube.api.obj.Streams
 import com.github.libretube.constants.PreferenceKeys
 import com.github.libretube.enums.PlayerEvent
 import com.github.libretube.enums.SbSkipOptions
+import com.github.libretube.obj.PreviewFrame
 import com.google.android.material.dialog.MaterialAlertDialogBuilder
 import kotlin.math.absoluteValue
 import kotlin.math.roundToInt
@@ -40,7 +42,7 @@ import kotlin.math.roundToInt
 object PlayerHelper {
     private const val ACTION_MEDIA_CONTROL = "media_control"
     const val CONTROL_TYPE = "control_type"
-    private val SPONSOR_CATEGORIES: Array<String> =
+    private val SPONSOR_CATEGORIES =
         arrayOf(
             "intro",
             "selfpromo",
@@ -51,6 +53,7 @@ object PlayerHelper {
             "music_offtopic",
             "preview"
         )
+    const val SPONSOR_HIGHLIGHT_CATEGORY = "poi_highlight"
 
     /**
      * Create a base64 encoded DASH stream manifest
@@ -82,21 +85,6 @@ object PlayerHelper {
         }
     }
 
-    /**
-     * get the categories for sponsorBlock
-     */
-    fun getSponsorBlockCategories(): MutableMap<String, SbSkipOptions> {
-        val categories: MutableMap<String, SbSkipOptions> = mutableMapOf()
-
-        for (category in SPONSOR_CATEGORIES) {
-            val state = PreferenceHelper.getString(category + "_category", "off").uppercase()
-            if (SbSkipOptions.valueOf(state) != SbSkipOptions.OFF) {
-                categories[category] = SbSkipOptions.valueOf(state)
-            }
-        }
-        return categories
-    }
-
     fun getOrientation(videoWidth: Int, videoHeight: Int): Int {
         val fullscreenOrientationPref = PreferenceHelper.getString(
             PreferenceKeys.FULLSCREEN_ORIENTATION,
@@ -181,6 +169,12 @@ object PlayerHelper {
             true,
         )
 
+    private val sponsorBlockHighlights: Boolean
+        get() = PreferenceHelper.getBoolean(
+            PreferenceKeys.SB_HIGHLIGHTS,
+            true
+        )
+
     val defaultSubtitleCode: String?
         get() {
             val code = PreferenceHelper.getString(
@@ -430,6 +424,45 @@ object PlayerHelper {
         return this
     }
 
+    /**
+     * Get the preview frame according to the current position
+     */
+    fun getPreviewFrame(previewFrames: List<PreviewFrames>, position: Long): PreviewFrame? {
+        var startPosition: Long = 0
+        // get the frames with the best quality
+        val frames = previewFrames.maxByOrNull { it.frameHeight }
+        frames?.urls?.forEach { url ->
+            // iterate over all available positions and find the one matching the current position
+            for (y in 0 until frames.framesPerPageY) {
+                for (x in 0 until frames.framesPerPageX) {
+                    val endPosition = startPosition + frames.durationPerFrame
+                    if (position in startPosition until endPosition) {
+                        return PreviewFrame(url, x, y, frames.framesPerPageX, frames.framesPerPageY)
+                    }
+                    startPosition = endPosition
+                }
+            }
+        }
+        return null
+    }
+
+    /**
+     * get the categories for sponsorBlock
+     */
+    fun getSponsorBlockCategories(): MutableMap<String, SbSkipOptions> {
+        val categories: MutableMap<String, SbSkipOptions> = mutableMapOf()
+
+        for (category in SPONSOR_CATEGORIES) {
+            val state = PreferenceHelper.getString(category + "_category", "off").uppercase()
+            if (SbSkipOptions.valueOf(state) != SbSkipOptions.OFF) {
+                categories[category] = SbSkipOptions.valueOf(state)
+            }
+        }
+        // Add the highlights category to display in the chapters
+        if (sponsorBlockHighlights) categories[SPONSOR_HIGHLIGHT_CATEGORY] = SbSkipOptions.OFF
+        return categories
+    }
+
     /**
      * Check for SponsorBlock segments matching the current player position
      * @param context A main dispatcher context
@@ -441,7 +474,7 @@ object PlayerHelper {
         segments: List<Segment>,
         sponsorBlockConfig: MutableMap<String, SbSkipOptions>,
     ): Long? {
-        for (segment in segments) {
+        for (segment in segments.filter { it.category != SPONSOR_HIGHLIGHT_CATEGORY }) {
             val segmentStart = (segment.segment[0] * 1000f).toLong()
             val segmentEnd = (segment.segment[1] * 1000f).toLong()
 

app/src/main/java/com/github/libretube/ui/adapters/ChaptersAdapter.kt

@@ -4,6 +4,7 @@ import android.graphics.Color
 import android.text.format.DateUtils
 import android.view.LayoutInflater
 import android.view.ViewGroup
+import androidx.core.graphics.ColorUtils
 import androidx.media3.exoplayer.ExoPlayer
 import androidx.recyclerview.widget.RecyclerView
 import com.github.libretube.api.obj.ChapterSegment
@@ -27,14 +28,27 @@ class ChaptersAdapter(
     override fun onBindViewHolder(holder: ChaptersViewHolder, position: Int) {
         val chapter = chapters[position]
         holder.binding.apply {
-            ImageHelper.loadImage(chapter.image, chapterImage)
+            if (chapter.drawable != null) {
+                chapterImage.setImageDrawable(chapter.drawable)
+            } else {
+                ImageHelper.loadImage(chapter.image, chapterImage)
+            }
             chapterTitle.text = chapter.title
             timeStamp.text = DateUtils.formatElapsedTime(chapter.start)
 
-            val color = if (selectedPosition == position) {
-                ThemeHelper.getThemeColor(root.context, android.R.attr.colorControlHighlight)
-            } else {
-                Color.TRANSPARENT
+            val color = when {
+                selectedPosition == position -> {
+                    ThemeHelper.getThemeColor(root.context, android.R.attr.colorControlHighlight)
+                }
+
+                chapter.drawable != null -> ColorUtils.setAlphaComponent(
+                    ThemeHelper.getThemeColor(
+                        root.context,
+                        android.R.attr.colorPrimary
+                    ), 50
+                )
+
+                else -> Color.TRANSPARENT
             }
             chapterLL.setBackgroundColor(color)
 

app/src/main/java/com/github/libretube/ui/fragments/PlayerFragment.kt

@@ -80,6 +80,7 @@ import com.github.libretube.helpers.ImageHelper
 import com.github.libretube.helpers.LocaleHelper
 import com.github.libretube.helpers.NavigationHelper
 import com.github.libretube.helpers.PlayerHelper
+import com.github.libretube.helpers.PlayerHelper.SPONSOR_HIGHLIGHT_CATEGORY
 import com.github.libretube.helpers.PlayerHelper.checkForSegments
 import com.github.libretube.helpers.PlayerHelper.isInSegment
 import com.github.libretube.helpers.PlayerHelper.loadPlaybackParams
@@ -167,7 +168,7 @@ class PlayerFragment : Fragment(), OnlinePlayerOptions {
     /**
      * Chapters and comments
      */
-    private lateinit var chapters: List<ChapterSegment>
+    private lateinit var chapters: MutableList<ChapterSegment>
 
     /**
      * for the player view
@@ -752,7 +753,12 @@ class PlayerFragment : Fragment(), OnlinePlayerOptions {
                 }
                 // show the player notification
                 initializePlayerNotification()
-                if (PlayerHelper.sponsorBlockEnabled) fetchSponsorBlockSegments()
+
+                // Since the highlight is also a chapter, we need to fetch the other segments
+                // first
+                fetchSponsorBlockSegments()
+
+                initializeChapters()
 
                 // add the video to the watch history
                 if (PlayerHelper.watchHistoryEnabled) {
@@ -768,19 +774,22 @@ class PlayerFragment : Fragment(), OnlinePlayerOptions {
     private fun fetchSponsorBlockSegments() {
         lifecycleScope.launch(Dispatchers.IO) {
             runCatching {
-                if (sponsorBlockConfig.isEmpty()) return@runCatching
+                if (sponsorBlockConfig.isEmpty()) return@launch
                 segments =
                     RetrofitInstance.api.getSegments(
                         videoId,
                         JsonHelper.json.encodeToString(sponsorBlockConfig.keys),
                     ).segments
-                if (segments.isEmpty()) return@runCatching
+                if (segments.isEmpty()) return@launch
 
                 withContext(Dispatchers.Main) {
                     playerBinding.exoProgress.setSegments(segments)
                     playerBinding.sbToggle.visibility = View.VISIBLE
                     updateDisplayedDuration()
                 }
+                segments.firstOrNull { it.category == SPONSOR_HIGHLIGHT_CATEGORY }?.let {
+                    initializeHighlight(it)
+                }
             }
         }
     }
@@ -910,8 +919,7 @@ class PlayerFragment : Fragment(), OnlinePlayerOptions {
         playerBinding.exoTitle.text = streams.title
 
         // init the chapters recyclerview
-        chapters = streams.chapters
-        initializeChapters()
+        chapters = streams.chapters.toMutableList()
 
         // Listener for play and pause icon change
         exoPlayer.addListener(object : Player.Listener {
@@ -1182,6 +1190,7 @@ class PlayerFragment : Fragment(), OnlinePlayerOptions {
                 LinearLayoutManager.HORIZONTAL,
                 false,
             )
+
         binding.chaptersRecView.adapter = ChaptersAdapter(chapters, exoPlayer)
 
         // enable the chapters dialog in the player
@@ -1192,6 +1201,26 @@ class PlayerFragment : Fragment(), OnlinePlayerOptions {
         setCurrentChapterName()
     }
 
+    private suspend fun initializeHighlight(highlight: Segment) {
+        val frame =
+            PlayerHelper.getPreviewFrame(streams.previewFrames, exoPlayer.currentPosition) ?: return
+        val drawable = withContext(Dispatchers.IO) {
+            ImageHelper.getImage(requireContext(), frame.previewUrl)
+        }.drawable ?: return
+        val highlightChapter = ChapterSegment(
+            title = getString(R.string.chapters_videoHighlight),
+            image = "",
+            start = highlight.segment[0].toLong(),
+            drawable = drawable
+        )
+        chapters.add(highlightChapter)
+        chapters.sortBy { it.start }
+
+        withContext(Dispatchers.Main) {
+            initializeChapters()
+        }
+    }
+
     // set the name of the video chapter in the exoPlayerView
     private fun setCurrentChapterName(forceUpdate: Boolean = false, enqueueNew: Boolean = true) {
         // return if chapters are empty to avoid crashes

app/src/main/java/com/github/libretube/ui/listeners/SeekbarPreviewListener.kt

@@ -1,6 +1,5 @@
 package com.github.libretube.ui.listeners
 
-import android.graphics.Bitmap
 import android.text.format.DateUtils
 import android.view.View
 import android.view.ViewGroup.MarginLayoutParams
@@ -13,7 +12,8 @@ import coil.request.ImageRequest
 import com.github.libretube.api.obj.PreviewFrames
 import com.github.libretube.databinding.ExoStyledPlayerControlViewBinding
 import com.github.libretube.helpers.ImageHelper
-import com.github.libretube.obj.PreviewFrame
+import com.github.libretube.helpers.PlayerHelper
+import com.github.libretube.util.BitmapUtil
 
 @UnstableApi
 class SeekbarPreviewListener(
@@ -23,10 +23,10 @@ class SeekbarPreviewListener(
     private val onScrub: (position: Long) -> Unit,
     private val onScrubEnd: (position: Long) -> Unit,
 ) : TimeBar.OnScrubListener {
-    private var moving = false
+    private var scrubInProgress = false
 
     override fun onScrubStart(timeBar: TimeBar, position: Long) {
-        moving = true
+        scrubInProgress = true
 
         processPreview(position)
     }
@@ -35,7 +35,7 @@ class SeekbarPreviewListener(
      * Show a preview of the scrubber position
      */
     override fun onScrubMove(timeBar: TimeBar, position: Long) {
-        moving = true
+        scrubInProgress = true
 
         playerBinding.seekbarPreviewPosition.text = DateUtils.formatElapsedTime(position / 1000)
         processPreview(position)
@@ -49,7 +49,7 @@ class SeekbarPreviewListener(
      * Hide the seekbar preview with a short delay
      */
     override fun onScrubStop(timeBar: TimeBar, position: Long, canceled: Boolean) {
-        moving = false
+        scrubInProgress = false
         // animate the disappearance of the preview image
         playerBinding.seekbarPreview.animate()
             .alpha(0f)
@@ -69,7 +69,7 @@ class SeekbarPreviewListener(
      * Make a request to get the image frame and update its position
      */
     private fun processPreview(position: Long) {
-        val previewFrame = getPreviewFrame(position) ?: return
+        val previewFrame = PlayerHelper.getPreviewFrame(previewFrames, position) ?: return
 
         // update the offset of the preview image view
         updatePreviewX(position)
@@ -77,8 +77,8 @@ class SeekbarPreviewListener(
         val request = ImageRequest.Builder(playerBinding.seekbarPreview.context)
             .data(previewFrame.previewUrl)
             .target {
-                if (!moving) return@target
-                val frame = cutOutBitmap(it.toBitmap(), previewFrame)
+                if (!scrubInProgress) return@target
+                val frame = BitmapUtil.cutBitmapFromPreviewFrame(it.toBitmap(), previewFrame)
                 playerBinding.seekbarPreviewImage.setImageBitmap(frame)
                 playerBinding.seekbarPreview.visibility = View.VISIBLE
             }
@@ -87,43 +87,6 @@ class SeekbarPreviewListener(
         ImageHelper.imageLoader.enqueue(request)
     }
 
-    /**
-     * Get the preview frame according to the current position
-     */
-    private fun getPreviewFrame(position: Long): PreviewFrame? {
-        var startPosition: Long = 0
-        // get the frames with the best quality
-        val frames = previewFrames.maxByOrNull { it.frameHeight }
-        frames?.urls?.forEach { url ->
-            // iterate over all available positions and find the one matching the current position
-            for (y in 0 until frames.framesPerPageY) {
-                for (x in 0 until frames.framesPerPageX) {
-                    val endPosition = startPosition + frames.durationPerFrame
-                    if (position in startPosition until endPosition) {
-                        return PreviewFrame(url, x, y, frames.framesPerPageX, frames.framesPerPageY)
-                    }
-                    startPosition = endPosition
-                }
-            }
-        }
-        return null
-    }
-
-    /**
-     * Cut off a new bitmap from the image that contains multiple preview thumbnails
-     */
-    private fun cutOutBitmap(bitmap: Bitmap, previewFrame: PreviewFrame): Bitmap {
-        val heightPerFrame = bitmap.height / previewFrame.framesPerPageY
-        val widthPerFrame = bitmap.width / previewFrame.framesPerPageX
-        return Bitmap.createBitmap(
-            bitmap,
-            previewFrame.positionX * widthPerFrame,
-            previewFrame.positionY * heightPerFrame,
-            widthPerFrame,
-            heightPerFrame,
-        )
-    }
-
     /**
      * Update the offset of the preview image to fit the current scrubber position
      */
@@ -132,7 +95,7 @@ class SeekbarPreviewListener(
             val parentWidth = (playerBinding.seekbarPreview.parent as View).width
             // calculate the center-offset of the preview image view
             val offset = parentWidth * (position.toFloat() / duration.toFloat()) -
-                playerBinding.seekbarPreview.width / 2
+                    playerBinding.seekbarPreview.width / 2
             // normalize the offset to keep a minimum distance at left and right
             val maxPadding = parentWidth - MIN_PADDING - playerBinding.seekbarPreview.width
             marginStart = MathUtils.clamp(offset.toInt(), MIN_PADDING, maxPadding)

app/src/main/java/com/github/libretube/util/BitmapUtil.kt

@@ -0,0 +1,21 @@
+package com.github.libretube.util
+
+import android.graphics.Bitmap
+import com.github.libretube.obj.PreviewFrame
+
+object BitmapUtil {
+    /**
+     * Cut off a new bitmap from the image that contains multiple preview thumbnails
+     */
+    fun cutBitmapFromPreviewFrame(bitmap: Bitmap, previewFrame: PreviewFrame): Bitmap {
+        val heightPerFrame = bitmap.height / previewFrame.framesPerPageY
+        val widthPerFrame = bitmap.width / previewFrame.framesPerPageX
+        return Bitmap.createBitmap(
+            bitmap,
+            previewFrame.positionX * widthPerFrame,
+            previewFrame.positionY * heightPerFrame,
+            widthPerFrame,
+            heightPerFrame,
+        )
+    }
+}

app/src/main/res/values/strings.xml

@@ -83,6 +83,7 @@
     <string name="sponsorblock">SponsorBlock</string>
     <string name="sponsorblock_summary">Uses the https://sponsor.ajay.app API</string>
     <string name="segment_skipped">Skipped segment</string>
+    <string name="jumped_to_highlight">Jumped to Video Highlight</string>
     <string name="category_segments">Segments</string>
     <string name="category_sponsor">Sponsor</string>
     <string name="category_sponsor_description">Paid promotion, paid referrals and direct advertisements. Not for self-promotion or free genuine shoutouts to causes, creators, websites, and products.</string>
@@ -100,6 +101,8 @@
     <string name="category_music_offtopic_description">Only for use in music videos. It should cover parts of the video not part of official mixes. In the end, the video should resemble the Spotify or any other mixed version as closely as possible, or reduce talking or other distractions.</string>
     <string name="category_preview">Preview/Recap</string>
     <string name="category_preview_description">For segments detailing future content without additional info. If it includes clips that only appear here, this is very likely the wrong category.</string>
+    <string name="category_highlight">Show video highlight</string>
+    <string name="category_highlight_description">Could either be the advertised title, the thumbnail or the most interesting part of the video. You can skip to it by tapping on it in the chapters section.</string>
     <string name="license">License</string>
     <string name="color_accent">Accents</string>
     <string name="color_red">Resting red</string>
@@ -449,6 +452,7 @@
     <string name="push_channel_name">Notification Worker</string>
     <string name="push_channel_description">Shows a notification when new streams are available.</string>
     <string name="unlimited_search_history">Unlimited search history</string>
+    <string name="chapters_videoHighlight">Video Highlight</string>
     <!-- Relative time formatting strings (remove when setting the minSdk to 24) -->
     <plurals name="years_ago">
         <item quantity="one">%d year ago</item>

app/src/main/res/xml/sponsorblock_settings.xml

@@ -24,6 +24,12 @@
         app:key="sb_show_markers"
         app:title="@string/sb_markers" />
 
+    <SwitchPreferenceCompat
+        app:key="sb_highlights"
+        app:summary="@string/category_highlight_description"
+        app:title="@string/category_highlight"
+        app:defaultValue="true"/>
+
     <SwitchPreferenceCompat
         android:summary="@string/sb_color_enable"
         app:defaultValue="false"