Merge branch 'tekniksprint-mark' into 'master'

Thumbnail improvements

See merge request videocore/encore!146

src/main/kotlin/se/svt/oss/encore/model/output/Output.kt

@@ -13,7 +13,6 @@ data class Output(
     val format: String = "mp4",
     val postProcessor: PostProcessor = PostProcessor { outputFolder -> listOf(outputFolder.resolve(output)) },
     val id: String,
-    val seekable: Boolean = true
 )
 
 fun interface PostProcessor {

src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncode.kt

@@ -8,76 +8,87 @@ import mu.KotlinLogging
 import se.svt.oss.encore.config.EncodingProperties
 import se.svt.oss.encore.model.EncoreJob
 import se.svt.oss.encore.model.input.DEFAULT_VIDEO_LABEL
+import se.svt.oss.encore.model.input.VideoIn
 import se.svt.oss.encore.model.input.videoInput
 import se.svt.oss.encore.model.mediafile.toParams
 import se.svt.oss.encore.model.output.Output
 import se.svt.oss.encore.model.output.VideoStreamEncode
-import se.svt.oss.mediaanalyzer.file.toFractionOrNull
-import kotlin.math.round
 
 data class ThumbnailEncode(
     val percentages: List<Int> = listOf(25, 50, 75),
     val thumbnailWidth: Int = -2,
     val thumbnailHeight: Int = 1080,
     val quality: Int = 5,
     val suffix: String = "_thumb",
+    val suffixZeroPad: Int = 2,
     val inputLabel: String = DEFAULT_VIDEO_LABEL,
-    val optional: Boolean = false
+    val optional: Boolean = false,
+    val intervalSeconds: Double? = null
 ) : OutputProducer {
 
     private val log = KotlinLogging.logger { }
 
     override fun getOutput(job: EncoreJob, encodingProperties: EncodingProperties): Output? {
         val videoInput = job.inputs.videoInput(inputLabel)
-        val inputSeekTo = videoInput?.seekTo
-        val videoStream = videoInput?.analyzedVideo?.highestBitrateVideoStream
             ?: return logOrThrow("Can not produce thumbnail $suffix. No video input with label $inputLabel!")
-
-        val frameRate = videoStream.frameRate.toFractionOrNull()?.toDouble()
-            ?: if (job.duration != null || job.seekTo != null || job.thumbnailTime != null || inputSeekTo != null) {
-                return logOrThrow("Can not produce thumbnail $suffix! No framerate detected in video input $inputLabel.")
-            } else {
-                0.0
-            }
-
-        val numFrames = job.duration?.let { round(it * frameRate).toInt() } ?: ((videoStream.numFrames) - (inputSeekTo?.let { round(it * frameRate).toInt() } ?: 0))
-        val skipFrames = job.seekTo?.let { round(it * frameRate).toInt() } ?: 0
-        val frames = job.thumbnailTime?.let {
-            listOf(round((it - (inputSeekTo ?: 0.0)) * frameRate).toInt())
-        } ?: percentages.map {
-            (it * numFrames) / 100 + skipFrames
+        val thumbnailTime = job.thumbnailTime?.let { time ->
+            videoInput.seekTo?.let { time - it } ?: time
+        }
+        val select = when {
+            thumbnailTime != null -> selectTimes(listOf(thumbnailTime))
+            intervalSeconds != null -> selectInterval(intervalSeconds, job.seekTo)
+            outputDuration(videoInput, job) <= 0 -> return logOrThrow("Can not produce thumbnail $suffix. Could not detect duration.")
+            percentages.isNotEmpty() -> selectTimes(percentagesToTimes(videoInput, job))
+            else -> return logOrThrow("Can not produce thumbnail $suffix. No times selected.")
         }
 
-        log.debug { "Thumbnail encode inputs: thumbnailTime= ${job.thumbnailTime}, framerate=$frameRate, duration= ${job.duration}, numFrames = $numFrames, skipFrames = $skipFrames. Resulting frames = $frames" }
-
-        val filter = frames.joinToString(
-            separator = "+",
-            prefix = "select=",
-            postfix = ",scale=$thumbnailWidth:$thumbnailHeight"
-        ) { "eq(n\\,$it)" }
-
-        val fileRegex = Regex("${job.baseName}$suffix\\d{2}\\.jpg")
+        val filter = "$select,scale=w=$thumbnailWidth:h=$thumbnailHeight:out_range=jpeg"
         val params = linkedMapOf(
-            "frames:v" to "${frames.size}",
-            "vsync" to "vfr",
+            "fps_mode" to "vfr",
             "q:v" to "$quality"
         )
 
+        val fileRegex = Regex("${job.baseName}$suffix\\d{$suffixZeroPad}\\.jpg")
+
         return Output(
-            id = "${suffix}02d.jpg",
+            id = "${suffix}0${suffixZeroPad}d.jpg",
             video = VideoStreamEncode(
                 params = params.toParams(),
                 filter = filter,
                 inputLabels = listOf(inputLabel)
             ),
-            output = "${job.baseName}$suffix%02d.jpg",
+            output = "${job.baseName}$suffix%0${suffixZeroPad}d.jpg",
             postProcessor = { outputFolder ->
                 outputFolder.listFiles().orEmpty().filter { it.name.matches(fileRegex) }
-            },
-            seekable = false
+            }
         )
     }
 
+    private fun selectInterval(interval: Double, outputSeek: Double?): String {
+        val select = outputSeek
+            ?.let { "gte(t\\,$it)*(isnan(prev_selected_t)+gt(floor((t-$it)/$interval)\\,floor((prev_selected_t-$it)/$interval)))" }
+            ?: "isnan(prev_selected_t)+gt(floor(t/$interval)\\,floor(prev_selected_t/$interval))"
+        return "select=$select"
+    }
+
+    private fun outputDuration(videoIn: VideoIn, job: EncoreJob): Double {
+        val videoStream = videoIn.analyzedVideo.highestBitrateVideoStream
+        var inputDuration = videoStream.duration
+        videoIn.seekTo?.let { inputDuration -= it }
+        job.seekTo?.let { inputDuration -= it }
+        return job.duration ?: inputDuration
+    }
+
+    private fun percentagesToTimes(videoIn: VideoIn, job: EncoreJob): List<Double> {
+        val outputDuration = outputDuration(videoIn, job)
+        return percentages
+            .map { it * outputDuration / 100 }
+            .map { t -> job.seekTo?.let { t + it } ?: t }
+    }
+
+    private fun selectTimes(times: List<Double>) =
+        "select=${times.joinToString("+") { "lt(prev_pts*TB\\,$it)*gte(pts*TB\\,$it)" }}"
+
     private fun logOrThrow(message: String): Output? {
         if (optional) {
             log.info { message }

src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncode.kt

@@ -11,12 +11,11 @@ import se.svt.oss.encore.model.EncoreJob
 import se.svt.oss.encore.model.input.DEFAULT_VIDEO_LABEL
 import se.svt.oss.encore.model.input.analyzedVideo
 import se.svt.oss.encore.model.input.videoInput
+import se.svt.oss.encore.model.mediafile.toParams
 import se.svt.oss.encore.model.output.Output
 import se.svt.oss.encore.model.output.VideoStreamEncode
 import se.svt.oss.mediaanalyzer.file.stringValue
-import se.svt.oss.mediaanalyzer.file.toFractionOrNull
 import kotlin.io.path.createTempDirectory
-import kotlin.math.round
 
 data class ThumbnailMapEncode(
     val tileWidth: Int = 160,
@@ -37,48 +36,49 @@ data class ThumbnailMapEncode(
         val videoStream = job.inputs.analyzedVideo(inputLabel)?.highestBitrateVideoStream
             ?: return logOrThrow("No input with label $inputLabel!")
 
-        var numFrames = videoStream.numFrames
-        val duration = job.duration
+        var inputDuration = videoStream.duration
+        inputSeekTo?.let { inputDuration -= it }
+        job.seekTo?.let { inputDuration -= it }
+        val outputDuration = job.duration ?: inputDuration
 
-        if (job.duration != null || job.seekTo != null || inputSeekTo != null) {
-            val frameRate = videoStream.frameRate.toFractionOrNull()?.toDouble()
-                ?: return logOrThrow("Can not generate thumbnail map $suffix! No framerate detected in video input $inputLabel.")
-            if (duration != null) {
-                numFrames = round(duration * frameRate).toInt()
-            } else {
-                job.seekTo?.let { numFrames -= round(it * frameRate).toInt() }
-                inputSeekTo?.let { numFrames -= round(it * frameRate).toInt() }
-            }
+        if (outputDuration <= 0) {
+            return logOrThrow("Cannot create thumbnail map $suffix! Could not detect duration.")
         }
 
-        if (numFrames < cols * rows) {
-            val message =
-                "Video input $inputLabel did not contain enough frames to generate thumbnail map $suffix: $numFrames < $cols cols * $rows rows"
-            return logOrThrow(message)
-        }
+        val interval = outputDuration / (cols * rows)
+        val select = job.seekTo
+            ?.let { "gte(t\\,$it)*(isnan(prev_selected_t)+gt(floor((t-$it)/$interval)\\,floor((prev_selected_t-$it)/$interval)))" }
+            ?: "isnan(prev_selected_t)+gt(floor(t/$interval)\\,floor(prev_selected_t/$interval))"
+
         val tempFolder = createTempDirectory(suffix).toFile()
         tempFolder.deleteOnExit()
-        val pad =
-            "aspect=${Fraction(tileWidth, tileHeight).stringValue()}:x=(ow-iw)/2:y=(oh-ih)/2" // pad to aspect ratio
-        val nthFrame = numFrames / (cols * rows)
-        var select = "not(mod(n\\,$nthFrame))"
-        job.seekTo?.let { select += "*gte(t\\,$it)" }
+
+        val pad = "aspect=${Fraction(tileWidth, tileHeight).stringValue()}:x=(ow-iw)/2:y=(oh-ih)/2"
+
+        val scale = if (format == "jpg") {
+            "-1:$tileHeight:out_range=jpeg"
+        } else {
+            "-1:$tileHeight"
+        }
+        val params = linkedMapOf(
+            "q:v" to "5",
+            "fps_mode" to "vfr"
+        )
         return Output(
             id = "$suffix.$format",
             video = VideoStreamEncode(
-                params = listOf("-q:v", "5"),
-                filter = "select=$select,pad=$pad,scale=-1:$tileHeight",
+                params = params.toParams(),
+                filter = "select=$select,pad=$pad,scale=$scale",
                 inputLabels = listOf(inputLabel)
             ),
-            output = tempFolder.resolve("${job.baseName}$suffix%03d.$format").toString(),
-            seekable = false,
+            output = tempFolder.resolve("${job.baseName}$suffix%04d.$format").toString(),
             postProcessor = { outputFolder ->
                 try {
                     val targetFile = outputFolder.resolve("${job.baseName}$suffix.$format")
                     val process = ProcessBuilder(
                         "ffmpeg",
                         "-i",
-                        "${job.baseName}$suffix%03d.$format",
+                        "${job.baseName}$suffix%04d.$format",
                         "-vf",
                         "tile=${cols}x$rows",
                         "-frames:v",

src/main/kotlin/se/svt/oss/encore/process/CommandBuilder.kt

@@ -219,16 +219,16 @@ class CommandBuilder(
             return emptyList()
         }
         return listOf("-map", MapName.VIDEO.mapLabel(output.id)) +
-            seekParams(output) +
+            seekParams() +
             "-an" +
-            durationParams(output) +
+            durationParams() +
             output.video.firstPassParams +
             listOf("-f", output.format, "/dev/null")
     }
 
     private fun secondPassParams(output: Output): List<String> {
         val mapV: List<String> =
-            output.video?.let { listOf("-map", MapName.VIDEO.mapLabel(output.id)) + seekParams(output) }
+            output.video?.let { listOf("-map", MapName.VIDEO.mapLabel(output.id)) + seekParams() }
                 ?: emptyList()
 
         val preserveAudioLayout = output.audioStreams.any { it.preserveLayout }
@@ -243,7 +243,7 @@ class CommandBuilder(
             } else {
                 MapName.AUDIO.mapLabel("${output.id}-$index")
             }
-            listOf("-map", mapLabel) + seekParams(output)
+            listOf("-map", mapLabel) + seekParams()
         }
 
         val maps = mapV + mapA
@@ -261,23 +261,17 @@ class CommandBuilder(
         val metaDataParams = listOf("-metadata", "comment=Transcoded using Encore")
 
         return maps +
-            durationParams(output) +
+            durationParams() +
             videoParams + audioParams +
             metaDataParams +
             File(outputFolder).resolve(output.output).toString()
     }
 
-    private fun seekParams(output: Output): List<String> = if (!output.seekable) {
-        emptyList()
-    } else {
+    private fun seekParams(): List<String> =
         encoreJob.seekTo?.let { listOf("-ss", "$it") } ?: emptyList()
-    }
 
-    private fun durationParams(output: Output): List<String> = if (!output.seekable) {
-        emptyList()
-    } else {
+    private fun durationParams(): List<String> =
         encoreJob.duration?.let { listOf("-t", "$it") } ?: emptyList()
-    }
 
     private enum class MapName {
         VIDEO,

src/test/kotlin/se/svt/oss/encore/model/profile/AudioEncodeTest.kt

@@ -55,7 +55,6 @@ class AudioEncodeTest {
         )
         assertThat(output)
             .hasOutput("test_aac_stereo.mp4")
-            .hasSeekable(true)
             .hasVideo(null)
             .hasId("_aac_stereo.mp4")
             .hasOnlyAudioStreams(