diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index 33102c8a..00000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: Create and publish a Docker image - -on: - push: - branches: ['master','main'] - tags: - - 'v*' - -env: - REGISTRY: ghcr.io - IMAGE_NAME: encore-debian - DOCKER_BASE_IMAGE: ghcr.io/svt/avtools-osadl-jre-debian:latest - -jobs: - build-artifact: - runs-on: ubuntu-latest - container: - image: ghcr.io/svt/avtools-osadl-debian:latest - options: --user root - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - java-version: '11' - distribution: 'adopt' - - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - - name: Build with Gradle - run: ./gradlew build -x test - - - name: cache build - uses: actions/cache@v2 - id: restore-build - with: - path: ./build - key: ${{ github.sha }} - - build-and-push-image: - runs-on: ubuntu-latest - needs: build-artifact - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Get Cache Build - uses: actions/cache@v2 - id: restore-build - with: - path: ./build - key: ${{ github.sha }} - - run: ls ./build - - - name: 'Echo download path' - run: echo ${{steps.download.outputs.download-path}} - - - name: Log in to the Container registry - uses: docker/login-action@v1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v3 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }} - flavor: | - latest=true - tags: | - type=raw,value={{branch}},priority=1,enable=${{ !startsWith(github.ref, 'refs/tags/v') }} - type=ref,event=tag,priority=2 - type=raw,value=${{ env.IMAGE_NAME }}-{{branch}}-{{date 'YYYYMMDD'}}-{{sha}},priority=31,enable=${{ !startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=${{ env.IMAGE_NAME }}-{{tag}}-{{date 'YYYYMMDD'}}-{{sha}},priority=32, enable=${{ startsWith(github.ref, 'refs/tags/v') }} - - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: . - build-args: | - DOCKER_BASE_IMAGE=${{ env.DOCKER_BASE_IMAGE }} - USR=avtools - platforms: x86_64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - diff --git a/.github/workflows/publishjar.yml b/.github/workflows/publishjar.yml index 7bd155e1..8b22d914 100644 --- a/.github/workflows/publishjar.yml +++ b/.github/workflows/publishjar.yml @@ -1,4 +1,4 @@ -name: Create and publish a Spring Boot jar/Spring Boot LaunchScript Jar +name: Create and publish Spring Boot jars and GraalVm native images on: push: @@ -13,25 +13,28 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v2 + - name: Set up GraalVm + uses: graalvm/setup-graalvm@v1 with: - java-version: '11' - distribution: 'adopt' + java-version: '17' + distribution: 'graalvm-community' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Grant execute permission for gradlew run: chmod +x gradlew - - name: Build jars with Gradle - run: | - ./gradlew build -x test + - name: Build jars and native images with Gradle + run: gradlew build nativeCompile -x test - - name: Release Jars + - name: Release Jars and native images uses: softprops/action-gh-release@v1 if: startsWith(github.ref, 'refs/tags/') with: files: | - build/libs/encore*.jar + encore-web/build/libs/encore-web*boot.jar + encore-web/build/native/nativeCompile/encore-web + encore-worker/build/libs/encore-worker*boot.jar + encore-worker/build/native/nativeCompile/encore-worker diff --git a/.idea/compiler.xml b/.idea/compiler.xml index 6b4e05a5..bcb93c49 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -4,14 +4,6 @@ - - - - - - + - - - + \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts deleted file mode 100644 index d946db2c..00000000 --- a/build.gradle.kts +++ /dev/null @@ -1,152 +0,0 @@ -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile - -plugins { - idea - jacoco - id("org.springframework.boot") version "2.7.6" - id("se.ascp.gradle.gradle-versions-filter") version "0.1.16" - kotlin("jvm") version "1.7.21" - kotlin("plugin.spring") version "1.7.21" - id("com.github.fhermansson.assertj-generator") version "1.1.4" - id("org.jmailen.kotlinter") version "3.12.0" - id("io.spring.dependency-management") version "1.1.0" - id("pl.allegro.tech.build.axion-release") version "1.14.2" - - //openapi generation - id("com.github.johnrengelman.processes") version "0.5.0" - id("org.springdoc.openapi-gradle-plugin") version "1.5.0" -} - - -project.version = scmVersion.version - -apply(from = "checks.gradle") - -group = "se.svt.oss" - -assertjGenerator { - classOrPackageNames = arrayOf( - "se.svt.oss.encore.model", - "se.svt.oss.mediaanalyzer.file" - ) - entryPointPackage = "se.svt.oss.encore" -} - -kotlinter { - disabledRules = arrayOf( - "import-ordering", - "trailing-comma-on-declaration-site", - "trailing-comma-on-call-site" - ) -} - -tasks.lintKotlinTest { - source = (source - fileTree("src/test/generated-java")).asFileTree -} - -tasks.test { - useJUnitPlatform() -} - -openApi { - customBootRun { - args.set(listOf("--spring.profiles.active=local")) - } -} - -tasks.withType { - kotlinOptions.jvmTarget = "11" -} - -repositories { - mavenCentral() -} - -//don't build the extra plain jars that was auto-added in Spring Boot 2.5.0, -//https://docs.spring.io/spring-boot/docs/current/gradle-plugin/reference/htmlsingle/#packaging-executable.and-plain-archives -tasks.getByName("jar") { - enabled = false -} - -configurations { - implementation { - exclude(module = "spring-boot-starter-logging") - exclude(module = "lombok") - } -} - -dependencyManagement { - imports { - mavenBom("org.springframework.cloud:spring-cloud-dependencies:2021.0.5") - } -} - -val redissonVersion = "3.18.1" - -dependencies { - implementation("se.svt.oss:media-analyzer:2.0.1") - implementation(kotlin("reflect")) - - implementation("org.springframework.boot:spring-boot-starter-web") - implementation("org.springframework.boot:spring-boot-starter-log4j2") - implementation("org.springframework.boot:spring-boot-starter-actuator") - implementation("org.springframework.boot:spring-boot-starter-data-rest") - implementation("org.springframework.boot:spring-boot-starter-data-redis") - implementation("org.springframework.boot:spring-boot-starter-validation") - implementation("org.springframework.cloud:spring-cloud-starter-config") - implementation("org.springframework.boot:spring-boot-starter-security") - - implementation("org.redisson:redisson-spring-boot-starter:$redissonVersion") - implementation("org.redisson:redisson-spring-data-27:$redissonVersion") // match boot version - implementation("io.github.microutils:kotlin-logging:3.0.2") - implementation("com.fasterxml.jackson.module:jackson-module-kotlin") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:1.6.4") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-slf4j:1.6.4") - - implementation("org.springframework.cloud:spring-cloud-starter-openfeign") - implementation("io.github.openfeign:feign-okhttp") - implementation("org.springframework.retry:spring-retry") - implementation("org.springframework.boot:spring-boot-starter-aop") - - //openapi generation - implementation("org.springdoc:springdoc-openapi-ui:1.6.12") - implementation("org.springdoc:springdoc-openapi-kotlin:1.6.12") - implementation("org.springdoc:springdoc-openapi-data-rest:1.6.12") - implementation("org.springdoc:springdoc-openapi-hateoas:1.6.12") - - testImplementation("se.svt.oss:junit5-redis-extension:3.0.0") - testImplementation("se.svt.oss:random-port-initializer:1.0.5") - testImplementation("org.awaitility:awaitility") - testImplementation("org.springframework.boot:spring-boot-starter-test") - testImplementation("org.springframework.security:spring-security-test") - testImplementation("org.assertj:assertj-core") - testImplementation("io.mockk:mockk:1.13.2") - testImplementation("com.github.tomakehurst:wiremock-jre8:2.35.0") - testImplementation("com.ninja-squad:springmockk:3.1.1") - testImplementation("org.junit.jupiter:junit-jupiter-api") - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") -} - - -tasks.wrapper { - distributionType = Wrapper.DistributionType.ALL - gradleVersion = "7.5.1" -} - -val integrationTestsPreReq = setOf("mediainfo", "ffmpeg", "ffprobe").map { - - tasks.create("Verify $it is in path, required for integration tests", Exec::class.java) { - isIgnoreExitValue = true - executable = it - - if (!it.equals("mediainfo")) { - args("-hide_banner") - } - } -} - -tasks.test { - dependsOn(integrationTestsPreReq) -} - diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 00000000..1fe42474 --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + `kotlin-dsl` +} + +repositories { + mavenCentral() + gradlePluginPortal() +} +kotlin { + jvmToolchain(17) +} +dependencies { + implementation(platform("org.jetbrains.kotlin:kotlin-bom:1.9.10")) + implementation(platform("org.springframework.boot:spring-boot-dependencies:3.1.3")) + implementation(platform("org.springframework.cloud:spring-cloud-dependencies:2022.0.4")) + implementation("org.jetbrains.kotlin:kotlin-gradle-plugin") + implementation("org.jetbrains.kotlin:kotlin-allopen") + implementation("org.springframework.boot:spring-boot-gradle-plugin:3.1.3") + implementation("org.jmailen.gradle:kotlinter-gradle:3.13.0") + implementation("pl.allegro.tech.build:axion-release-plugin:1.14.3") + implementation("org.graalvm.buildtools:native-gradle-plugin:0.9.25") + implementation("com.github.fhermansson:assertj-gradle-plugin:1.1.5") + implementation("se.ascp.gradle:gradle-versions-filter:0.1.16") +} diff --git a/buildSrc/src/main/kotlin/encore.kotlin-conventions.gradle.kts b/buildSrc/src/main/kotlin/encore.kotlin-conventions.gradle.kts new file mode 100644 index 00000000..1f7cb68c --- /dev/null +++ b/buildSrc/src/main/kotlin/encore.kotlin-conventions.gradle.kts @@ -0,0 +1,64 @@ +plugins { + idea + jacoco + kotlin("jvm") + kotlin("plugin.spring") + id("pl.allegro.tech.build.axion-release") + id("com.github.fhermansson.assertj-generator") + id("org.jmailen.kotlinter") + id("se.ascp.gradle.gradle-versions-filter") +} + +group = "se.svt.oss" +project.version = scmVersion.version + +tasks.withType { + useJUnitPlatform() +} +apply { from("../checks.gradle") } +repositories { + mavenCentral() +} + +kotlin { + jvmToolchain(17) +} +tasks.lintKotlinTest { + source = (source - fileTree("src/test/generated-java")).asFileTree +} +tasks.formatKotlinTest { + source = (source - fileTree("src/test/generated-java")).asFileTree +} +kotlinter { + disabledRules = arrayOf( + "import-ordering", + "trailing-comma-on-declaration-site", + "trailing-comma-on-call-site" + ) +} +assertjGenerator { + classOrPackageNames = arrayOf( + "se.svt.oss.encore.model", + "se.svt.oss.mediaanalyzer.file" + ) + entryPointPackage = "se.svt.oss.encore" + useJakartaAnnotations = true +} + +val redissonVersion = "3.23.2" + +dependencies { + implementation(platform("org.jetbrains.kotlin:kotlin-bom:1.9.10")) + implementation(platform("org.springframework.boot:spring-boot-dependencies:3.1.3")) + implementation(platform("org.springframework.cloud:spring-cloud-dependencies:2022.0.4")) + implementation("com.fasterxml.jackson.module:jackson-module-kotlin") + implementation("io.github.microutils:kotlin-logging:3.0.5") + implementation("org.redisson:redisson-spring-boot-starter:$redissonVersion") + implementation("org.redisson:redisson-spring-data-31:$redissonVersion") // match boot version + testImplementation("org.springframework.boot:spring-boot-starter-test") + testImplementation("org.assertj:assertj-core") + testImplementation("io.mockk:mockk:1.13.7") +} + + + diff --git a/buildSrc/src/main/kotlin/encore.spring-boot-app-conventions.gradle.kts b/buildSrc/src/main/kotlin/encore.spring-boot-app-conventions.gradle.kts new file mode 100644 index 00000000..530c692f --- /dev/null +++ b/buildSrc/src/main/kotlin/encore.spring-boot-app-conventions.gradle.kts @@ -0,0 +1,21 @@ +import org.springframework.boot.gradle.tasks.bundling.BootJar +plugins { + kotlin("jvm") + kotlin("plugin.spring") + id("org.springframework.boot") + id("org.graalvm.buildtools.native") +} + +tasks.named("bootJar") { + archiveClassifier.set("boot") +} + +dependencies { + implementation("org.springframework.boot:spring-boot-starter") + implementation("org.springframework.cloud:spring-cloud-starter-config") + implementation("org.springframework.boot:spring-boot-starter-logging") + implementation("net.logstash.logback:logstash-logback-encoder:7.4") +} + + + diff --git a/checks.gradle b/checks.gradle index afc1d2a3..1d1e2b12 100644 --- a/checks.gradle +++ b/checks.gradle @@ -5,12 +5,12 @@ jacocoTestCoverageVerification { includes = ['se.svt.oss.encore.*'] excludes = [ '*.invoke()', - '*.Security.getEnabled()', + '*.EncoreProperties*.get*()', '*.DefaultConstructorMarker*', - '*.EncoreApplicationKt.*', + '*ApplicationKt.main*', '*.static {...}', - '*.model.*.*.get*', - '*.service.localencode.LocalEncodeService.moveFile*' + '*.model.*.get*', + '*.service.localencode.LocalEncodeService.moveFile*', ] limit { counter = 'LINE' @@ -21,8 +21,6 @@ jacocoTestCoverageVerification { failOnViolation = true } } -jacoco { - toolVersion = '0.8.7' -} + jacocoTestCoverageVerification.dependsOn jacocoTestReport check.dependsOn jacocoTestCoverageVerification diff --git a/encore-common/build.gradle.kts b/encore-common/build.gradle.kts new file mode 100644 index 00000000..4075f0d2 --- /dev/null +++ b/encore-common/build.gradle.kts @@ -0,0 +1,43 @@ +plugins { + id("encore.kotlin-conventions") +} + +dependencies { + + api("se.svt.oss:media-analyzer:2.0.3") + implementation(kotlin("reflect")) + + compileOnly("org.springdoc:springdoc-openapi-starter-webmvc-api:2.2.0") + compileOnly("org.springframework.data:spring-data-rest-core") + implementation("org.springframework.boot:spring-boot-starter-webflux") + implementation("org.springframework.boot:spring-boot-starter-validation") + + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-slf4j:1.7.3") + + testImplementation(project(":encore-web")) + testImplementation("se.svt.oss:junit5-redis-extension:3.0.0") + testImplementation("org.springframework.security:spring-security-test") + testImplementation("org.awaitility:awaitility") + testImplementation("com.github.tomakehurst:wiremock-jre8-standalone:2.35.0") + testImplementation("org.springframework.boot:spring-boot-starter-webflux") + testImplementation("org.springframework.boot:spring-boot-starter-data-rest") +} + + +val integrationTestsPreReq = setOf("mediainfo", "ffmpeg", "ffprobe").map { + + tasks.create("Verify $it is in path, required for integration tests", Exec::class.java) { + isIgnoreExitValue = true + executable = it + + if (!it.equals("mediainfo")) { + args("-hide_banner") + } + } +} + +tasks.test { + dependsOn(integrationTestsPreReq) +} + diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/ClientConfiguration.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/ClientConfiguration.kt new file mode 100644 index 00000000..a99e75bd --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/ClientConfiguration.kt @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore + +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.http.HttpHeaders +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.support.WebClientAdapter +import org.springframework.web.service.invoker.HttpServiceProxyFactory +import se.svt.oss.encore.service.callback.CallbackClient + +@Configuration +class ClientConfiguration { + + @Bean + fun callbackClient(@Value("\${service.name:encore}") userAgent: String): CallbackClient { + val webClient = WebClient.builder() + .defaultHeader(HttpHeaders.USER_AGENT, userAgent) + .build() + return HttpServiceProxyFactory.builder(WebClientAdapter.forClient(webClient)) + .build() + .createClient(CallbackClient::class.java) + } +} diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/EncoreRuntimeHints.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/EncoreRuntimeHints.kt new file mode 100644 index 00000000..c69d664e --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/EncoreRuntimeHints.kt @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore + +import org.springframework.aot.hint.MemberCategory +import org.springframework.aot.hint.RuntimeHints +import org.springframework.aot.hint.RuntimeHintsRegistrar +import se.svt.oss.encore.config.AudioMixPreset +import se.svt.oss.encore.config.EncodingProperties +import se.svt.oss.encore.config.EncoreProperties + +class EncoreRuntimeHints : RuntimeHintsRegistrar { + override fun registerHints(hints: RuntimeHints, classLoader: ClassLoader?) { + hints.reflection() + .registerType( + EncoreProperties::class.java, + MemberCategory.INVOKE_DECLARED_CONSTRUCTORS, + ) + .registerType( + EncodingProperties::class.java, + MemberCategory.INVOKE_DECLARED_CONSTRUCTORS + ) + .registerType( + AudioMixPreset::class.java, + MemberCategory.INVOKE_DECLARED_CONSTRUCTORS + ) + } +} diff --git a/src/main/kotlin/se/svt/oss/encore/MediaAnalyzerConfiguration.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/MediaAnalyzerConfiguration.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/MediaAnalyzerConfiguration.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/MediaAnalyzerConfiguration.kt diff --git a/src/main/kotlin/se/svt/oss/encore/RedisConfiguration.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/RedisConfiguration.kt similarity index 64% rename from src/main/kotlin/se/svt/oss/encore/RedisConfiguration.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/RedisConfiguration.kt index c51453a8..816a9396 100644 --- a/src/main/kotlin/se/svt/oss/encore/RedisConfiguration.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/RedisConfiguration.kt @@ -7,21 +7,42 @@ package se.svt.oss.encore import com.fasterxml.jackson.databind.ObjectMapper import org.redisson.codec.JsonJacksonCodec import org.redisson.spring.starter.RedissonAutoConfigurationCustomizer +import org.springframework.aot.hint.annotation.RegisterReflectionForBinding import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.data.redis.core.RedisKeyValueAdapter import org.springframework.data.redis.core.convert.RedisCustomConversions import org.springframework.data.redis.repository.configuration.EnableRedisRepositories +import se.svt.oss.encore.model.CancelEvent +import se.svt.oss.encore.model.EncoreJob +import se.svt.oss.encore.model.SegmentProgressEvent +import se.svt.oss.encore.model.input.AudioInput +import se.svt.oss.encore.model.input.AudioVideoInput +import se.svt.oss.encore.model.input.VideoInput +import se.svt.oss.encore.model.queue.QueueItem import se.svt.oss.encore.repository.ByteArrayToChannelLayoutConverter import se.svt.oss.encore.repository.ByteArrayToOffsetDateTimeConverter -import se.svt.oss.encore.repository.ByteArrayToURIConverter -import se.svt.oss.encore.repository.ByteArrayToUUIDConverter import se.svt.oss.encore.repository.ChannelLayoutToByteArrayConverter import se.svt.oss.encore.repository.OffsetDateTimeToByteArrayConverter -import se.svt.oss.encore.repository.URIToByteArrayConverter -import se.svt.oss.encore.repository.UUIDToByteArrayConverter +import se.svt.oss.mediaanalyzer.file.AudioFile +import se.svt.oss.mediaanalyzer.file.ImageFile +import se.svt.oss.mediaanalyzer.file.SubtitleFile +import se.svt.oss.mediaanalyzer.file.VideoFile @Configuration +@RegisterReflectionForBinding( + EncoreJob::class, + AudioVideoInput::class, + VideoInput::class, + AudioInput::class, + ImageFile::class, + VideoFile::class, + AudioFile::class, + SubtitleFile::class, + CancelEvent::class, + SegmentProgressEvent::class, + QueueItem::class +) @EnableRedisRepositories(enableKeyspaceEvents = RedisKeyValueAdapter.EnableKeyspaceEvents.ON_STARTUP) class RedisConfiguration { @@ -30,10 +51,6 @@ class RedisConfiguration { listOf( OffsetDateTimeToByteArrayConverter(), ByteArrayToOffsetDateTimeConverter(), - UUIDToByteArrayConverter(), - ByteArrayToUUIDConverter(), - URIToByteArrayConverter(), - ByteArrayToURIConverter(), ChannelLayoutToByteArrayConverter(), ByteArrayToChannelLayoutConverter() ) diff --git a/src/main/kotlin/se/svt/oss/encore/cancellation/CancellationListener.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/cancellation/CancellationListener.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/cancellation/CancellationListener.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/cancellation/CancellationListener.kt diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/cancellation/SegmentProgressListener.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/cancellation/SegmentProgressListener.kt new file mode 100644 index 00000000..f3f81db5 --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/cancellation/SegmentProgressListener.kt @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.cancellation + +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.SendChannel +import kotlinx.coroutines.channels.trySendBlocking +import org.redisson.api.listener.MessageListener +import se.svt.oss.encore.model.SegmentProgressEvent +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean + +class SegmentProgressListener( + private val encoreJobId: UUID, + private val coroutineJob: Job, + private val totalSegments: Int, + private val progressChannel: SendChannel +) : MessageListener { + + private val completedSegments: MutableSet = ConcurrentHashMap.newKeySet() + val anyFailed = AtomicBoolean(false) + + fun completed() = anyFailed.get() || completedSegments.size == totalSegments + + fun completionCount() = completedSegments.size + + override fun onMessage(channel: CharSequence?, msg: SegmentProgressEvent?) { + if (msg?.jobId == encoreJobId) { + if (!msg.success) { + progressChannel.close() + anyFailed.set(true) + coroutineJob.cancel("Segment ${msg.segment} failed!") + } else if (completedSegments.add(msg.segment)) { + val percent = (completedSegments.size * 100.0 / totalSegments).toInt() + progressChannel.trySendBlocking(percent) + if (percent == 100) { + progressChannel.close() + } + } + } + } +} diff --git a/src/main/kotlin/se/svt/oss/encore/config/AudioMixPreset.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/config/AudioMixPreset.kt similarity index 73% rename from src/main/kotlin/se/svt/oss/encore/config/AudioMixPreset.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/config/AudioMixPreset.kt index e8b8a3cd..f8e1d063 100644 --- a/src/main/kotlin/se/svt/oss/encore/config/AudioMixPreset.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/config/AudioMixPreset.kt @@ -3,10 +3,13 @@ // SPDX-License-Identifier: EUPL-1.2 package se.svt.oss.encore.config +import org.springframework.boot.context.properties.NestedConfigurationProperty import se.svt.oss.encore.model.profile.ChannelLayout data class AudioMixPreset( val fallbackToAuto: Boolean = true, + @NestedConfigurationProperty val defaultPan: Map = emptyMap(), + @NestedConfigurationProperty val panMapping: Map> = emptyMap(), ) diff --git a/src/main/kotlin/se/svt/oss/encore/config/EncodingProperties.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/config/EncodingProperties.kt similarity index 58% rename from src/main/kotlin/se/svt/oss/encore/config/EncodingProperties.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/config/EncodingProperties.kt index bb044237..af78ad46 100644 --- a/src/main/kotlin/se/svt/oss/encore/config/EncodingProperties.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/config/EncodingProperties.kt @@ -1,9 +1,16 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + package se.svt.oss.encore.config +import org.springframework.boot.context.properties.NestedConfigurationProperty import se.svt.oss.encore.model.profile.ChannelLayout data class EncodingProperties( + @NestedConfigurationProperty val audioMixPresets: Map = mapOf("default" to AudioMixPreset()), + @NestedConfigurationProperty val defaultChannelLayouts: Map = emptyMap(), val flipWidthHeightIfPortrait: Boolean = true ) diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/config/EncoreProperties.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/config/EncoreProperties.kt new file mode 100644 index 00000000..f76b7c63 --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/config/EncoreProperties.kt @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.config + +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.boot.context.properties.NestedConfigurationProperty +import java.io.File +import java.time.Duration + +@ConfigurationProperties("encore-settings") +data class EncoreProperties( + /** + * transcode to local tmp dir before copying to output folder + */ + val localTemporaryEncode: Boolean = false, + /** + * number of work queues and threads + */ + val concurrency: Int = 2, + /** + * time to wait after application start before polling queue + */ + val pollInitialDelay: Duration = Duration.ofSeconds(10), + /** + * time to wait between polls + */ + val pollDelay: Duration = Duration.ofSeconds(5), + /** + * poll only the specified queue + */ + val pollQueue: Int? = null, + /** + * disable polling. could be set on encore-web if all transcoding is to be done by encore-workers + */ + val pollDisabled: Boolean = false, + /** + * should queues with higher prio be poller before the queue assigned to thread or worker + */ + val pollHigherPrio: Boolean = true, + /** + * if true, encore-worker will poll the queue until empty before shutting down, otherwise just poll once + */ + val workerDrainQueue: Boolean = false, + val redisKeyPrefix: String = "encore", + /** + * optional web security settings + */ + val security: Security = Security(), + /** + * open api contact information + */ + val openApi: OpenApi = OpenApi(), + /** + * path to directory shared by encore instances. required for encoding in segments + */ + val sharedWorkDir: File? = null, + /** + * timeout for segemnted encode before failing + */ + val segmentedEncodeTimeout: Duration = Duration.ofMinutes(120), + @NestedConfigurationProperty + val encoding: EncodingProperties = EncodingProperties() +) { + data class Security( + val enabled: Boolean = false, + val userPassword: String = "", + val adminPassword: String = "" + ) + + data class OpenApi( + val title: String = "Encore OpenAPI", + val description: String = "Endpoints for Encore", + val contactName: String = "", + val contactUrl: String = "", + val contactEmail: String = "" + ) +} diff --git a/src/main/kotlin/se/svt/oss/encore/model/CancelEvent.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/CancelEvent.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/CancelEvent.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/CancelEvent.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/EncoreJob.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/EncoreJob.kt similarity index 91% rename from src/main/kotlin/se/svt/oss/encore/model/EncoreJob.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/EncoreJob.kt index 37ab924d..1ec8f339 100644 --- a/src/main/kotlin/se/svt/oss/encore/model/EncoreJob.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/model/EncoreJob.kt @@ -13,14 +13,13 @@ import org.springframework.data.redis.core.index.Indexed import org.springframework.validation.annotation.Validated import se.svt.oss.encore.model.input.Input import se.svt.oss.mediaanalyzer.file.MediaFile -import java.net.URI import java.time.OffsetDateTime import java.util.UUID -import javax.validation.constraints.Max -import javax.validation.constraints.Min -import javax.validation.constraints.NotBlank -import javax.validation.constraints.NotEmpty -import javax.validation.constraints.Positive +import jakarta.validation.constraints.Max +import jakarta.validation.constraints.Min +import jakarta.validation.constraints.NotBlank +import jakarta.validation.constraints.NotEmpty +import jakarta.validation.constraints.Positive @Validated @RedisHash("encore-jobs", timeToLive = (60 * 60 * 24 * 7).toLong()) // 1 week ttl @@ -82,7 +81,7 @@ data class EncoreJob( example = "http://projectx/encorecallback", nullable = true ) - val progressCallbackUri: URI? = null, + val progressCallbackUri: String? = null, @Schema( description = "The queue priority of the EncoreJob", @@ -94,6 +93,14 @@ data class EncoreJob( @Max(100) val priority: Int = 0, + @Schema( + description = "Transcode segments of specified length in seconds in parallell. Should be a multiple of target GOP.", + example = "19.2", + nullable = true + ) + @Positive + val segmentLength: Double? = null, + @Schema( description = "The exception message, if the EncoreJob failed", example = "input/output error", diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/model/SegmentProgressEvent.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/SegmentProgressEvent.kt new file mode 100644 index 00000000..b521b8f7 --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/model/SegmentProgressEvent.kt @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.model + +import com.fasterxml.jackson.annotation.JsonTypeInfo +import java.util.UUID + +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) +data class SegmentProgressEvent(val jobId: UUID, val segment: Int, val success: Boolean) diff --git a/src/main/kotlin/se/svt/oss/encore/model/Status.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/Status.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/Status.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/Status.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/callback/JobProgress.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/callback/JobProgress.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/callback/JobProgress.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/callback/JobProgress.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt similarity index 92% rename from src/main/kotlin/se/svt/oss/encore/model/input/Input.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt index 95bc460b..44ce36dd 100644 --- a/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt @@ -14,8 +14,8 @@ import se.svt.oss.mediaanalyzer.file.FractionString import se.svt.oss.mediaanalyzer.file.MediaContainer import se.svt.oss.mediaanalyzer.file.MediaFile import se.svt.oss.mediaanalyzer.file.VideoFile -import javax.validation.constraints.Pattern -import javax.validation.constraints.PositiveOrZero +import jakarta.validation.constraints.Pattern +import jakarta.validation.constraints.PositiveOrZero const val TYPE_AUDIO_VIDEO = "AudioVideo" const val TYPE_AUDIO = "Audio" @@ -57,6 +57,10 @@ sealed interface Input { nullable = true ) val seekTo: Double? + + val copyTs: Boolean + + fun withSeekTo(seekTo: Double): Input } sealed interface AudioIn : Input { @@ -153,7 +157,8 @@ data class AudioInput( override var analyzed: MediaFile? = null, override val audioStream: Int? = null, override val channelLayout: ChannelLayout? = null, - override val seekTo: Double? = null + override val seekTo: Double? = null, + override val copyTs: Boolean = false ) : AudioIn { override val analyzedAudio: MediaContainer @JsonIgnore @@ -162,6 +167,8 @@ data class AudioInput( override val type: String get() = TYPE_AUDIO + override fun withSeekTo(seekTo: Double) = copy(seekTo = seekTo) + val duration: Double @JsonIgnore get() = analyzedAudio.duration @@ -178,7 +185,8 @@ data class VideoInput( override var analyzed: MediaFile? = null, override val videoStream: Int? = null, override val probeInterlaced: Boolean = true, - override val seekTo: Double? = null + override val seekTo: Double? = null, + override val copyTs: Boolean = false ) : VideoIn { override val analyzedVideo: VideoFile @JsonIgnore @@ -187,6 +195,8 @@ data class VideoInput( override val type: String get() = TYPE_VIDEO + override fun withSeekTo(seekTo: Double) = copy(seekTo = seekTo) + val duration: Double @JsonIgnore get() = analyzedVideo.duration @@ -207,7 +217,8 @@ data class AudioVideoInput( override val audioStream: Int? = null, override val probeInterlaced: Boolean = true, override val channelLayout: ChannelLayout? = null, - override val seekTo: Double? = null + override val seekTo: Double? = null, + override val copyTs: Boolean = false ) : VideoIn, AudioIn { override val analyzedVideo: VideoFile @JsonIgnore @@ -220,6 +231,8 @@ data class AudioVideoInput( override val type: String get() = TYPE_AUDIO_VIDEO + override fun withSeekTo(seekTo: Double) = copy(seekTo = seekTo) + val duration: Double @JsonIgnore get() = analyzedVideo.duration @@ -230,6 +243,7 @@ fun List.inputParams(readDuration: Double?): List = input.params.toParams() + (readDuration?.let { listOf("-t", "$it") } ?: emptyList()) + (input.seekTo?.let { listOf("-ss", "$it") } ?: emptyList()) + + (if (input.copyTs) listOf("-copyts") else emptyList()) + listOf("-i", input.uri) } diff --git a/src/main/kotlin/se/svt/oss/encore/model/mediafile/AudioLayout.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/mediafile/AudioLayout.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/mediafile/AudioLayout.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/mediafile/AudioLayout.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/mediafile/Extensions.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/mediafile/Extensions.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/mediafile/Extensions.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/mediafile/Extensions.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/output/Output.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/output/Output.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/output/Output.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/output/Output.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/AudioEncode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/AudioEncode.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/AudioEncode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/AudioEncode.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/AudioEncoder.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/AudioEncoder.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/AudioEncoder.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/AudioEncoder.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/ChannelId.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ChannelId.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/ChannelId.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ChannelId.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/ChannelLayout.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ChannelLayout.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/ChannelLayout.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ChannelLayout.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/GenericVideoEncode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/GenericVideoEncode.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/GenericVideoEncode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/GenericVideoEncode.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/OutputProducer.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/OutputProducer.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/OutputProducer.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/OutputProducer.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/Profile.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/Profile.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/Profile.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/Profile.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/SimpleAudioEncode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/SimpleAudioEncode.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/SimpleAudioEncode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/SimpleAudioEncode.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncode.kt similarity index 96% rename from src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncode.kt index 011de7fa..0ef510a9 100644 --- a/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncode.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncode.kt @@ -29,6 +29,9 @@ data class ThumbnailEncode( private val log = KotlinLogging.logger { } override fun getOutput(job: EncoreJob, encodingProperties: EncodingProperties): Output? { + if (job.segmentLength != null) { + return logOrThrow("Thumbnail is not supported in segmented encode!") + } val videoInput = job.inputs.videoInput(inputLabel) ?: return logOrThrow("Can not produce thumbnail $suffix. No video input with label $inputLabel!") val thumbnailTime = job.thumbnailTime?.let { time -> diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncode.kt similarity index 96% rename from src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncode.kt index 9f0576ff..d3cd406d 100644 --- a/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncode.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncode.kt @@ -31,6 +31,9 @@ data class ThumbnailMapEncode( private val log = KotlinLogging.logger { } override fun getOutput(job: EncoreJob, encodingProperties: EncodingProperties): Output? { + if (job.segmentLength != null) { + return logOrThrow("Thumbnail map is not supported in segmented encode!") + } val videoInput = job.inputs.videoInput(inputLabel) val inputSeekTo = videoInput?.seekTo val videoStream = job.inputs.analyzedVideo(inputLabel)?.highestBitrateVideoStream diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/VideoEncode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/VideoEncode.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/VideoEncode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/VideoEncode.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/X264Encode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/X264Encode.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/X264Encode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/X264Encode.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/X265Encode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/X265Encode.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/X265Encode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/X265Encode.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/profile/X26XEncode.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/X26XEncode.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/model/profile/X26XEncode.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/profile/X26XEncode.kt diff --git a/src/main/kotlin/se/svt/oss/encore/model/queue/QueueItem.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/queue/QueueItem.kt similarity index 65% rename from src/main/kotlin/se/svt/oss/encore/model/queue/QueueItem.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/model/queue/QueueItem.kt index 25dc4ba5..966acb24 100644 --- a/src/main/kotlin/se/svt/oss/encore/model/queue/QueueItem.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/model/queue/QueueItem.kt @@ -4,7 +4,12 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo import java.time.LocalDateTime @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY) -data class QueueItem(val id: String, val priority: Int = 0, val created: LocalDateTime = LocalDateTime.now()) : +data class QueueItem( + val id: String, + val priority: Int = 0, + val created: LocalDateTime = LocalDateTime.now(), + val segment: Int? = null +) : Comparable { override fun compareTo(other: QueueItem): Int { if (this == other) { @@ -18,6 +23,10 @@ data class QueueItem(val id: String, val priority: Int = 0, val created: LocalDa if (createdCompare != 0) { return createdCompare } - return id.compareTo(other.id) + val idCompare = id.compareTo(other.id) + if (idCompare != 0) { + return idCompare + } + return (segment ?: 0).compareTo((other.segment ?: 0)) } } diff --git a/src/main/kotlin/se/svt/oss/encore/process/CommandBuilder.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/process/CommandBuilder.kt similarity index 96% rename from src/main/kotlin/se/svt/oss/encore/process/CommandBuilder.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/process/CommandBuilder.kt index e5251f11..0e2d7f27 100644 --- a/src/main/kotlin/se/svt/oss/encore/process/CommandBuilder.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/process/CommandBuilder.kt @@ -267,8 +267,18 @@ class CommandBuilder( File(outputFolder).resolve(output.output).toString() } - private fun seekParams(): List = - encoreJob.seekTo?.let { listOf("-ss", "$it") } ?: emptyList() + private fun seekParams(): List { + val copyTsAdjustment = encoreJob.inputs + .filter { it.copyTs } + .mapNotNull { it.seekTo } + .maxOrNull() + val seekTo = listOfNotNull(copyTsAdjustment, encoreJob.seekTo).sum() + return if (seekTo > 0) { + listOf("-ss", "$seekTo") + } else { + emptyList() + } + } private fun durationParams(): List = encoreJob.duration?.let { listOf("-t", "$it") } ?: emptyList() diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/process/SegmentUtil.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/process/SegmentUtil.kt new file mode 100644 index 00000000..c0c8ec05 --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/process/SegmentUtil.kt @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.process + +import se.svt.oss.encore.model.EncoreJob +import se.svt.oss.mediaanalyzer.file.MediaContainer +import kotlin.math.ceil + +fun EncoreJob.segmentLengthOrThrow() = segmentLength ?: throw RuntimeException("No segmentLength in job!") + +fun EncoreJob.numSegments(): Int { + val segLen = segmentLengthOrThrow() + val readDuration = duration + return if (readDuration != null) { + ceil(readDuration / segLen).toInt() + } else { + val segments = + inputs.map { ceil(((it.analyzed as MediaContainer).duration - (it.seekTo ?: 0.0)) / segLen).toInt() }.toSet() + if (segments.size > 1) { + throw RuntimeException("Inputs differ in length") + } + segments.first() + } +} + +fun EncoreJob.segmentDuration(segmentNumber: Int): Double = when { + duration == null -> segmentLengthOrThrow() + segmentNumber < numSegments() - 1 -> segmentLengthOrThrow() + else -> duration!! % segmentLengthOrThrow() +} + +fun EncoreJob.baseName(segmentNumber: Int) = "${baseName}_%05d".format(segmentNumber) diff --git a/src/main/kotlin/se/svt/oss/encore/repository/ChannelLayoutConverters.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/repository/ChannelLayoutConverters.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/repository/ChannelLayoutConverters.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/repository/ChannelLayoutConverters.kt diff --git a/src/main/kotlin/se/svt/oss/encore/repository/EncoreJobRepository.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/repository/EncoreJobRepository.kt similarity index 89% rename from src/main/kotlin/se/svt/oss/encore/repository/EncoreJobRepository.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/repository/EncoreJobRepository.kt index fce56fb8..2bb9a760 100644 --- a/src/main/kotlin/se/svt/oss/encore/repository/EncoreJobRepository.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/repository/EncoreJobRepository.kt @@ -6,17 +6,18 @@ package se.svt.oss.encore.repository import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.tags.Tag -import java.util.UUID import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable +import org.springframework.data.repository.CrudRepository import org.springframework.data.repository.PagingAndSortingRepository import org.springframework.data.rest.core.annotation.RepositoryRestResource import se.svt.oss.encore.model.EncoreJob import se.svt.oss.encore.model.Status +import java.util.UUID @RepositoryRestResource @Tag(name = "encorejob") -interface EncoreJobRepository : PagingAndSortingRepository { +interface EncoreJobRepository : PagingAndSortingRepository, CrudRepository { @Operation(summary = "Find EncoreJobs By Status", description = "Returns EncoreJobs according to the given Status") fun findByStatus(status: Status, pageable: Pageable): Page diff --git a/src/main/kotlin/se/svt/oss/encore/repository/OffsetDateTimeConverters.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/repository/OffsetDateTimeConverters.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/repository/OffsetDateTimeConverters.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/repository/OffsetDateTimeConverters.kt diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/EncoreService.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/EncoreService.kt new file mode 100644 index 00000000..7bbfe527 --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/EncoreService.kt @@ -0,0 +1,278 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.service + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.consumeAsFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.sample +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.slf4j.MDCContext +import kotlinx.coroutines.time.withTimeout +import mu.KotlinLogging +import org.redisson.api.RTopic +import org.redisson.api.RedissonClient +import org.springframework.data.redis.core.PartialUpdate +import org.springframework.data.redis.core.RedisKeyValueTemplate +import org.springframework.stereotype.Service +import se.svt.oss.encore.cancellation.CancellationListener +import se.svt.oss.encore.cancellation.SegmentProgressListener +import se.svt.oss.encore.config.EncoreProperties +import se.svt.oss.encore.model.CancelEvent +import se.svt.oss.encore.model.EncoreJob +import se.svt.oss.encore.model.SegmentProgressEvent +import se.svt.oss.encore.model.Status +import se.svt.oss.encore.model.queue.QueueItem +import se.svt.oss.encore.process.baseName +import se.svt.oss.encore.process.numSegments +import se.svt.oss.encore.process.segmentDuration +import se.svt.oss.encore.process.segmentLengthOrThrow +import se.svt.oss.encore.repository.EncoreJobRepository +import se.svt.oss.encore.service.callback.CallbackService +import se.svt.oss.encore.service.localencode.LocalEncodeService +import se.svt.oss.encore.service.mediaanalyzer.MediaAnalyzerService +import se.svt.oss.encore.service.queue.QueueService +import se.svt.oss.mediaanalyzer.file.MediaContainer +import se.svt.oss.mediaanalyzer.file.MediaFile +import java.io.File +import java.util.Locale +import kotlin.time.TimedValue +import kotlin.time.measureTimedValue + +@Service +class EncoreService( + private val callbackService: CallbackService, + private val repository: EncoreJobRepository, + private val ffmpegExecutor: FfmpegExecutor, + private val redissonClient: RedissonClient, + private val redisKeyValueTemplate: RedisKeyValueTemplate, + private val mediaAnalyzerService: MediaAnalyzerService, + private val localEncodeService: LocalEncodeService, + private val encoreProperties: EncoreProperties, + private val queueService: QueueService, +) { + + private val log = KotlinLogging.logger {} + + private val cancelTopicName = "cancel" + + private fun sharedWorkDirOrNull(encoreJob: EncoreJob): File? = + encoreProperties.sharedWorkDir?.resolve(encoreJob.id.toString()) + + private fun sharedWorkDir(encoreJob: EncoreJob): File = + sharedWorkDirOrNull(encoreJob) + ?: throw IllegalStateException("Shared work dir has not been configured") + + fun encode(queueItem: QueueItem, job: EncoreJob) { + when { + queueItem.segment != null -> encodeSegment(job, queueItem.segment) + job.segmentLength != null -> encodeSegmented(job) + else -> encode(job) + } + } + + private fun encodeSegmented(encoreJob: EncoreJob) { + val coroutineJob = Job() + val cancelListener = CancellationListener(encoreJob.id, coroutineJob) + var progressListener: SegmentProgressListener? = null + var cancelTopic: RTopic? = null + var progressTopic: RTopic? = null + try { + initJob(encoreJob) + val numSegments = encoreJob.numSegments() + log.debug { "Encoding using $numSegments segments" } + cancelTopic = redissonClient.getTopic(cancelTopicName) + cancelTopic.addListener(CancelEvent::class.java, cancelListener) + progressTopic = redissonClient.getTopic("segment-progress") + val progressChannel = Channel() + progressListener = SegmentProgressListener(encoreJob.id, coroutineJob, numSegments, progressChannel) + progressTopic.addListener(SegmentProgressEvent::class.java, progressListener) + val timedOutput = measureTimedValue { + sharedWorkDir(encoreJob).mkdirs() + repeat(numSegments) { + queueService.enqueue( + QueueItem( + id = encoreJob.id.toString(), + priority = encoreJob.priority, + segment = it + ) + ) + } + + runBlocking(coroutineJob + MDCContext()) { + withTimeout(encoreProperties.segmentedEncodeTimeout) { + handleProgress(progressChannel, encoreJob) + while (!progressListener.completed()) { + log.info { "Awaiting completion ${progressListener.completionCount()}/$numSegments..." } + delay(1000) + } + } + } + if (progressListener.anyFailed.get()) { + throw RuntimeException("Some segments failed") + } + log.info { "All segments completed" } + val outWorkDir = sharedWorkDir(encoreJob) + val suffixes = mutableSetOf() + repeat(numSegments) { segmentNum -> + val segmentBaseName = encoreJob.baseName(segmentNum) + outWorkDir.list()?.filter { it.startsWith(segmentBaseName) } + ?.forEach { + val suffix = it.replaceFirst(segmentBaseName, "") + suffixes.add(suffix) + outWorkDir.resolve("$suffix.txt").appendText("file $it\n") + } + } + val outputFolder = File(encoreJob.outputFolder) + outputFolder.mkdirs() + val outputFiles = suffixes.map { + val targetName = encoreJob.baseName + it + log.info { "Joining segments for $targetName" } + val targetFile = outputFolder.resolve(targetName) + ffmpegExecutor.joinSegments(outWorkDir.resolve("$it.txt"), targetFile) + } + outputFiles + } + updateSuccessfulJob(encoreJob, timedOutput) + } catch (e: CancellationException) { + log.error(e) { "Job execution cancelled: ${e.message}" } + encoreJob.status = Status.CANCELLED + encoreJob.message = e.message + } catch (e: Exception) { + log.error(e) { "Job execution failed: ${e.message}" } + encoreJob.status = Status.FAILED + encoreJob.message = e.message + } finally { + repository.save(encoreJob) + sharedWorkDirOrNull(encoreJob)?.deleteRecursively() + cancelTopic?.removeListener(cancelListener) + progressListener?.let { progressTopic?.removeListener(it) } + callbackService.sendProgressCallback(encoreJob) + } + } + + private fun encodeSegment(encoreJob: EncoreJob, segmentNumber: Int) { + try { + log.info { "Start encoding ${encoreJob.baseName} segment $segmentNumber/${encoreJob.numSegments()} " } + val outputFolder = sharedWorkDir(encoreJob).absolutePath + val job = encoreJob.copy( + baseName = encoreJob.baseName(segmentNumber), + duration = encoreJob.segmentDuration(segmentNumber), + inputs = encoreJob.inputs.map { + it.withSeekTo((it.seekTo ?: 0.0) + encoreJob.segmentLengthOrThrow() * segmentNumber) + } + ) + ffmpegExecutor.run(job, outputFolder, null) + redissonClient.getTopic("segment-progress").publish(SegmentProgressEvent(encoreJob.id, segmentNumber, true)) + log.info { "Completed ${encoreJob.baseName} segment $segmentNumber/${encoreJob.numSegments()} " } + } catch (e: Exception) { + log.error(e) { "Error encoding segment $segmentNumber: ${e.message}" } + redissonClient.getTopic("segment-progress") + .publish(SegmentProgressEvent(encoreJob.id, segmentNumber, false)) + } + } + + private fun encode(encoreJob: EncoreJob) { + val coroutineJob = Job() + val cancelListener = CancellationListener(encoreJob.id, coroutineJob) + var cancelTopic: RTopic? = null + var outputFolder: String? = null + + try { + cancelTopic = redissonClient.getTopic(cancelTopicName) + cancelTopic.addListener(CancelEvent::class.java, cancelListener) + outputFolder = localEncodeService.outputFolder(encoreJob) + + val timedOutput = measureTimedValue { + initJob(encoreJob) + + val outputFiles = runBlocking(coroutineJob + MDCContext()) { + val progressChannel = Channel() + handleProgress(progressChannel, encoreJob) + ffmpegExecutor.run(encoreJob, outputFolder, progressChannel) + } + + localEncodeService.localEncodedFilesToCorrectDir(outputFolder, outputFiles, encoreJob) + } + + updateSuccessfulJob(encoreJob, timedOutput) + log.info { "Done with $encoreJob" } + } catch (e: InterruptedException) { + val message = "Job execution interrupted" + log.error(e) { message } + encoreJob.status = Status.QUEUED + encoreJob.message = message + throw e + } catch (e: CancellationException) { + log.error(e) { "Job execution cancelled: ${e.message}" } + encoreJob.status = Status.CANCELLED + encoreJob.message = e.message + } catch (e: Exception) { + log.error(e) { "Job execution failed: ${e.message}" } + encoreJob.status = Status.FAILED + encoreJob.message = e.message + } finally { + repository.save(encoreJob) + cancelTopic?.removeListener(cancelListener) + callbackService.sendProgressCallback(encoreJob) + localEncodeService.cleanup(outputFolder) + } + } + + @OptIn(FlowPreview::class) + private fun CoroutineScope.handleProgress( + progressChannel: ReceiveChannel, + encoreJob: EncoreJob + ) { + launch { + progressChannel.consumeAsFlow() + .conflate() + .distinctUntilChanged() + .sample(10_000) + .collect { + log.info { "Received progress $it" } + try { + encoreJob.progress = it + val partialUpdate = PartialUpdate(encoreJob.id, EncoreJob::class.java) + .set(encoreJob::progress.name, encoreJob.progress) + redisKeyValueTemplate.update(partialUpdate) + callbackService.sendProgressCallback(encoreJob) + } catch (e: Exception) { + log.warn(e) { "Error updating progress!" } + } + } + } + } + + private fun updateSuccessfulJob(encoreJob: EncoreJob, timedOutput: TimedValue>) { + val outputFiles = timedOutput.value + val timeInSeconds = timedOutput.duration.inWholeSeconds + val speed = outputFiles.filterIsInstance().firstOrNull()?.let { + "%.3f".format(Locale.US, it.duration / timeInSeconds).toDouble() + } ?: 0.0 + log.info { "Done encoding, time: ${timeInSeconds}s, speed: ${speed}X" } + encoreJob.output = outputFiles + encoreJob.status = Status.SUCCESSFUL + encoreJob.progress = 100 + encoreJob.speed = speed + } + + private fun initJob(encoreJob: EncoreJob) { + encoreJob.inputs.forEach { input -> + mediaAnalyzerService.analyzeInput(input) + } + log.info { "Start encoding" } + encoreJob.status = Status.IN_PROGRESS + repository.save(encoreJob) + } +} diff --git a/src/main/kotlin/se/svt/oss/encore/service/FfmpegExecutor.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/FfmpegExecutor.kt similarity index 80% rename from src/main/kotlin/se/svt/oss/encore/service/FfmpegExecutor.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/service/FfmpegExecutor.kt index fd988938..bc1742cb 100644 --- a/src/main/kotlin/se/svt/oss/encore/service/FfmpegExecutor.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/FfmpegExecutor.kt @@ -12,9 +12,8 @@ import org.springframework.stereotype.Service import se.svt.oss.encore.config.EncoreProperties import se.svt.oss.encore.model.EncoreJob import se.svt.oss.encore.model.input.maxDuration -import se.svt.oss.encore.model.output.Output -import se.svt.oss.encore.model.profile.Profile import se.svt.oss.encore.process.CommandBuilder +import se.svt.oss.encore.service.profile.ProfileService import se.svt.oss.mediaanalyzer.MediaAnalyzer import se.svt.oss.mediaanalyzer.file.MediaFile import java.io.File @@ -26,6 +25,7 @@ import kotlin.math.round @Service class FfmpegExecutor( private val mediaAnalyzer: MediaAnalyzer, + private val profileService: ProfileService, private val encoreProperties: EncoreProperties ) { @@ -39,11 +39,20 @@ class FfmpegExecutor( fun run( encoreJob: EncoreJob, - profile: Profile, - outputs: List, outputFolder: String, - progressChannel: SendChannel + progressChannel: SendChannel? ): List { + val profile = profileService.getProfile(encoreJob.profile) + val outputs = profile.encodes.mapNotNull { + it.getOutput( + encoreJob, + encoreProperties.encoding + ) + } + + check(outputs.distinctBy { it.id }.size == outputs.size) { + "Profile ${encoreJob.profile} contains duplicate suffixes: ${outputs.map { it.id }}!" + } val commands = CommandBuilder(encoreJob, profile, outputFolder, encoreProperties.encoding).buildCommands(outputs) log.info { "Start encoding ${encoreJob.baseName}..." } @@ -51,13 +60,13 @@ class FfmpegExecutor( val duration = encoreJob.duration ?: encoreJob.inputs.maxDuration() return try { File(outputFolder).mkdirs() - progressChannel.trySendBlocking(0).getOrThrow() + progressChannel?.trySendBlocking(0)?.getOrThrow() commands.forEachIndexed { index, command -> runFfmpeg(command, workDir, duration) { progress -> - progressChannel.trySendBlocking(totalProgress(progress, index, commands.size)).getOrThrow() + progressChannel?.trySendBlocking(totalProgress(progress, index, commands.size))?.getOrThrow() } } - progressChannel.close() + progressChannel?.close() outputs.flatMap { out -> out.postProcessor.process(File(outputFolder)) .map { mediaAnalyzer.analyze(it.toString()) } @@ -65,9 +74,6 @@ class FfmpegExecutor( } catch (e: CancellationException) { log.info { "Job was cancelled" } emptyList() - } catch (e: Exception) { - log.error(e) { "Failed Job" } - throw e } finally { workDir.deleteRecursively() } @@ -160,4 +166,28 @@ class FfmpegExecutor( null } } + + fun joinSegments(segmentList: File, targetFile: File): MediaFile { + val command = listOf( + "ffmpeg", + "-hide_banner", + "-loglevel", + "+level", + "-y", + "-f", + "concat", + "-safe", + "0", + "-i", + "$segmentList", + "-map", + "0", + "-ignore_unknown", + "-c", + "copy", + "$targetFile" + ) + runFfmpeg(command, segmentList.parentFile, null) {} + return mediaAnalyzer.analyze(targetFile.absolutePath) + } } diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackClient.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackClient.kt new file mode 100644 index 00000000..97de4102 --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackClient.kt @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.service.callback + +import java.net.URI +import org.springframework.http.MediaType +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.service.annotation.HttpExchange +import org.springframework.web.service.annotation.PostExchange +import se.svt.oss.encore.model.callback.JobProgress + +@HttpExchange(contentType = MediaType.APPLICATION_JSON_VALUE) +interface CallbackClient { + + @PostExchange + fun sendProgressCallback(callbackUri: URI, @RequestBody progress: JobProgress) +} diff --git a/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackService.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackService.kt similarity index 94% rename from src/main/kotlin/se/svt/oss/encore/service/callback/CallbackService.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackService.kt index db0f1e59..a1fab212 100644 --- a/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackService.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackService.kt @@ -8,6 +8,7 @@ import mu.KotlinLogging import org.springframework.stereotype.Service import se.svt.oss.encore.model.EncoreJob import se.svt.oss.encore.model.callback.JobProgress +import java.net.URI @Service class CallbackService(private val callbackClient: CallbackClient) { @@ -18,7 +19,7 @@ class CallbackService(private val callbackClient: CallbackClient) { encoreJob.progressCallbackUri?.let { try { callbackClient.sendProgressCallback( - it, + URI.create(it), JobProgress( encoreJob.id, encoreJob.externalId, diff --git a/src/main/kotlin/se/svt/oss/encore/service/localencode/LocalEncodeService.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/localencode/LocalEncodeService.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/service/localencode/LocalEncodeService.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/service/localencode/LocalEncodeService.kt diff --git a/src/main/kotlin/se/svt/oss/encore/service/mediaanalyzer/MediaAnalyzerService.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/mediaanalyzer/MediaAnalyzerService.kt similarity index 63% rename from src/main/kotlin/se/svt/oss/encore/service/mediaanalyzer/MediaAnalyzerService.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/service/mediaanalyzer/MediaAnalyzerService.kt index 76d8968a..40acf66c 100644 --- a/src/main/kotlin/se/svt/oss/encore/service/mediaanalyzer/MediaAnalyzerService.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/mediaanalyzer/MediaAnalyzerService.kt @@ -5,6 +5,7 @@ package se.svt.oss.encore.service.mediaanalyzer import mu.KotlinLogging +import org.springframework.aot.hint.annotation.RegisterReflectionForBinding import org.springframework.stereotype.Service import se.svt.oss.encore.model.input.AudioIn import se.svt.oss.encore.model.input.Input @@ -13,10 +14,34 @@ import se.svt.oss.encore.model.mediafile.selectAudioStream import se.svt.oss.encore.model.mediafile.selectVideoStream import se.svt.oss.encore.model.mediafile.trimAudio import se.svt.oss.mediaanalyzer.MediaAnalyzer +import se.svt.oss.mediaanalyzer.ffprobe.FfAudioStream +import se.svt.oss.mediaanalyzer.ffprobe.FfVideoStream +import se.svt.oss.mediaanalyzer.ffprobe.ProbeResult +import se.svt.oss.mediaanalyzer.ffprobe.UnknownStream import se.svt.oss.mediaanalyzer.file.AudioFile import se.svt.oss.mediaanalyzer.file.VideoFile +import se.svt.oss.mediaanalyzer.mediainfo.AudioTrack +import se.svt.oss.mediaanalyzer.mediainfo.GeneralTrack +import se.svt.oss.mediaanalyzer.mediainfo.ImageTrack +import se.svt.oss.mediaanalyzer.mediainfo.MediaInfo +import se.svt.oss.mediaanalyzer.mediainfo.OtherTrack +import se.svt.oss.mediaanalyzer.mediainfo.TextTrack +import se.svt.oss.mediaanalyzer.mediainfo.VideoTrack @Service +@RegisterReflectionForBinding( + MediaInfo::class, + AudioTrack::class, + GeneralTrack::class, + ImageTrack::class, + OtherTrack::class, + TextTrack::class, + VideoTrack::class, + ProbeResult::class, + FfAudioStream::class, + FfVideoStream::class, + UnknownStream::class +) class MediaAnalyzerService(private val mediaAnalyzer: MediaAnalyzer) { private val log = KotlinLogging.logger {} diff --git a/src/main/kotlin/se/svt/oss/encore/service/profile/ProfileService.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/profile/ProfileService.kt similarity index 71% rename from src/main/kotlin/se/svt/oss/encore/service/profile/ProfileService.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/service/profile/ProfileService.kt index 74205ccf..f2d9aa09 100644 --- a/src/main/kotlin/se/svt/oss/encore/service/profile/ProfileService.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/profile/ProfileService.kt @@ -10,17 +10,34 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLMapper import com.fasterxml.jackson.module.kotlin.readValue import mu.KotlinLogging +import org.springframework.aot.hint.annotation.RegisterReflectionForBinding import org.springframework.beans.factory.annotation.Value import org.springframework.core.io.Resource -import org.springframework.retry.annotation.Backoff -import org.springframework.retry.annotation.Retryable import org.springframework.stereotype.Service +import se.svt.oss.encore.model.profile.AudioEncode +import se.svt.oss.encore.model.profile.GenericVideoEncode +import se.svt.oss.encore.model.profile.OutputProducer import se.svt.oss.encore.model.profile.Profile +import se.svt.oss.encore.model.profile.SimpleAudioEncode +import se.svt.oss.encore.model.profile.ThumbnailEncode +import se.svt.oss.encore.model.profile.ThumbnailMapEncode +import se.svt.oss.encore.model.profile.X264Encode +import se.svt.oss.encore.model.profile.X265Encode import java.io.File -import java.io.IOException import java.util.Locale @Service +@RegisterReflectionForBinding( + Profile::class, + OutputProducer::class, + AudioEncode::class, + SimpleAudioEncode::class, + X264Encode::class, + X265Encode::class, + GenericVideoEncode::class, + ThumbnailEncode::class, + ThumbnailMapEncode::class +) class ProfileService( @Value("\${profile.location}") private val profileLocation: Resource, @@ -35,16 +52,6 @@ class ProfileService( objectMapper } - @Retryable( - include = [IOException::class], - maxAttempts = 3, - backoff = Backoff( - random = true, - delayExpression = "\${profiles.retry.delay:500}", - maxDelayExpression = "\${profiles.retry.max:15000}", - multiplier = 2.0 - ) - ) fun getProfile(name: String): Profile = try { log.debug { "Get profile $name. Reading profiles from $profileLocation" } val profiles = mapper.readValue>(profileLocation.inputStream) diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/queue/QueueService.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/queue/QueueService.kt new file mode 100644 index 00000000..5a16b615 --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/queue/QueueService.kt @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 +package se.svt.oss.encore.service.queue + +import jakarta.annotation.PostConstruct +import mu.KotlinLogging +import mu.withLoggingContext +import org.redisson.api.RPriorityBlockingQueue +import org.redisson.api.RedissonClient +import org.springframework.data.repository.findByIdOrNull +import org.springframework.stereotype.Component +import se.svt.oss.encore.config.EncoreProperties +import se.svt.oss.encore.model.EncoreJob +import se.svt.oss.encore.model.Status +import se.svt.oss.encore.model.queue.QueueItem +import se.svt.oss.encore.repository.EncoreJobRepository +import se.svt.oss.encore.service.queue.QueueUtil.getQueueNumberByPriority +import java.util.UUID +import java.util.concurrent.ConcurrentSkipListMap +import java.util.concurrent.TimeUnit + +@Component +class QueueService( + private val encoreProperties: EncoreProperties, + private val redisson: RedissonClient, + private val repository: EncoreJobRepository, +) { + + private val log = KotlinLogging.logger { } + private val queues = ConcurrentSkipListMap>() + + fun poll(queueNo: Int, action: (QueueItem, EncoreJob) -> Unit): Boolean { + val queueItem = if (encoreProperties.pollHigherPrio) { + pollUntil(queueNo) + } else { + getQueue(queueNo).poll() + } + if (queueItem == null) { + return false + } + log.info { "Picked up $queueItem" } + val id = UUID.fromString(queueItem.id) + val job = repository.findByIdOrNull(id) + ?: retry(id) // Sometimes there has been sync issues + ?: throw RuntimeException("Job ${queueItem.id} does not exist") + if (job.status.isCancelled) { + log.info { "Job was cancelled" } + return true + } + if (queueItem.segment != null && job.status == Status.FAILED) { + log.info { "Main job has failed" } + return true + } + withLoggingContext(job.contextMap) { + try { + action.invoke(queueItem, job) + } catch (e: InterruptedException) { + repostJob(queueItem, job) + } + } + return true + } + + private fun pollUntil(queueNo: Int): QueueItem? = + (0..queueNo) + .asSequence() + .mapNotNull { getQueue(it).poll() } + .firstOrNull() + + private fun retry(id: UUID): EncoreJob? { + Thread.sleep(5000) + log.info { "Retrying read of job from repository " } + return repository.findByIdOrNull(id) + } + + private fun repostJob(queueItem: QueueItem, job: EncoreJob) { + try { + log.info { "Adding job to queue (repost on interrupt)" } + enqueue(queueItem) + if (queueItem.segment == null) { + job.status = Status.QUEUED + repository.save(job) + } + log.info { "Added job to queue (repost on interrupt)" } + } catch (e: Exception) { + if (queueItem.segment == null) { + val message = "Failed to add interrupted job to queue" + log.error(e) { message } + job.message = message + job.status = Status.FAILED + repository.save(job) + } + } + } + + fun enqueue(job: EncoreJob) { + val queueItem = QueueItem( + id = job.id.toString(), + priority = job.priority, + created = job.createdDate.toLocalDateTime() + ) + enqueue(queueItem) + } + + fun enqueue(item: QueueItem) { + if (!queueByPrio(item.priority).offer(item, 5, TimeUnit.SECONDS)) { + throw RuntimeException("Job could not be added to queue!") + } + } + + fun getQueue(): List { + return (0 until encoreProperties.concurrency).flatMap { getQueue(it).toList() } + } + + private fun queueByPrio(priority: Int) = + getQueue(getQueueNumberByPriority(encoreProperties.concurrency, priority)) + + private fun getQueue(queueNo: Int) = queues.computeIfAbsent(queueNo) { + redisson.getPriorityBlockingQueue("${encoreProperties.redisKeyPrefix}-queue-$queueNo") + } + + @PostConstruct + internal fun handleOrphanedQueues() { + try { + val oldConcurrency = + redisson.getAtomicLong("${encoreProperties.redisKeyPrefix}-concurrency").getAndSet(encoreProperties.concurrency.toLong()).toInt() + if (oldConcurrency > encoreProperties.concurrency) { + log.info { "Moving orphaned queue items to lowest priority queue. Old concurrency: $oldConcurrency, new concurrency: ${encoreProperties.concurrency}" } + val lowestPrioQueue = getQueue(encoreProperties.concurrency - 1) + (encoreProperties.concurrency until oldConcurrency).forEach { queueNo -> + val orphanedQueue = getQueue(queueNo) + val transferred = orphanedQueue.drainTo(lowestPrioQueue) + log.info { "Moved $transferred orphaned items from queue $queueNo to lowest priority queue." } + orphanedQueue.delete() + } + } + } catch (e: Exception) { + log.error(e) { "Error checking for concurrency change: ${e.message}" } + } + } +} diff --git a/src/main/kotlin/se/svt/oss/encore/service/queue/QueueUtil.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/queue/QueueUtil.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/service/queue/QueueUtil.kt rename to encore-common/src/main/kotlin/se/svt/oss/encore/service/queue/QueueUtil.kt diff --git a/src/test/kotlin/se/svt/oss/encore/EncoreClient.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreClient.kt similarity index 51% rename from src/test/kotlin/se/svt/oss/encore/EncoreClient.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/EncoreClient.kt index b490a780..d939c759 100644 --- a/src/test/kotlin/se/svt/oss/encore/EncoreClient.kt +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreClient.kt @@ -4,51 +4,48 @@ package se.svt.oss.encore -import java.util.UUID -import org.springframework.cloud.openfeign.FeignClient import org.springframework.data.domain.Pageable import org.springframework.hateoas.PagedModel import org.springframework.http.MediaType -import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.service.annotation.GetExchange +import org.springframework.web.service.annotation.HttpExchange +import org.springframework.web.service.annotation.PostExchange import se.svt.oss.encore.model.EncoreJob import se.svt.oss.encore.model.Status import se.svt.oss.encore.model.queue.QueueItem +import java.util.UUID -@FeignClient("encore", url = "http://localhost:\${server.port}") +@HttpExchange(accept = [MediaType.APPLICATION_JSON_VALUE], contentType = MediaType.APPLICATION_JSON_VALUE) interface EncoreClient { - @GetMapping("/encoreJobs") + @GetExchange("/encoreJobs") fun jobs(): PagedModel - @GetMapping("/encoreJobs/search/findByStatus") + @GetExchange("/encoreJobs/search/findByStatus") fun findByStatus(@RequestParam("status") status: Status, pageable: Pageable): PagedModel - @PostMapping("/encoreJobs/{jobId}/cancel") + @PostExchange("/encoreJobs/{jobId}/cancel") fun cancel(@PathVariable("jobId") jobId: UUID) - @PostMapping( - "/encoreJobs", - consumes = [MediaType.APPLICATION_JSON_VALUE], - produces = [MediaType.APPLICATION_JSON_VALUE] + @PostExchange( + "/encoreJobs" ) - fun createJob(jobRequest: EncoreJob): EncoreJob + fun createJob(@RequestBody jobRequest: EncoreJob): EncoreJob - @GetMapping("/health") + @GetExchange("/health") fun health(): String - @PostMapping( - "/encoreJobs", - consumes = [MediaType.APPLICATION_JSON_VALUE], - produces = [MediaType.APPLICATION_JSON_VALUE] + @PostExchange( + "/encoreJobs" ) - fun postJson(json: String): EncoreJob + fun postJson(@RequestBody json: String): EncoreJob - @GetMapping("/encoreJobs/{jobId}") + @GetExchange("/encoreJobs/{jobId}") fun getJob(@PathVariable("jobId") jobId: UUID): EncoreJob - @GetMapping("/queue") + @GetExchange("/queue") fun queue(): List } diff --git a/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTest.kt similarity index 97% rename from src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTest.kt index d67bf870..34ee69b2 100644 --- a/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTest.kt +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTest.kt @@ -39,11 +39,13 @@ class EncoreIntegrationTest : EncoreIntegrationTestBase() { } @Test - fun multipleAudioStreamsOutput(@TempDir outputDir: File) { + fun multipleAudioStreamsOutputSegmentedEncode(@TempDir outputDir: File) { val baseName = "multiple_audio" val job = job(outputDir).copy( baseName = baseName, profile = "audio-streams", + segmentLength = 3.84, + priority = 100 ) val expectedOutPut = listOf(outputDir.resolve("$baseName.mp4").absolutePath) val createdJob = successfulTest(job, expectedOutPut) @@ -94,7 +96,7 @@ class EncoreIntegrationTest : EncoreIntegrationTestBase() { channelLayout = ChannelLayout.CH_LAYOUT_5POINT1, audioLabel = "alt", seekTo = 1.0 - ), + ) ) ) @@ -139,7 +141,7 @@ class EncoreIntegrationTest : EncoreIntegrationTestBase() { val standardPriorityJob = createAndAwaitJob( job = job( outputDir = outputDir1, - priority = 0, + priority = 0 ), pollInterval = Duration.ofMillis(200) ) { it.status == Status.IN_PROGRESS } diff --git a/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTestBase.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTestBase.kt similarity index 91% rename from src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTestBase.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTestBase.kt index 5832209f..b7a41654 100644 --- a/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTestBase.kt +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreIntegrationTestBase.kt @@ -10,6 +10,7 @@ import com.github.tomakehurst.wiremock.WireMockServer import com.github.tomakehurst.wiremock.client.WireMock.anyUrl import com.github.tomakehurst.wiremock.client.WireMock.ok import com.github.tomakehurst.wiremock.client.WireMock.post +import com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig import org.awaitility.Awaitility.await import org.awaitility.Durations import org.junit.jupiter.api.AfterEach @@ -18,29 +19,27 @@ import org.junit.jupiter.api.extension.ExtendWith import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.annotation.Import import org.springframework.core.io.Resource import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler import org.springframework.test.annotation.DirtiesContext -import org.springframework.test.context.ContextConfiguration -import se.svt.oss.junit5.redis.EmbeddedRedisExtension -import se.svt.oss.randomportinitializer.RandomPortInitializer import se.svt.oss.encore.Assertions.assertThat import se.svt.oss.encore.config.EncoreProperties -import se.svt.oss.encore.model.input.AudioVideoInput import se.svt.oss.encore.model.EncoreJob import se.svt.oss.encore.model.Status import se.svt.oss.encore.model.callback.JobProgress +import se.svt.oss.encore.model.input.AudioVideoInput import se.svt.oss.encore.model.profile.ChannelLayout +import se.svt.oss.junit5.redis.EmbeddedRedisExtension import java.io.File -import java.net.URI import java.time.Duration import java.util.UUID -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ExtendWith(EmbeddedRedisExtension::class) -@ContextConfiguration(initializers = [RandomPortInitializer::class]) @DirtiesContext -class EncoreIntegrationTestBase() { +@Import(TestConfig::class) +class EncoreIntegrationTestBase { @Autowired lateinit var encoreClient: EncoreClient @@ -70,7 +69,7 @@ class EncoreIntegrationTestBase() { @BeforeEach fun setUp() { - wireMockServer = WireMockServer() + wireMockServer = WireMockServer(wireMockConfig().dynamicPort()) wireMockServer.start() wireMockServer.stubFor( post(anyUrl()) @@ -85,7 +84,7 @@ class EncoreIntegrationTestBase() { fun successfulTest( job: EncoreJob, - expectedOutputFiles: List, + expectedOutputFiles: List ): EncoreJob { val createdJob = createAndAwaitJob( job = job, @@ -147,13 +146,13 @@ class EncoreIntegrationTestBase() { baseName = file.file.nameWithoutExtension, profile = "program", outputFolder = outputDir.absolutePath, - progressCallbackUri = URI.create("http://localhost:${wireMockServer.port()}/callbacks/111"), + progressCallbackUri = "http://localhost:${wireMockServer.port()}/callbacks/111", debugOverlay = true, priority = priority, inputs = listOf( AudioVideoInput( uri = file.file.absolutePath, - channelLayout = ChannelLayout.CH_LAYOUT_5POINT1, + channelLayout = ChannelLayout.CH_LAYOUT_5POINT1 ) ), logContext = mapOf("FlowId" to UUID.randomUUID().toString()) diff --git a/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreRuntimeHintsTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreRuntimeHintsTest.kt new file mode 100644 index 00000000..6d337a8b --- /dev/null +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/EncoreRuntimeHintsTest.kt @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore + +import org.junit.jupiter.api.Test +import org.springframework.aot.hint.RuntimeHints +import org.springframework.aot.hint.predicate.RuntimeHintsPredicates +import se.svt.oss.encore.Assertions.assertThat +import se.svt.oss.encore.config.AudioMixPreset +import se.svt.oss.encore.config.EncodingProperties +import se.svt.oss.encore.config.EncoreProperties +import kotlin.reflect.jvm.javaConstructor + +class EncoreRuntimeHintsTest { + @Test + fun shouldRegisterHints() { + val hints = RuntimeHints() + EncoreRuntimeHints().registerHints(hints, javaClass.classLoader) + assertThat( + RuntimeHintsPredicates.reflection().onConstructor(EncoreProperties::class.constructors.first().javaConstructor!!) + ).accepts(hints) + assertThat( + RuntimeHintsPredicates.reflection().onConstructor(EncodingProperties::class.constructors.first().javaConstructor!!) + ).accepts(hints) + assertThat( + RuntimeHintsPredicates.reflection().onConstructor(AudioMixPreset::class.constructors.first().javaConstructor!!) + ).accepts(hints) + } +} diff --git a/src/test/kotlin/se/svt/oss/encore/LocalEncodeIntegrationTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/LocalEncodeIntegrationTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/LocalEncodeIntegrationTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/LocalEncodeIntegrationTest.kt diff --git a/encore-common/src/test/kotlin/se/svt/oss/encore/TestConfig.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/TestConfig.kt new file mode 100644 index 00000000..8bb610d2 --- /dev/null +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/TestConfig.kt @@ -0,0 +1,22 @@ +package se.svt.oss.encore + +import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Lazy +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.support.WebClientAdapter +import org.springframework.web.service.invoker.HttpServiceProxyFactory + +@TestConfiguration(proxyBeanMethods = false) +@Lazy +class TestConfig { + + @Bean + fun encoreClient(@Value("\${local.server.port}") localPort: Int): EncoreClient { + return HttpServiceProxyFactory + .builder(WebClientAdapter.forClient(WebClient.create("http://localhost:$localPort"))) + .build() + .createClient(EncoreClient::class.java) + } +} diff --git a/src/test/kotlin/se/svt/oss/encore/TestUtils.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/TestUtils.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/TestUtils.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/TestUtils.kt diff --git a/src/test/kotlin/se/svt/oss/encore/model/input/InputTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/input/InputTest.kt similarity index 96% rename from src/test/kotlin/se/svt/oss/encore/model/input/InputTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/input/InputTest.kt index 407231b2..1882390b 100644 --- a/src/test/kotlin/se/svt/oss/encore/model/input/InputTest.kt +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/model/input/InputTest.kt @@ -124,4 +124,10 @@ internal class InputTest { .isInstanceOf(IllegalArgumentException::class.java) .hasMessage("Inputs contains duplicate video labels!") } + + @Test + fun testWithSeekTo() { + assertThat(inputs.map { it.withSeekTo(20.0) }) + .allSatisfy { assertThat(it).hasSeekTo(20.0) } + } } diff --git a/src/test/kotlin/se/svt/oss/encore/model/mediafile/MediaFileExtensionsTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/mediafile/MediaFileExtensionsTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/model/mediafile/MediaFileExtensionsTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/mediafile/MediaFileExtensionsTest.kt diff --git a/src/test/kotlin/se/svt/oss/encore/model/profile/AudioEncodeTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/AudioEncodeTest.kt similarity index 99% rename from src/test/kotlin/se/svt/oss/encore/model/profile/AudioEncodeTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/AudioEncodeTest.kt index ff538360..f534eca3 100644 --- a/src/test/kotlin/se/svt/oss/encore/model/profile/AudioEncodeTest.kt +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/AudioEncodeTest.kt @@ -188,6 +188,7 @@ class AudioEncodeTest { channels = channelCount, channelLayout = ChannelLayout.defaultChannelLayout(channelCount)?.layoutName, samplingRate = 23123, - bitrate = 213123 + bitrate = 213123, + profile = null ) } diff --git a/src/test/kotlin/se/svt/oss/encore/model/profile/GenericVideoEncodeTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/GenericVideoEncodeTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/model/profile/GenericVideoEncodeTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/GenericVideoEncodeTest.kt diff --git a/src/test/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncodeTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncodeTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncodeTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/ThumbnailEncodeTest.kt diff --git a/src/test/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncodeTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncodeTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncodeTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/ThumbnailMapEncodeTest.kt diff --git a/src/test/kotlin/se/svt/oss/encore/model/profile/VideoEncodeTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/VideoEncodeTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/model/profile/VideoEncodeTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/VideoEncodeTest.kt diff --git a/src/test/kotlin/se/svt/oss/encore/model/profile/X264EncodeTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/X264EncodeTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/model/profile/X264EncodeTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/X264EncodeTest.kt diff --git a/src/test/kotlin/se/svt/oss/encore/model/profile/X265EncodeTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/X265EncodeTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/model/profile/X265EncodeTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/profile/X265EncodeTest.kt diff --git a/src/test/kotlin/se/svt/oss/encore/model/queue/QueueItemTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/model/queue/QueueItemTest.kt similarity index 66% rename from src/test/kotlin/se/svt/oss/encore/model/queue/QueueItemTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/model/queue/QueueItemTest.kt index 51f31345..f2bcdbf9 100644 --- a/src/test/kotlin/se/svt/oss/encore/model/queue/QueueItemTest.kt +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/model/queue/QueueItemTest.kt @@ -9,17 +9,22 @@ internal class QueueItemTest { @Test fun testSortOrder() { val newHighPrioItem = QueueItem("new-high-prio", 100) - val oldHighPrioItem = QueueItem("old-high-prio", 100, LocalDateTime.now().minusHours(1)) - val olderHighPrioItem = QueueItem("older-high-prio", 100, LocalDateTime.now().minusHours(2)) + val now = LocalDateTime.now() + val oldHighPrioItem = QueueItem("old-high-prio", 100, now.minusHours(1)) + val olderHighPrioItem = QueueItem("older-high-prio", 100, now.minusHours(2)) + val segmentOne = QueueItem("segmented", 99, now, 1) + val segmentTwo = QueueItem("segmented", 99, now, 2) val newLowPrioItem = QueueItem("new-low-prio", 10) - val oldLowPrioItem = QueueItem("old-low-prio", 10, LocalDateTime.now().minusHours(1)) - val olderLowPrioItem = QueueItem("older-low-prio", 10, LocalDateTime.now().minusHours(2)) + val oldLowPrioItem = QueueItem("old-low-prio", 10, now.minusHours(1)) + val olderLowPrioItem = QueueItem("older-low-prio", 10, now.minusHours(2)) val expectedSorted = listOf( olderHighPrioItem, olderHighPrioItem, oldHighPrioItem, newHighPrioItem, + segmentOne, + segmentTwo, olderLowPrioItem, oldLowPrioItem, newLowPrioItem @@ -30,6 +35,8 @@ internal class QueueItemTest { newHighPrioItem, olderHighPrioItem, oldHighPrioItem, + segmentOne, + segmentTwo, olderLowPrioItem, olderHighPrioItem, newLowPrioItem, diff --git a/src/test/kotlin/se/svt/oss/encore/process/CommandBuilderTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/process/CommandBuilderTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/process/CommandBuilderTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/process/CommandBuilderTest.kt diff --git a/encore-common/src/test/kotlin/se/svt/oss/encore/process/SegmentUtilTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/process/SegmentUtilTest.kt new file mode 100644 index 00000000..fa1d0473 --- /dev/null +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/process/SegmentUtilTest.kt @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.process + +import org.assertj.core.data.Offset +import org.junit.jupiter.api.Test +import se.svt.oss.encore.defaultEncoreJob +import se.svt.oss.encore.Assertions.assertThat +import se.svt.oss.encore.Assertions.assertThatThrownBy +import se.svt.oss.encore.defaultVideoFile +import se.svt.oss.encore.longVideoFile +import se.svt.oss.encore.model.input.AudioVideoInput + +class SegmentUtilTest { + + private val job = defaultEncoreJob().copy( + baseName = "segment_test", + segmentLength = 19.2, + duration = null, + inputs = listOf(AudioVideoInput(uri = "test", analyzed = longVideoFile)) + ) + + @Test + fun baseName() { + assertThat(job.baseName(2)).isEqualTo("segment_test_00002") + } + + @Test + fun missingSegmentLength() { + val encoreJob = job.copy(segmentLength = null) + val message = "No segmentLength in job!" + assertThatThrownBy { + encoreJob.segmentLengthOrThrow() + }.hasMessage(message) + assertThatThrownBy { + encoreJob.numSegments() + }.hasMessage(message) + assertThatThrownBy { + encoreJob.segmentDuration(1) + }.hasMessage(message) + } + + @Test + fun hasSegmentLength() { + assertThat(job.segmentLengthOrThrow()).isEqualTo(19.2) + } + + @Test + fun numSegmentsDurationSet() { + val encoreJob = job.copy(duration = 125.0) + assertThat(encoreJob.numSegments()).isEqualTo(7) + } + + @Test + fun numSegmentsDurationNotSet() { + assertThat(job.numSegments()).isEqualTo(141) + } + + @Test + fun numSegmentsInputsDiffer() { + val encoreJob = job.copy(inputs = job.inputs + AudioVideoInput(uri = "test", analyzed = defaultVideoFile)) + assertThatThrownBy { encoreJob.numSegments() } + .hasMessage("Inputs differ in length") + } + + @Test + fun segmentDurationDurationNotSet() { + assertThat(job.segmentDuration(140)).isEqualTo(19.2) + } + + @Test + fun segmentDurationDurationSetFirst() { + assertThat(job.copy(duration = 125.0).segmentDuration(0)).isEqualTo(19.2) + } + + @Test + fun segmentDurationDurationSetLast() { + assertThat(job.copy(duration = 125.0).segmentDuration(6)).isCloseTo(9.8, Offset.offset(0.001)) + } +} diff --git a/src/test/kotlin/se/svt/oss/encore/repository/EncoreJobRepositoryTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/repository/EncoreJobRepositoryTest.kt similarity index 79% rename from src/test/kotlin/se/svt/oss/encore/repository/EncoreJobRepositoryTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/repository/EncoreJobRepositoryTest.kt index ad7cc02f..f7a2227f 100644 --- a/src/test/kotlin/se/svt/oss/encore/repository/EncoreJobRepositoryTest.kt +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/repository/EncoreJobRepositoryTest.kt @@ -9,23 +9,17 @@ import org.junit.jupiter.api.extension.ExtendWith import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.data.domain.PageRequest -import org.springframework.test.annotation.DirtiesContext import org.springframework.test.context.ActiveProfiles -import org.springframework.test.context.ContextConfiguration -import se.svt.oss.junit5.redis.EmbeddedRedisExtension -import se.svt.oss.randomportinitializer.RandomPortInitializer import se.svt.oss.encore.Assertions.assertThat import se.svt.oss.encore.model.EncoreJob import se.svt.oss.encore.model.Status -import java.net.URI +import se.svt.oss.junit5.redis.EmbeddedRedisExtension import java.time.OffsetDateTime import java.util.UUID -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ExtendWith(EmbeddedRedisExtension::class) @ActiveProfiles("test") -@ContextConfiguration(initializers = [RandomPortInitializer::class]) -@DirtiesContext class EncoreJobRepositoryTest { @Autowired @@ -41,8 +35,8 @@ class EncoreJobRepositoryTest { assertThat(findByStatus.totalElements).isEqualTo(2) val callbackUrls = findByStatus.map { it.progressCallbackUri } assertThat(callbackUrls).containsExactlyInAnyOrder( - URI.create("http://transcoder2"), - URI.create("http://transcoder3") + "http://transcoder2", + "http://transcoder3" ) repository.deleteAll() } @@ -54,7 +48,7 @@ class EncoreJobRepositoryTest { profile = "animerat", outputFolder = "/shares/test", createdDate = OffsetDateTime.now(), - progressCallbackUri = URI.create(url), + progressCallbackUri = url, baseName = "test" ) encoreJob.status = status diff --git a/src/test/kotlin/se/svt/oss/encore/service/callback/CallbackServiceTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/service/callback/CallbackServiceTest.kt similarity index 76% rename from src/test/kotlin/se/svt/oss/encore/service/callback/CallbackServiceTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/service/callback/CallbackServiceTest.kt index c024fa61..3c058f04 100644 --- a/src/test/kotlin/se/svt/oss/encore/service/callback/CallbackServiceTest.kt +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/service/callback/CallbackServiceTest.kt @@ -29,7 +29,7 @@ class CallbackServiceTest { private val encoreJob = EncoreJob( outputFolder = "/some/output", profile = "program", - progressCallbackUri = URI.create("wwww.callback.com"), + progressCallbackUri = "wwww.callback.com", progress = 50, baseName = "file" ) @@ -43,24 +43,24 @@ class CallbackServiceTest { @Test fun `successful callback`() { - every { callackClient.sendProgressCallback(encoreJob.progressCallbackUri!!, progress) } just Runs + every { callackClient.sendProgressCallback(URI.create(encoreJob.progressCallbackUri!!), progress) } just Runs callbackService.sendProgressCallback(encoreJob) - verify { callackClient.sendProgressCallback(encoreJob.progressCallbackUri!!, progress) } + verify { callackClient.sendProgressCallback(URI.create(encoreJob.progressCallbackUri!!), progress) } } @Test fun `some error upon callback`() { every { callackClient.sendProgressCallback( - encoreJob.progressCallbackUri!!, + URI.create(encoreJob.progressCallbackUri!!), progress ) } throws Exception("error") callbackService.sendProgressCallback(encoreJob) - verify { callackClient.sendProgressCallback(encoreJob.progressCallbackUri!!, progress) } + verify { callackClient.sendProgressCallback(URI.create(encoreJob.progressCallbackUri!!), progress) } } } diff --git a/src/test/kotlin/se/svt/oss/encore/service/profile/ProfileServiceTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/service/profile/ProfileServiceTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/service/profile/ProfileServiceTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/service/profile/ProfileServiceTest.kt diff --git a/encore-common/src/test/kotlin/se/svt/oss/encore/service/queue/QueueServiceTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/service/queue/QueueServiceTest.kt new file mode 100644 index 00000000..46ce23df --- /dev/null +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/service/queue/QueueServiceTest.kt @@ -0,0 +1,318 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.service.queue + +import io.mockk.Called +import io.mockk.Runs +import io.mockk.every +import io.mockk.impl.annotations.InjectMockKs +import io.mockk.impl.annotations.MockK +import io.mockk.junit5.MockKExtension +import io.mockk.just +import io.mockk.mockk +import io.mockk.verify +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.redisson.api.RPriorityBlockingQueue +import org.redisson.api.RedissonClient +import org.springframework.data.repository.findByIdOrNull +import se.svt.oss.encore.config.EncoreProperties +import se.svt.oss.encore.defaultEncoreJob +import se.svt.oss.encore.model.EncoreJob +import se.svt.oss.encore.model.Status +import se.svt.oss.encore.model.queue.QueueItem +import se.svt.oss.encore.repository.EncoreJobRepository +import java.util.UUID +import java.util.concurrent.TimeUnit + +@ExtendWith(MockKExtension::class) +internal class QueueServiceTest { + private val highPriorityQueue = mockk>() + private val standardPriorityQueue = mockk>() + private val lowPriorityQueue = mockk>() + + val keyPrefix = "encore" + private val encoreProperties = mockk { + every { concurrency } returns 3 + every { redisKeyPrefix } returns keyPrefix + every { pollHigherPrio } returns true + } + + @MockK + private lateinit var redisson: RedissonClient + + @MockK + private lateinit var repository: EncoreJobRepository + + @InjectMockKs + private lateinit var queueService: QueueService + + private val queueItemHighPrio = QueueItem(UUID.randomUUID().toString(), 90) + private val queueItemStandardPrio = QueueItem(UUID.randomUUID().toString(), 51) + private val queueItemLowPrio = QueueItem(UUID.randomUUID().toString(), 10) + private val highPrioJob = mockk { + every { contextMap } returns emptyMap() + every { status } returns Status.QUEUED + } + private val standardPrioJob = mockk { + every { contextMap } returns emptyMap() + every { status } returns Status.QUEUED + every { status = any() } just Runs + every { message = any() } just Runs + } + private val lowPrioJob = mockk { + every { status } returns Status.QUEUED + every { contextMap } returns emptyMap() + } + + private fun mockLambda(expectedQueueItem: QueueItem, expectedEncoreJob: EncoreJob): (QueueItem, EncoreJob) -> Unit = + { item, job -> + assertThat(item).isSameAs(expectedQueueItem) + assertThat(job).isSameAs(expectedEncoreJob) + } + + private val expectHighPrio = mockLambda(queueItemHighPrio, highPrioJob) + private val expectStandardPrio = mockLambda(queueItemStandardPrio, standardPrioJob) + private val expectLowPrio = mockLambda(queueItemLowPrio, lowPrioJob) + private val expectNone: (QueueItem, EncoreJob) -> Unit = { queueItem, _ -> + throw RuntimeException("Unexpected call with $queueItem") + } + + @BeforeEach + internal fun setUp() { + every { highPriorityQueue.poll() } returns queueItemHighPrio + every { standardPriorityQueue.poll() } returns queueItemStandardPrio + every { lowPriorityQueue.poll() } returns queueItemLowPrio + every { repository.findByIdOrNull(UUID.fromString(queueItemHighPrio.id)) } returns highPrioJob + every { repository.findByIdOrNull(UUID.fromString(queueItemStandardPrio.id)) } returns standardPrioJob + every { repository.findByIdOrNull(UUID.fromString(queueItemLowPrio.id)) } returns lowPrioJob + every { redisson.getPriorityBlockingQueue("$keyPrefix-queue-0") } returns highPriorityQueue + every { redisson.getPriorityBlockingQueue("$keyPrefix-queue-1") } returns standardPriorityQueue + every { redisson.getPriorityBlockingQueue("$keyPrefix-queue-2") } returns lowPriorityQueue + } + + @Nested + inner class Init { + + @Test + fun concurrencyReduced() { + val orphanedQueue1 = mockk>() + val orphanedQueue2 = mockk>() + every { orphanedQueue1.drainTo(any()) } returns 1 + every { orphanedQueue2.drainTo(any()) } returns 1 + every { orphanedQueue1.delete() } returns true + every { orphanedQueue2.delete() } returns true + val concurrency = encoreProperties.concurrency + every { + redisson.getAtomicLong("$keyPrefix-concurrency").getAndSet(concurrency.toLong()) + } returns concurrency.toLong() + 2 + every { redisson.getPriorityBlockingQueue("$keyPrefix-queue-$concurrency") } returns orphanedQueue1 + every { redisson.getPriorityBlockingQueue("$keyPrefix-queue-${concurrency + 1}") } returns orphanedQueue2 + + queueService.handleOrphanedQueues() + + verify { orphanedQueue1.drainTo(lowPriorityQueue) } + verify { orphanedQueue2.drainTo(lowPriorityQueue) } + verify { orphanedQueue1.delete() } + verify { orphanedQueue2.delete() } + } + } + + @Nested + inner class PollHighPriorityQueue { + + @AfterEach + fun tearDown() { + verify { highPriorityQueue.poll() } + verify { standardPriorityQueue wasNot Called } + verify { lowPriorityQueue wasNot Called } + } + + @Test + fun `returns item from high priority queue if any present`() { + assertThat(queueService.poll(0, expectHighPrio)).isTrue + } + + @Test + fun `returns null if high priority queue is empty`() { + every { highPriorityQueue.poll() } returns null + assertThat(queueService.poll(0, expectNone)).isFalse + } + } + + @Nested + inner class PollHighOrStandardPriorityQueue { + + @AfterEach + fun tearDown() { + verify { highPriorityQueue.poll() } + verify { lowPriorityQueue wasNot Called } + } + + @Test + fun `returns item from high priority queue if any present`() { + assertThat(queueService.poll(1, expectHighPrio)).isTrue + verify { standardPriorityQueue wasNot Called } + } + + @Test + fun `returns items from standard priority queue if high priority queue is empty`() { + every { highPriorityQueue.poll() } returns null + assertThat(queueService.poll(1, expectStandardPrio)).isTrue + verify { standardPriorityQueue.poll() } + } + + @Test + fun `returns null if both queues empty`() { + every { highPriorityQueue.poll() } returns null + every { standardPriorityQueue.poll() } returns null + assertThat(queueService.poll(1, expectNone)).isFalse + verify { standardPriorityQueue.poll() } + } + } + + @Nested + inner class PollStandardPriorityQueue { + + @BeforeEach + fun setUp() { + every { encoreProperties.pollHigherPrio } returns false + } + + @AfterEach + fun tearDown() { + verify { standardPriorityQueue.poll() } + verify { highPriorityQueue wasNot Called } + verify { lowPriorityQueue wasNot Called } + } + + @Test + fun `returns item from queue`() { + assertThat(queueService.poll(1, expectStandardPrio)).isTrue + } + + @Test + fun `empty queue`() { + every { standardPriorityQueue.poll() } returns null + assertThat(queueService.poll(1, expectNone)).isFalse + } + + @Test + fun `job not synced yet is retried`() { + every { repository.findByIdOrNull(UUID.fromString(queueItemStandardPrio.id)) } returns null andThen standardPrioJob + assertThat(queueService.poll(1, expectStandardPrio)).isTrue + verify(exactly = 2) { repository.findByIdOrNull(UUID.fromString(queueItemStandardPrio.id)) } + } + + @Test + fun `non-existing job throws`() { + every { repository.findByIdOrNull(UUID.fromString(queueItemStandardPrio.id)) } returns null + assertThatThrownBy { queueService.poll(1, expectStandardPrio) } + .hasMessageEndingWith("does not exist") + verify(exactly = 2) { repository.findByIdOrNull(UUID.fromString(queueItemStandardPrio.id)) } + } + + @Test + fun `reenqueue on interrupt`() { + every { standardPriorityQueue.offer(any(), any(), any()) } returns true + every { repository.save(any()) } answers { firstArg() } + queueService.poll(1) { _, _ -> throw InterruptedException("shut down") } + verify { standardPriorityQueue.offer(queueItemStandardPrio, 5, TimeUnit.SECONDS) } + verify { standardPrioJob.status = Status.QUEUED } + verify { repository.save(standardPrioJob) } + } + + @Test + fun `reenqueue on interrupt fails`() { + every { standardPriorityQueue.offer(any(), any(), any()) } returns false + every { repository.save(any()) } answers { firstArg() } + queueService.poll(1) { _, _ -> throw InterruptedException("shut down") } + verify { standardPriorityQueue.offer(queueItemStandardPrio, 5, TimeUnit.SECONDS) } + verify { standardPrioJob.status = Status.FAILED } + verify { repository.save(standardPrioJob) } + } + } + + @Nested + inner class PollHighOrLowPriorityQueue { + + @AfterEach + fun tearDown() { + verify { highPriorityQueue.poll() } + } + + @Test + fun `returns item from high priority queue if any present`() { + assertThat(queueService.poll(2, expectHighPrio)).isTrue + verify { standardPriorityQueue wasNot Called } + verify { lowPriorityQueue wasNot Called } + } + + @Test + fun `returns items from low priority queue if present and high priority queue is empty`() { + every { highPriorityQueue.poll() } returns null + every { standardPriorityQueue.poll() } returns null + assertThat(queueService.poll(2, expectLowPrio)).isTrue + verify { standardPriorityQueue.poll() } + verify { lowPriorityQueue.poll() } + } + + @Test + fun `returns null if all queues are empty`() { + every { highPriorityQueue.poll() } returns null + every { standardPriorityQueue.poll() } returns null + every { lowPriorityQueue.poll() } returns null + assertThat(queueService.poll(2, expectNone)).isFalse + verify { standardPriorityQueue.poll() } + verify { lowPriorityQueue.poll() } + } + } + + @Nested + inner class Enqueue { + + @Test + fun `low priority job is enqueued on low priority queue`() { + val job = defaultEncoreJob(10) + every { lowPriorityQueue.offer(any(), any(), any()) } returns true + queueService.enqueue(job) + verify { lowPriorityQueue.offer(expectedQueueItem(job), 5, TimeUnit.SECONDS) } + verify(exactly = 0) { standardPriorityQueue.offer(any(), any(), any()) } + verify(exactly = 0) { highPriorityQueue.offer(any(), any(), any()) } + } + + @Test + fun `standard priority job is enqueued on standard priority queue`() { + val job = defaultEncoreJob(55) + every { standardPriorityQueue.offer(any(), any(), any()) } returns true + queueService.enqueue(job) + verify { standardPriorityQueue.offer(expectedQueueItem(job), 5, TimeUnit.SECONDS) } + verify(exactly = 0) { highPriorityQueue.offer(any(), any(), any()) } + verify(exactly = 0) { lowPriorityQueue.offer(any(), any(), any()) } + } + + @Test + fun `high priority job is enqueued on high priority queue`() { + val job = defaultEncoreJob(priority = 90) + every { highPriorityQueue.offer(any(), any(), any()) } returns true + queueService.enqueue(job) + verify { highPriorityQueue.offer(expectedQueueItem(job), 5, TimeUnit.SECONDS) } + verify(exactly = 0) { standardPriorityQueue.offer(any(), any(), any()) } + verify(exactly = 0) { lowPriorityQueue.offer(any(), any(), any()) } + } + + private fun expectedQueueItem(job: EncoreJob) = + QueueItem( + id = job.id.toString(), + priority = job.priority, + created = job.createdDate.toLocalDateTime() + ) + } +} diff --git a/src/test/kotlin/se/svt/oss/encore/service/queue/QueueUtilTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/service/queue/QueueUtilTest.kt similarity index 100% rename from src/test/kotlin/se/svt/oss/encore/service/queue/QueueUtilTest.kt rename to encore-common/src/test/kotlin/se/svt/oss/encore/service/queue/QueueUtilTest.kt diff --git a/src/test/resources/application-test-local.yml b/encore-common/src/test/resources/application-test-local.yml similarity index 83% rename from src/test/resources/application-test-local.yml rename to encore-common/src/test/resources/application-test-local.yml index 55811db8..2f384c81 100644 --- a/src/test/resources/application-test-local.yml +++ b/encore-common/src/test/resources/application-test-local.yml @@ -1,6 +1,7 @@ spring: - redis: - port: ${embedded-redis.port} + data: + redis: + port: ${embedded-redis.port} main: allow-bean-definition-overriding: true @@ -8,9 +9,6 @@ logging: level: se.svt: debug -server: - port: ${random-port.server} - service: name: encore-test diff --git a/src/test/resources/application-test.yml b/encore-common/src/test/resources/application-test.yml similarity index 91% rename from src/test/resources/application-test.yml rename to encore-common/src/test/resources/application-test.yml index da270833..35106ec0 100644 --- a/src/test/resources/application-test.yml +++ b/encore-common/src/test/resources/application-test.yml @@ -1,6 +1,7 @@ spring: - redis: - port: ${embedded-redis.port} + data: + redis: + port: ${embedded-redis.port} main: allow-bean-definition-overriding: true @@ -25,6 +26,7 @@ encore-settings: local-temporary-encode: false poll-initial-delay: 1s poll-delaly: 1s + shared-work-dir: ${java.io.tmpdir}/encore-shared encoding: default-channel-layouts: 3: "3.0" diff --git a/src/test/resources/input/multiple-audio-file.json b/encore-common/src/test/resources/input/multiple-audio-file.json similarity index 100% rename from src/test/resources/input/multiple-audio-file.json rename to encore-common/src/test/resources/input/multiple-audio-file.json diff --git a/src/test/resources/input/multiple-video-file.json b/encore-common/src/test/resources/input/multiple-video-file.json similarity index 100% rename from src/test/resources/input/multiple-video-file.json rename to encore-common/src/test/resources/input/multiple-video-file.json diff --git a/src/test/resources/input/multiple_audio.mp4 b/encore-common/src/test/resources/input/multiple_audio.mp4 similarity index 100% rename from src/test/resources/input/multiple_audio.mp4 rename to encore-common/src/test/resources/input/multiple_audio.mp4 diff --git a/src/test/resources/input/multiple_video.mp4 b/encore-common/src/test/resources/input/multiple_video.mp4 similarity index 100% rename from src/test/resources/input/multiple_video.mp4 rename to encore-common/src/test/resources/input/multiple_video.mp4 diff --git a/src/test/resources/input/portrait-video-file.json b/encore-common/src/test/resources/input/portrait-video-file.json similarity index 100% rename from src/test/resources/input/portrait-video-file.json rename to encore-common/src/test/resources/input/portrait-video-file.json diff --git a/src/test/resources/input/test.mp4 b/encore-common/src/test/resources/input/test.mp4 similarity index 100% rename from src/test/resources/input/test.mp4 rename to encore-common/src/test/resources/input/test.mp4 diff --git a/src/test/resources/input/test_stereo.mp4 b/encore-common/src/test/resources/input/test_stereo.mp4 similarity index 100% rename from src/test/resources/input/test_stereo.mp4 rename to encore-common/src/test/resources/input/test_stereo.mp4 diff --git a/src/test/resources/input/video-file-long.json b/encore-common/src/test/resources/input/video-file-long.json similarity index 100% rename from src/test/resources/input/video-file-long.json rename to encore-common/src/test/resources/input/video-file-long.json diff --git a/src/test/resources/input/video-file.json b/encore-common/src/test/resources/input/video-file.json similarity index 100% rename from src/test/resources/input/video-file.json rename to encore-common/src/test/resources/input/video-file.json diff --git a/src/test/resources/profile/archive.yml b/encore-common/src/test/resources/profile/archive.yml similarity index 100% rename from src/test/resources/profile/archive.yml rename to encore-common/src/test/resources/profile/archive.yml diff --git a/src/test/resources/profile/audio-streams.yml b/encore-common/src/test/resources/profile/audio-streams.yml similarity index 100% rename from src/test/resources/profile/audio-streams.yml rename to encore-common/src/test/resources/profile/audio-streams.yml diff --git a/src/test/resources/profile/dpb_size_failed.yml b/encore-common/src/test/resources/profile/dpb_size_failed.yml similarity index 100% rename from src/test/resources/profile/dpb_size_failed.yml rename to encore-common/src/test/resources/profile/dpb_size_failed.yml diff --git a/src/test/resources/profile/multiple_inputs.yml b/encore-common/src/test/resources/profile/multiple_inputs.yml similarity index 100% rename from src/test/resources/profile/multiple_inputs.yml rename to encore-common/src/test/resources/profile/multiple_inputs.yml diff --git a/src/test/resources/profile/profiles.yml b/encore-common/src/test/resources/profile/profiles.yml similarity index 100% rename from src/test/resources/profile/profiles.yml rename to encore-common/src/test/resources/profile/profiles.yml diff --git a/src/test/resources/profile/program-x265.yml b/encore-common/src/test/resources/profile/program-x265.yml similarity index 100% rename from src/test/resources/profile/program-x265.yml rename to encore-common/src/test/resources/profile/program-x265.yml diff --git a/src/test/resources/profile/program.yml b/encore-common/src/test/resources/profile/program.yml similarity index 100% rename from src/test/resources/profile/program.yml rename to encore-common/src/test/resources/profile/program.yml diff --git a/src/test/resources/profile/test_profile_invalid.yml b/encore-common/src/test/resources/profile/test_profile_invalid.yml similarity index 100% rename from src/test/resources/profile/test_profile_invalid.yml rename to encore-common/src/test/resources/profile/test_profile_invalid.yml diff --git a/encore-web/Dockerfile b/encore-web/Dockerfile new file mode 100644 index 00000000..31c104a9 --- /dev/null +++ b/encore-web/Dockerfile @@ -0,0 +1,12 @@ +ARG DOCKER_BASE_IMAGE +FROM ${DOCKER_BASE_IMAGE} + +LABEL org.opencontainers.image.url="https://github.com/svt/encore" +LABEL org.opencontainers.image.source="https://github.com/svt/encore" + +# produced by gradle target nativeCompile +COPY build/native/nativeCompile/encore-web /app/ + +WORKDIR /app + +CMD ["/app/encore-web"] diff --git a/Dockerfile b/encore-web/Dockerfile-jar similarity index 67% rename from Dockerfile rename to encore-web/Dockerfile-jar index 7524a3cb..8dcc92b8 100644 --- a/Dockerfile +++ b/encore-web/Dockerfile-jar @@ -4,8 +4,8 @@ FROM ${DOCKER_BASE_IMAGE} LABEL org.opencontainers.image.url="https://github.com/svt/encore" LABEL org.opencontainers.image.source="https://github.com/svt/encore" -COPY build/libs/encore*.jar /app/encore.jar +COPY build/libs/encore-web-*-boot.jar /app/encore-web.jar WORKDIR /app -CMD ["java", "-jar", "encore.jar"] +CMD ["java", "-jar", "encore-web.jar"] diff --git a/encore-web/build.gradle.kts b/encore-web/build.gradle.kts new file mode 100644 index 00000000..1451249c --- /dev/null +++ b/encore-web/build.gradle.kts @@ -0,0 +1,15 @@ +plugins { + id("encore.kotlin-conventions") + id("encore.spring-boot-app-conventions") +} + +dependencies { + implementation(project(":encore-common")) + implementation("org.springframework.boot:spring-boot-starter-data-rest") + implementation("org.springdoc:springdoc-openapi-starter-webmvc-ui:2.2.0") + implementation("org.springframework.boot:spring-boot-starter-security") + implementation("org.springframework.boot:spring-boot-starter-actuator") + + testImplementation("com.ninja-squad:springmockk:4.0.2") + testImplementation("se.svt.oss:junit5-redis-extension:3.0.0") +} \ No newline at end of file diff --git a/src/main/kotlin/se/svt/oss/encore/EncoreApplication.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/EncoreApplication.kt similarity index 85% rename from src/main/kotlin/se/svt/oss/encore/EncoreApplication.kt rename to encore-web/src/main/kotlin/se/svt/oss/encore/EncoreApplication.kt index 63b469ec..5dc5d57a 100644 --- a/src/main/kotlin/se/svt/oss/encore/EncoreApplication.kt +++ b/encore-web/src/main/kotlin/se/svt/oss/encore/EncoreApplication.kt @@ -9,14 +9,14 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration import org.springframework.boot.context.properties.EnableConfigurationProperties -import org.springframework.retry.annotation.EnableRetry +import org.springframework.context.annotation.ImportRuntimeHints import se.svt.oss.encore.config.EncoreProperties -@EnableRetry @EnableConfigurationProperties(EncoreProperties::class) @SpringBootApplication( exclude = [SecurityAutoConfiguration::class, ManagementWebSecurityAutoConfiguration::class] ) +@ImportRuntimeHints(EncoreRuntimeHints::class, EncoreWebRuntimeHints::class) class EncoreApplication fun main(args: Array) { diff --git a/encore-web/src/main/kotlin/se/svt/oss/encore/EncoreWebRuntimeHints.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/EncoreWebRuntimeHints.kt new file mode 100644 index 00000000..d6c82481 --- /dev/null +++ b/encore-web/src/main/kotlin/se/svt/oss/encore/EncoreWebRuntimeHints.kt @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore + +import org.springframework.aot.hint.MemberCategory +import org.springframework.aot.hint.RuntimeHints +import org.springframework.aot.hint.RuntimeHintsRegistrar +import se.svt.oss.encore.handlers.EncoreJobHandler + +class EncoreWebRuntimeHints : RuntimeHintsRegistrar { + override fun registerHints(hints: RuntimeHints, classLoader: ClassLoader?) { + hints.reflection() + .registerType( + EncoreJobHandler::class.java, + MemberCategory.INVOKE_PUBLIC_METHODS + ) + } +} diff --git a/src/main/kotlin/se/svt/oss/encore/OpenAPIConfiguration.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/OpenAPIConfiguration.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/OpenAPIConfiguration.kt rename to encore-web/src/main/kotlin/se/svt/oss/encore/OpenAPIConfiguration.kt diff --git a/src/main/kotlin/se/svt/oss/encore/RepositoryConfiguration.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/RepositoryConfiguration.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/RepositoryConfiguration.kt rename to encore-web/src/main/kotlin/se/svt/oss/encore/RepositoryConfiguration.kt diff --git a/src/main/kotlin/se/svt/oss/encore/SchedulingConfiguration.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/SchedulingConfiguration.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/SchedulingConfiguration.kt rename to encore-web/src/main/kotlin/se/svt/oss/encore/SchedulingConfiguration.kt diff --git a/src/main/kotlin/se/svt/oss/encore/SecurityConfiguration.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/SecurityConfiguration.kt similarity index 51% rename from src/main/kotlin/se/svt/oss/encore/SecurityConfiguration.kt rename to encore-web/src/main/kotlin/se/svt/oss/encore/SecurityConfiguration.kt index da0e2ec6..f79845b2 100644 --- a/src/main/kotlin/se/svt/oss/encore/SecurityConfiguration.kt +++ b/encore-web/src/main/kotlin/se/svt/oss/encore/SecurityConfiguration.kt @@ -1,56 +1,64 @@ // SPDX-FileCopyrightText: 2020 Sveriges Television AB // // SPDX-License-Identifier: EUPL-1.2 + package se.svt.oss.encore +import org.springframework.boot.actuate.autoconfigure.endpoint.web.WebEndpointProperties +import org.springframework.boot.actuate.autoconfigure.security.servlet.EndpointRequest +import org.springframework.boot.actuate.health.HealthEndpoint import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.http.HttpMethod -import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder import org.springframework.security.config.annotation.web.builders.HttpSecurity import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter -import org.springframework.security.config.web.servlet.invoke +import org.springframework.security.config.annotation.web.invoke import org.springframework.security.core.authority.SimpleGrantedAuthority -import org.springframework.security.crypto.factory.PasswordEncoderFactories +import org.springframework.security.core.userdetails.User +import org.springframework.security.core.userdetails.UserDetailsService +import org.springframework.security.provisioning.InMemoryUserDetailsManager +import org.springframework.security.web.SecurityFilterChain import se.svt.oss.encore.config.EncoreProperties private const val ROLE_USER = "USER" private const val ROLE_ADMIN = "ADMIN" -private val ROLE_ANON = "ANON" +private const val ROLE_ANON = "ANON" -@ConditionalOnProperty(prefix = "encore-settings.security", name = ["enabled"]) -@EnableWebSecurity @Configuration -class SecurityConfiguration( - private val encoreProperties: EncoreProperties -) : WebSecurityConfigurerAdapter() { +@EnableWebSecurity +@ConditionalOnProperty(prefix = "encore-settings.security", name = ["enabled"]) +class SecurityConfiguration(private val encoreProperties: EncoreProperties) { - override fun configure(auth: AuthenticationManagerBuilder) { - val encoder = PasswordEncoderFactories.createDelegatingPasswordEncoder() - auth.inMemoryAuthentication() - .withUser("user") - .password(encoder.encode(encoreProperties.security.userPassword)).roles(ROLE_USER) - .and() - .withUser("admin").password(encoder.encode(encoreProperties.security.adminPassword)).roles( - ROLE_USER, - ROLE_ADMIN - ) + @Bean + fun users(): UserDetailsService { + val user = User.builder() + .username("user") + .password(encoreProperties.security.userPassword) + .roles(ROLE_USER) + .build() + val admin = User.builder() + .username("admin") + .password(encoreProperties.security.adminPassword) + .roles(ROLE_USER, ROLE_ADMIN) + .build() + return InMemoryUserDetailsManager(user, admin) } - override fun configure(http: HttpSecurity) { + @Bean + fun filterChain(http: HttpSecurity, webEndPointProperties: WebEndpointProperties): SecurityFilterChain { http { headers { httpStrictTransportSecurity { } } authorizeRequests { - authorize(HttpMethod.GET, "/health", anonymous) + authorize(EndpointRequest.to(HealthEndpoint::class.java), permitAll) authorize(HttpMethod.GET, "/**", hasRole(ROLE_USER)) - authorize(HttpMethod.PATCH, "/**", hasRole(ROLE_ADMIN)) authorize(HttpMethod.PUT, "/**", hasRole(ROLE_ADMIN)) authorize(HttpMethod.DELETE, "/**", hasRole(ROLE_ADMIN)) authorize(HttpMethod.POST, "/**", hasRole(ROLE_ADMIN)) authorize(HttpMethod.PATCH, "/**", hasRole(ROLE_ADMIN)) authorize(HttpMethod.OPTIONS, "/**", hasRole(ROLE_ADMIN)) authorize(HttpMethod.TRACE, "/**", hasRole(ROLE_ADMIN)) + authorize(anyRequest, denyAll) } httpBasic { } csrf { disable() } @@ -58,5 +66,6 @@ class SecurityConfiguration( authorities = listOf(SimpleGrantedAuthority(ROLE_ANON)) } } + return http.build() } } diff --git a/src/main/kotlin/se/svt/oss/encore/controller/EncoreController.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/controller/EncoreController.kt similarity index 98% rename from src/main/kotlin/se/svt/oss/encore/controller/EncoreController.kt rename to encore-web/src/main/kotlin/se/svt/oss/encore/controller/EncoreController.kt index 1eca689d..7abee87c 100644 --- a/src/main/kotlin/se/svt/oss/encore/controller/EncoreController.kt +++ b/encore-web/src/main/kotlin/se/svt/oss/encore/controller/EncoreController.kt @@ -26,7 +26,7 @@ import java.util.UUID class EncoreController( private val repository: EncoreJobRepository, private val redissonClient: RedissonClient, - private val queueService: QueueService, + private val queueService: QueueService ) { @Operation(summary = "Get Queues", description = "Returns a list of queues (QueueItems)", tags = ["queue"]) diff --git a/src/main/kotlin/se/svt/oss/encore/handlers/EncoreJobHandler.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/handlers/EncoreJobHandler.kt similarity index 100% rename from src/main/kotlin/se/svt/oss/encore/handlers/EncoreJobHandler.kt rename to encore-web/src/main/kotlin/se/svt/oss/encore/handlers/EncoreJobHandler.kt diff --git a/encore-web/src/main/kotlin/se/svt/oss/encore/poll/JobPoller.kt b/encore-web/src/main/kotlin/se/svt/oss/encore/poll/JobPoller.kt new file mode 100644 index 00000000..da737241 --- /dev/null +++ b/encore-web/src/main/kotlin/se/svt/oss/encore/poll/JobPoller.kt @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.poll + +import jakarta.annotation.PostConstruct +import jakarta.annotation.PreDestroy +import mu.KotlinLogging +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.stereotype.Service +import se.svt.oss.encore.config.EncoreProperties +import se.svt.oss.encore.service.EncoreService +import se.svt.oss.encore.service.queue.QueueService +import java.time.Instant +import java.util.concurrent.ScheduledFuture + +@Service +class JobPoller( + private val queueService: QueueService, + private val encoreService: EncoreService, + private val scheduler: ThreadPoolTaskScheduler, + private val encoreProperties: EncoreProperties +) { + + private val log = KotlinLogging.logger {} + private var scheduledTasks = emptyList>() + + @PostConstruct + fun init() { + if (encoreProperties.pollDisabled) { + return + } + val pollQueue = encoreProperties.pollQueue + scheduledTasks = if (pollQueue != null) { + listOf(scheduledFuture(pollQueue)) + } else { + (0 until encoreProperties.concurrency).map { queueNo -> + scheduledFuture(queueNo) + } + } + } + + private fun scheduledFuture(queueNo: Int): ScheduledFuture<*> = + scheduler.scheduleWithFixedDelay( + { + try { + queueService.poll(queueNo, encoreService::encode) + } catch (e: Throwable) { + log.error(e) { "Error polling queue $queueNo!" } + } + }, + Instant.now().plus(encoreProperties.pollInitialDelay), + encoreProperties.pollDelay + ) + + @PreDestroy + fun destroy() { + scheduledTasks.forEach { it.cancel(false) } + } +} diff --git a/encore-web/src/main/resources/application.yml b/encore-web/src/main/resources/application.yml new file mode 100644 index 00000000..48fe2a04 --- /dev/null +++ b/encore-web/src/main/resources/application.yml @@ -0,0 +1,20 @@ +spring: + application: + name: encore + banner: + location: classpath:asciilogo.txt + cloud: + config: + import-check: + enabled: false +logging: + config: classpath:logback-json.xml + +springdoc: + paths-to-exclude: /profile/encoreJobs,/profile + swagger-ui: + operations-sorter: alpha + tags-sorter: alpha + disable-swagger-default-url: true +server: + forward-headers-strategy: framework \ No newline at end of file diff --git a/src/main/resources/asciilogo.txt b/encore-web/src/main/resources/asciilogo.txt similarity index 100% rename from src/main/resources/asciilogo.txt rename to encore-web/src/main/resources/asciilogo.txt diff --git a/encore-web/src/main/resources/logback-json.xml b/encore-web/src/main/resources/logback-json.xml new file mode 100644 index 00000000..885d39f1 --- /dev/null +++ b/encore-web/src/main/resources/logback-json.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/encore-web/src/test/kotlin/se/svt/oss/encore/EncoreEndpointAccessIntegrationTest.kt b/encore-web/src/test/kotlin/se/svt/oss/encore/EncoreEndpointAccessIntegrationTest.kt new file mode 100644 index 00000000..7edc9966 --- /dev/null +++ b/encore-web/src/test/kotlin/se/svt/oss/encore/EncoreEndpointAccessIntegrationTest.kt @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore + +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.NoSuchBeanDefinitionException +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.context.assertj.AssertableApplicationContext +import org.springframework.boot.test.context.runner.ApplicationContextRunner +import org.springframework.http.MediaType +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.web.reactive.server.WebTestClient +import se.svt.oss.encore.config.EncoreProperties +import se.svt.oss.encore.model.EncoreJob +import se.svt.oss.junit5.redis.EmbeddedRedisExtension + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = ["encore-settings.security.enabled=true"] +) +@ActiveProfiles("test") +@ExtendWith(EmbeddedRedisExtension::class) +class EncoreEndpointAccessIntegrationTest { + + @Autowired + lateinit var webTestClient: WebTestClient + + @Test + fun `security configuration is not loaded in context when security disabled`() { + val contextRunner = ApplicationContextRunner() + contextRunner + .withBean(EncoreProperties::class.java) + .withPropertyValues("encore-settings.security.enabled=false") + .withUserConfiguration(SecurityConfiguration::class.java) + .run { context: AssertableApplicationContext -> + assertThrows { + context.getBean(SecurityConfiguration::class.java) + } + } + } + + @Nested + inner class User { + @Test + fun `User user is allowed GET`() { + webTestClient.get() + .uri("/encoreJobs") + .headers { it.setBasicAuth("user", "upw") } + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus() + .is2xxSuccessful + } + + @Test + fun `User user is Forbidden POST`() { + webTestClient.post() + .uri("/encoreJobs") + .headers { + it.setBasicAuth("user", "upw") + it.contentType = MediaType.APPLICATION_JSON + } + .bodyValue(EncoreJob(baseName = "TEST", profile = "program", outputFolder = "/test")) + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus() + .isForbidden + } + } + + @Nested + inner class Admin { + @Test + fun `Admin user is allowed GET`() { + webTestClient.get() + .uri("/encoreJobs") + .headers { it.setBasicAuth("admin", "apw") } + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus() + .is2xxSuccessful + } + + @Test + fun `Admin user is allowed POST`() { + webTestClient.post() + .uri("/encoreJobs") + .headers { + it.setBasicAuth("admin", "apw") + it.contentType = MediaType.APPLICATION_JSON + } + .bodyValue(EncoreJob(baseName = "TEST", profile = "program", outputFolder = "/test")) + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus() + .is2xxSuccessful + } + + @Test + fun `Admin user is authorized GET health with details`() { + webTestClient.get() + .uri("/actuator/health") + .headers { + it.setBasicAuth("admin", "apw") + } + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus().is2xxSuccessful + .expectBody() + .jsonPath("\$.status").isEqualTo("UP") + .jsonPath("\$.components..details").isNotEmpty + } + } + + @Nested + inner class Anonymous { + @Test + fun `Anonymous user is not authorized GET`() { + webTestClient.get() + .uri("/encoreJobs") + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus() + .isUnauthorized + } + + @Test + fun `Anonymous user is authorized GET health without details`() { + webTestClient.get() + .uri("/actuator/health") + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectBody() + .json("""{status: "UP", groups:["liveness","readiness"]}""", true) + } + + @Test + fun `Anonymous user is authorized GET health readiness without details`() { + webTestClient.get() + .uri("/actuator/health/readiness") + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectBody() + .json("""{status: "UP"}""", true) + } + + @Test + fun `Anonymous user is not authorized POST`() { + webTestClient.post() + .uri("/encoreJobs") + .headers { + it.contentType = MediaType.APPLICATION_JSON + } + .bodyValue(EncoreJob(baseName = "TEST", profile = "program", outputFolder = "/test")) + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus() + .isUnauthorized + } + } +} diff --git a/encore-web/src/test/kotlin/se/svt/oss/encore/EncoreWebRuntimeHintsTest.kt b/encore-web/src/test/kotlin/se/svt/oss/encore/EncoreWebRuntimeHintsTest.kt new file mode 100644 index 00000000..24417f04 --- /dev/null +++ b/encore-web/src/test/kotlin/se/svt/oss/encore/EncoreWebRuntimeHintsTest.kt @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore + +import org.junit.jupiter.api.Test +import org.springframework.aot.hint.RuntimeHints +import org.springframework.aot.hint.predicate.RuntimeHintsPredicates +import se.svt.oss.encore.Assertions.assertThat +import se.svt.oss.encore.handlers.EncoreJobHandler +import kotlin.reflect.jvm.javaMethod + +class EncoreWebRuntimeHintsTest { + @Test + fun shouldRegisterHints() { + val hints = RuntimeHints() + EncoreWebRuntimeHints().registerHints(hints, javaClass.classLoader) + assertThat(RuntimeHintsPredicates.reflection().onMethod(EncoreJobHandler::onAfterCreate.javaMethod!!)).accepts(hints) + } +} diff --git a/src/test/kotlin/se/svt/oss/encore/controller/EncoreControllerTest.kt b/encore-web/src/test/kotlin/se/svt/oss/encore/controller/EncoreControllerTest.kt similarity index 97% rename from src/test/kotlin/se/svt/oss/encore/controller/EncoreControllerTest.kt rename to encore-web/src/test/kotlin/se/svt/oss/encore/controller/EncoreControllerTest.kt index 0cf88c04..4fd1967a 100644 --- a/src/test/kotlin/se/svt/oss/encore/controller/EncoreControllerTest.kt +++ b/encore-web/src/test/kotlin/se/svt/oss/encore/controller/EncoreControllerTest.kt @@ -22,8 +22,8 @@ import org.springframework.http.MediaType import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.get import org.springframework.test.web.servlet.post -import se.svt.oss.encore.defaultEncoreJob import se.svt.oss.encore.model.CancelEvent +import se.svt.oss.encore.model.EncoreJob import se.svt.oss.encore.model.Status import se.svt.oss.encore.model.queue.QueueItem import se.svt.oss.encore.repository.EncoreJobRepository @@ -73,7 +73,7 @@ class EncoreControllerTest { @Nested inner class Cancel { - private val encoreJob = defaultEncoreJob() + private val encoreJob = EncoreJob(baseName = "TEST", outputFolder = "/test", profile = "test") private fun cancelAndAssertStatus(jobId: UUID, expectedStatus: Int) { mockMvc.post("/encoreJobs/$jobId/cancel") { diff --git a/src/test/kotlin/se/svt/oss/encore/handlers/EncoreJobHandlerTest.kt b/encore-web/src/test/kotlin/se/svt/oss/encore/handlers/EncoreJobHandlerTest.kt similarity index 82% rename from src/test/kotlin/se/svt/oss/encore/handlers/EncoreJobHandlerTest.kt rename to encore-web/src/test/kotlin/se/svt/oss/encore/handlers/EncoreJobHandlerTest.kt index 53feb449..5b08299b 100644 --- a/src/test/kotlin/se/svt/oss/encore/handlers/EncoreJobHandlerTest.kt +++ b/encore-web/src/test/kotlin/se/svt/oss/encore/handlers/EncoreJobHandlerTest.kt @@ -15,12 +15,10 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import se.svt.oss.encore.Assertions.assertThat -import se.svt.oss.encore.defaultVideoFile -import se.svt.oss.encore.defaultEncoreJob +import se.svt.oss.encore.model.EncoreJob import se.svt.oss.encore.model.Status import se.svt.oss.encore.repository.EncoreJobRepository import se.svt.oss.encore.service.queue.QueueService -import se.svt.oss.mediaanalyzer.MediaAnalyzer @ExtendWith(MockKExtension::class) class EncoreJobHandlerTest { @@ -31,19 +29,13 @@ class EncoreJobHandlerTest { @MockK private lateinit var repository: EncoreJobRepository - @MockK - private lateinit var mediaAnalyzer: MediaAnalyzer - @InjectMockKs private lateinit var encoreJobHandler: EncoreJobHandler - private val job = defaultEncoreJob() - - private val videoFile = defaultVideoFile + private val job = EncoreJob(baseName = "TEST", outputFolder = "/test", profile = "test") @BeforeEach fun setUp() { - every { mediaAnalyzer.analyze(any()) } returns videoFile every { repository.save(job) } returns job } diff --git a/encore-web/src/test/kotlin/se/svt/oss/encore/poll/JobPollerTest.kt b/encore-web/src/test/kotlin/se/svt/oss/encore/poll/JobPollerTest.kt new file mode 100644 index 00000000..5d156f36 --- /dev/null +++ b/encore-web/src/test/kotlin/se/svt/oss/encore/poll/JobPollerTest.kt @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.poll + +import io.mockk.Called +import io.mockk.Runs +import io.mockk.every +import io.mockk.impl.annotations.InjectMockKs +import io.mockk.impl.annotations.MockK +import io.mockk.junit5.MockKExtension +import io.mockk.just +import io.mockk.mockk +import io.mockk.verify +import io.mockk.verifySequence +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import se.svt.oss.encore.Assertions.assertThat +import se.svt.oss.encore.config.EncoreProperties +import se.svt.oss.encore.model.EncoreJob +import se.svt.oss.encore.model.queue.QueueItem +import se.svt.oss.encore.service.EncoreService +import se.svt.oss.encore.service.queue.QueueService +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ScheduledFuture + +@ExtendWith(MockKExtension::class) +class JobPollerTest { + + @MockK + private lateinit var queueService: QueueService + + @MockK + private lateinit var encoreProperties: EncoreProperties + + @MockK + private lateinit var encoreService: EncoreService + + @MockK + private lateinit var scheduler: ThreadPoolTaskScheduler + + @InjectMockKs + private lateinit var jobPoller: JobPoller + + private val encoreJob = EncoreJob(baseName = "TEST", outputFolder = "/test", profile = "test") + + private val queueItem = QueueItem(encoreJob.id.toString()) + + private val capturedRunnables = mutableListOf() + private val scheduledTasks = mutableListOf>() + + @BeforeEach + fun setUp() { + every { scheduler.scheduleWithFixedDelay(capture(capturedRunnables), any(), any()) } answers { + val scheduled = mockk>() + scheduledTasks.add(scheduled) + scheduled + } + every { encoreService.encode(any(), any()) } just Runs + every { queueService.poll(any(), captureLambda()) } answers { + lambda<(QueueItem, EncoreJob) -> Unit>().captured.invoke(queueItem, encoreJob) + true + } + every { encoreProperties.concurrency } returns 3 + every { encoreProperties.pollDelay } returns Duration.ofSeconds(1) + every { encoreProperties.pollInitialDelay } returns Duration.ofSeconds(10) + every { encoreProperties.pollQueue } returns null + every { encoreProperties.pollDisabled } returns false + } + + @Test + fun doesNothingWhenPollDisabled() { + every { encoreProperties.pollDisabled } returns true + jobPoller.init() + verify { scheduler wasNot Called } + verify { encoreService wasNot Called } + assertThat(capturedRunnables).isEmpty() + } + + @Test + fun testDestroy() { + jobPoller.init() + assertThat(capturedRunnables).hasSize(3) + assertThat(scheduledTasks).hasSize(3) + scheduledTasks.forEach { + every { it.cancel(false) } returns true + } + jobPoller.destroy() + scheduledTasks.forEach { + verify { it.cancel(false) } + } + } + + @ParameterizedTest + @ValueSource(ints = [0, 1, 2]) + fun pollAll(thread: Int) { + jobPoller.init() + assertThat(capturedRunnables).hasSize(3) + capturedRunnables[thread].run() + + verifySequence { + queueService.poll(thread, any()) + encoreService.encode(queueItem, encoreJob) + } + } + + @ParameterizedTest + @ValueSource(ints = [0, 1, 2]) + fun pollSpecific(queueNo: Int) { + every { encoreProperties.pollQueue } returns queueNo + jobPoller.init() + assertThat(capturedRunnables).hasSize(1) + capturedRunnables.first().run() + verifySequence { + queueService.poll(queueNo, any()) + encoreService.encode(queueItem, encoreJob) + } + } + + @ParameterizedTest + @ValueSource(ints = [0, 1, 2]) + fun `poll causes exception`(thread: Int) { + every { queueService.poll(thread, any()) } throws Exception("error") + jobPoller.init() + + capturedRunnables[thread].run() + + verifySequence { + queueService.poll(thread, any()) + } + } +} diff --git a/encore-web/src/test/resources/application-test.yml b/encore-web/src/test/resources/application-test.yml new file mode 100644 index 00000000..5ce3bffd --- /dev/null +++ b/encore-web/src/test/resources/application-test.yml @@ -0,0 +1,47 @@ +spring: + data: + redis: + port: ${embedded-redis.port} + main: + allow-bean-definition-overriding: true +management: + endpoint: + health: + show-details: when_authorized + roles: ADMIN + probes: + enabled: true + +logging: + level: + se.svt: debug + +encore-settings: + concurrency: 3 + local-temporary-encode: false + poll-initial-delay: 1s + poll-delaly: 1s + security: + user-password: '{bcrypt}$2a$10$5UQDlMEE6PDHNtr.pY/Jh.06Dq0BTkFtQyYNQint/R0KhC4muu4PO' # 'upw' encypted with spring boot cli + admin-password: '{bcrypt}$2a$10$Yp8gtLGnpyFtlyOrL6/ajOO/hPXOCKMf4IpCW41ptMUzAUpJmmGOC' # 'apw' encypted with spring boot cli + encoding: + default-channel-layouts: + 3: "3.0" + audio-mix-presets: + default: + default-pan: + "[5.1]": c0 = c0 | c1 = c1 | c2 = c2 | c3 = c3 | c4 = c4 | c5 = c5 + stereo: c0 = c0 + 0.707*c2 + 0.707*c4 | c1 = c1 + 0.707*c2 + 0.707*c5 + pan-mapping: + "[5.1]": + stereo: c0=1.0*c0+0.707*c2+0.707*c4|c1=1.0*c1+0.707*c2+0.707*c5 + de: + fallback-to-auto: false + pan-mapping: + "[5.1]": + stereo: c0<0.25*c0+1.5*c2+0.25*c4|c1<0.25*c1+1.5*c2+0.25*c5 + "[5.1(side)]": + stereo: c0<0.25*c0+1.5*c2+0.25*c4|c1<0.25*c1+1.5*c2+0.25*c5 + +profile: + location: classpath:profile/profiles.yml diff --git a/encore-web/src/test/resources/profile/archive.yml b/encore-web/src/test/resources/profile/archive.yml new file mode 100644 index 00000000..7a3d38b5 --- /dev/null +++ b/encore-web/src/test/resources/profile/archive.yml @@ -0,0 +1,15 @@ +name: archive +description: Archive format +encodes: + - type: VideoEncode + codec: dnxhd + height: 1080 + params: + b:v: 185M + pix_fmt: yuv422p10le + suffix: _DNxHD_185x + format: mxf + twoPass: false + audioEncode: + type: SimpleAudioEncode + codec: pcm_s24le diff --git a/encore-web/src/test/resources/profile/audio-streams.yml b/encore-web/src/test/resources/profile/audio-streams.yml new file mode 100644 index 00000000..fe3aa5aa --- /dev/null +++ b/encore-web/src/test/resources/profile/audio-streams.yml @@ -0,0 +1,20 @@ +name: audio-streams +description: Video file with multiple audio streams +encodes: + - type: X264Encode + suffix: '' + twoPass: false + format: mp4 + height: 720 + params: + preset: fast + pix_fmt: yuv420p + audioEncodes: + - type: AudioEncode + codec: ac3 + bitrate: 448k + channelLayout: '5.1' + optional: true + - type: AudioEncode + bitrate: 128k + channelLayout: 'stereo' \ No newline at end of file diff --git a/encore-web/src/test/resources/profile/dpb_size_failed.yml b/encore-web/src/test/resources/profile/dpb_size_failed.yml new file mode 100644 index 00000000..cbbe8855 --- /dev/null +++ b/encore-web/src/test/resources/profile/dpb_size_failed.yml @@ -0,0 +1,42 @@ +name: dpb_size_failed +description: Test profile, should fail +scaling: fast_bilinear +encodes: + - type: X264Encode + suffix: _x264_1400 + twoPass: true + width: -2 + height: 1080 + params: + b:v: 1400k + maxrate: 2100k + bufsize: 4800k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + level: 4.1 + profile:v: high + x264-params: + deblock: 1,1 + aq-mode: 1 + aq-strength: 0.6 + b-adapt: 2 + bframes: 8 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 70 + keyint: 192 + keyint_min: 96 + me: umh + merange: 40 + cabac: 1 + partitions: all + psy-rd: 0.4 + ref: 7 + scenecut: 40 + subme: 10 + trellis: 2 + weightp: 2 diff --git a/encore-web/src/test/resources/profile/multiple_inputs.yml b/encore-web/src/test/resources/profile/multiple_inputs.yml new file mode 100644 index 00000000..30a8d3d7 --- /dev/null +++ b/encore-web/src/test/resources/profile/multiple_inputs.yml @@ -0,0 +1,73 @@ +name: multiple-inputs +description: Test profile multiple inputs +scaling: bicubic +encodes: + - type: X264Encode + suffix: _x264_3100 + twoPass: true + params: + b:v: 3100k + maxrate: 4700k + bufsize: 6200k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: high + level: 4.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 128k + suffix: STEREO + + - type: AudioEncode + bitrate: 128k + suffix: _STEREO + + - type: AudioEncode + bitrate: 128k + suffix: _STEREO_DE + audioMixPreset: de + optional: true + + - type: AudioEncode + codec: ac3 + bitrate: 448k + suffix: _SURROUND + optional: true + channelLayout: '5.1(side)' + + - type: AudioEncode + bitrate: 128k + suffix: _STEREO_ALT + inputLabel: alt + + - type: ThumbnailMapEncode + cols: 6 + rows: 10 + + - type: ThumbnailEncode + + diff --git a/encore-web/src/test/resources/profile/profiles.yml b/encore-web/src/test/resources/profile/profiles.yml new file mode 100644 index 00000000..1e6174cf --- /dev/null +++ b/encore-web/src/test/resources/profile/profiles.yml @@ -0,0 +1,9 @@ +program: program.yml +multiple-inputs: multiple_inputs.yml +dpb_size_failed: dpb_size_failed.yml +program-x265: program-x265.yml +archive: archive.yml +audio-streams: audio-streams.yml +test-invalid: test_profile_invalid.yml +test-invalid-location: test_profile_invalid_location.yml +none: diff --git a/encore-web/src/test/resources/profile/program-x265.yml b/encore-web/src/test/resources/profile/program-x265.yml new file mode 100644 index 00000000..baf7ad6a --- /dev/null +++ b/encore-web/src/test/resources/profile/program-x265.yml @@ -0,0 +1,475 @@ +name: program-x265 +description: HEVC profile +scaling: bicubic +encodes: + - type: X265Encode + suffix: _x265_2600 + twoPass: true + height: 1080 + params: + b:v: 2600k + maxrate: 3900k + bufsize: 5200k + r: 25 + pix_fmt: yuv420p10le + profile:v: main10 + tag:v: hvc1 + force_key_frames: expr:not(mod(n,96)) + x265-params: + min-keyint: 96 + keyint: 96 + pmode: 1 + level-idc: 4.1 + ctu: 64 + min-cu-size: 8 + bframes: 6 + b-adapt: 2 + rc-lookahead: 25 + lookahead-slices: 1 + scenecut: 40 + ref: 4 + limit-refs: 1 + me: star + merange: 57 + subme: 3 + rect: 1 + amp: 0 + limit-modes: 0 + max-merge: 4 + early-skip: 0 + rskip: 1 + fast-intra: 0 + b-intra: 1 + sao: 1 + signhide: 1 + weightp: 1 + weightb: 1 + aq-mode: 2 + cutree: 1 + rd: 4 + rdoq-level: 2 + tu-intra-depth: 4 + tu-inter-depth: 4 + limit-tu: 0 + audioEncode: + type: AudioEncode + bitrate: 192k + suffix: STEREO + + - type: X265Encode + suffix: _x265_1598 + twoPass: true + height: 720 + params: + b:v: 1536847 + maxrate: 2305271 + bufsize: 3073694 + r: 25 + pix_fmt: yuv420p10le + profile:v: main10 + tag:v: hvc1 + force_key_frames: expr:not(mod(n,96)) + x265-params: + min-keyint: 96 + keyint: 96 + pmode: 1 + level-idc: 4.1 + ctu: 64 + min-cu-size: 8 + bframes: 4 + b-adapt: 2 + rc-lookahead: 20 + lookahead-slices: 1 + scenecut: 40 + ref: 4 + limit-refs: 1 + me: star + merange: 57 + subme: 3 + rect: 1 + amp: 0 + limit-modes: 0 + max-merge: 3 + early-skip: 0 + rskip: 1 + fast-intra: 0 + b-intra: 0 + sao: 1 + signhide: 1 + weightp: 1 + weightb: 0 + aq-mode: 2 + cutree: 1 + rd: 4 + rdoq-level: 2 + tu-intra-depth: 4 + tu-inter-depth: 4 + limit-tu: 0 + audioEncode: + type: AudioEncode + bitrate: 128k + suffix: STEREO + + - type: X265Encode + suffix: _x265_865 + twoPass: true + height: 540 + params: + b:v: 865857 + maxrate: 1298786 + bufsize: 1731714 + r: 25 + pix_fmt: yuv420p10le + profile:v: main10 + tag:v: hvc1 + force_key_frames: expr:not(mod(n,96)) + x265-params: + min-keyint: 96 + keyint: 96 + pmode: 1 + ctu: 64 + level-idc: 4.1 + min-cu-size: 8 + bframes: 4 + b-adapt: 2 + rc-lookahead: 20 + lookahead-slices: 1 + scenecut: 40 + ref: 4 + limit-refs: 1 + me: star + merange: 57 + subme: 3 + rect: 1 + amp: 0 + limit-modes: 0 + max-merge: 3 + early-skip: 0 + rskip: 1 + fast-intra: 0 + b-intra: 0 + sao: 1 + signhide: 1 + weightp: 1 + weightb: 0 + aq-mode: 2 + cutree: 1 + rd: 4 + rdoq-level: 2 + tu-intra-depth: 3 + tu-inter-depth: 3 + limit-tu: 0 + audioEncode: + type: AudioEncode + bitrate: 128k + suffix: STEREO + samplerate: 48000 + channels: 2 + + - type: X265Encode + suffix: _x265_474 + twoPass: true + height: 360 + params: + b:v: 695711 + maxrate: 1043567 + bufsize: 1391422 + r: 25 + pix_fmt: yuv420p + profile:v: main + tag:v: hvc1 + force_key_frames: expr:not(mod(n,96)) + x265-params: + min-keyint: 96 + keyint: 96 + level-idc: 4.1 + pmode: 1 + ctu: 64 + min-cu-size: 8 + bframes: 4 + b-adapt: 2 + rc-lookahead: 20 + lookahead-slices: 1 + scenecut: 40 + ref: 4 + limit-refs: 1 + me: star + merange: 57 + subme: 3 + rect: 1 + amp: 0 + limit-modes: 0 + max-merge: 3 + early-skip: 0 + rskip: 1 + fast-intra: 0 + b-intra: 0 + sao: 1 + signhide: 1 + weightp: 1 + weightb: 0 + aq-mode: 2 + cutree: 1 + rd: 4 + rdoq-level: 2 + tu-intra-depth: 3 + tu-inter-depth: 3 + limit-tu: 0 + audioEncode: + type: AudioEncode + bitrate: 128k + suffix: STEREO + + - type: X264Encode + suffix: _x264_243 + height: 234 + twoPass: true + params: + b:v: 324051 + maxrate: 486077 + bufsize: 648102 + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: baseline + level: 3.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + chroma-qp-offset: -2 + direct: auto + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 0 + 8x8dct: 0 + ref: 3 + scenecut: 40 + subme: 9 + trellis: 2 + audioEncode: + type: AudioEncode + bitrate: 96k + suffix: STEREO + samplerate: 48000 + channels: 2 + + - type: X264Encode + suffix: _x264_2500 + twoPass: true + height: 1080 + params: + b:v: 3100k + maxrate: 4700k + bufsize: 6200k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: high + level: 4.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 128k + suffix: STEREO + + - type: X264Encode + suffix: _x264_1300 + height: 720 + twoPass: true + params: + b:v: 2069k + maxrate: 3104k + bufsize: 4138k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: main + level: 3.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 128k + suffix: STEREO + + - type: X264Encode + suffix: _x264_870 + twoPass: true + height: 540 + params: + b:v: 1312k + maxrate: 1968k + bufsize: 2524k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + level: 3.1 + profile:v: main + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 96k + suffix: STEREO + samplerate: 48000 + channels: 2 + + - type: X264Encode + suffix: _x264_470 + twoPass: true + height: 360 + params: + b:v: 806121 + maxrate: 1209182 + bufsize: 1612242 + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: main + level: 3.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 96k + suffix: STEREO + samplerate: 48000 + channels: 2 + + - type: X264Encode + suffix: _x264_240 + twoPass: true + height: 234 + params: + b:v: 324051 + maxrate: 486077 + bufsize: 648102 + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: baseline + level: 3.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + chroma-qp-offset: -2 + direct: auto + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 0 + 8x8dct: 0 + ref: 3 + scenecut: 40 + subme: 9 + trellis: 2 + audioEncode: + type: AudioEncode + bitrate: 96k + suffix: STEREO + + - type: AudioEncode + bitrate: 128k + suffix: _STEREO + + - type: AudioEncode + codec: ac3 + bitrate: 448k + suffix: _SURROUND + channelLayout: '5.1' + + - type: ThumbnailMapEncode + + - type: ThumbnailEncode diff --git a/encore-web/src/test/resources/profile/program.yml b/encore-web/src/test/resources/profile/program.yml new file mode 100644 index 00000000..42bbf5f9 --- /dev/null +++ b/encore-web/src/test/resources/profile/program.yml @@ -0,0 +1,225 @@ +name: program +description: Program profile +scaling: bicubic +encodes: + - type: X264Encode + suffix: _x264_3100 + twoPass: true + height: 1080 + params: + b:v: 3100k + maxrate: 4700k + bufsize: 6200k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: high + level: 4.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 128k + suffix: STEREO + + - type: X264Encode + suffix: _x264_2069 + twoPass: true + height: 720 + params: + b:v: 2069k + maxrate: 3104k + bufsize: 4138k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: main + level: 3.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 128k + suffix: STEREO + + - type: X264Encode + suffix: _x264_1312 + twoPass: true + height: 540 + params: + b:v: 1312k + maxrate: 1968k + bufsize: 2524k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + level: 3.1 + profile:v: main + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 96k + suffix: STEREO + + - type: X264Encode + suffix: _x264_806 + twoPass: true + height: 360 + params: + b:v: 806121 + maxrate: 1209182 + bufsize: 1612242 + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: main + level: 3.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + b-adapt: 2 + bframes: 6 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 60 + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 1 + partitions: all + ref: 4 + scenecut: 40 + subme: 9 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + bitrate: 96k + suffix: STEREO + + - type: X264Encode + suffix: _x264_324 + twoPass: true + height: 234 + params: + b:v: 324051 + maxrate: 486077 + bufsize: 648102 + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + profile:v: baseline + level: 3.1 + x264-params: + deblock: 0,0 + aq-mode: 1 + aq-strength: 1.0 + chroma-qp-offset: -2 + direct: auto + keyint: 192 + keyint_min: 96 + me: hex + merange: 16 + cabac: 0 + 8x8dct: 0 + ref: 3 + scenecut: 40 + subme: 9 + trellis: 2 + audioEncode: + type: AudioEncode + bitrate: 96k + suffix: STEREO + + - type: AudioEncode + bitrate: 128k + suffix: _STEREO + + - type: AudioEncode + bitrate: 128k + suffix: _STEREO_DE + audioMixPreset: de + optional: true + + - type: AudioEncode + codec: ac3 + bitrate: 448k + suffix: _SURROUND + optional: true + channelLayout: '5.1' + + - type: ThumbnailMapEncode + + - type: ThumbnailEncode + + diff --git a/encore-web/src/test/resources/profile/test_profile_invalid.yml b/encore-web/src/test/resources/profile/test_profile_invalid.yml new file mode 100644 index 00000000..1a2d7616 --- /dev/null +++ b/encore-web/src/test/resources/profile/test_profile_invalid.yml @@ -0,0 +1,58 @@ +name: animerat +description: Animated profile, optimized for animated content +scaling: lanczos +encodes: + - type: X264Encode + suffixBroken: _x264_1400 + twoPass: true + width: -2 + height: 1080 + params: + b:v: 1400k + maxrate: 2100k + bufsize: 4800k + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) + level: 4.1 + profile:v: high + x264-params: + deblock: 1,1 + aq-mode: 1 + aq-strength: 0.6 + b-adapt: 2 + bframes: 8 + b-bias: 0 + b-pyramid: 2 + chroma-qp-offset: -2 + direct: auto + rc-lookahead: 70 + keyint: 192 + keyint_min: 96 + me: umh + merange: 40 + cabac: 1 + partitions: all + psy-rd: 0.4 + ref: 4 + scenecut: 40 + subme: 10 + trellis: 2 + weightp: 2 + audioEncode: + type: AudioEncode + codec: aac + channels: 2 + bitrate: 128k + suffix: STEREO + + - type: ThumbnailEncode + + - type: ThumbnailMapEncode + + - type: AudioEncode + codec: aac + channels: 2 + bitrate: 128k + suffix: _STEREO diff --git a/encore-worker/Dockerfile b/encore-worker/Dockerfile new file mode 100644 index 00000000..52698e39 --- /dev/null +++ b/encore-worker/Dockerfile @@ -0,0 +1,12 @@ +ARG DOCKER_BASE_IMAGE +FROM ${DOCKER_BASE_IMAGE} + +LABEL org.opencontainers.image.url="https://github.com/svt/encore" +LABEL org.opencontainers.image.source="https://github.com/svt/encore" + +# produced by gradle target nativeCompile +COPY build/native/nativeCompile/encore-worker /app/ + +WORKDIR /app + +CMD ["/app/encore-worker"] diff --git a/encore-worker/build.gradle.kts b/encore-worker/build.gradle.kts new file mode 100644 index 00000000..84318877 --- /dev/null +++ b/encore-worker/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + id("encore.kotlin-conventions") + id("encore.spring-boot-app-conventions") +} + +dependencies { + implementation(project(":encore-common")) +} \ No newline at end of file diff --git a/encore-worker/src/main/kotlin/se/svt/oss/encore/EncoreWorkerApplication.kt b/encore-worker/src/main/kotlin/se/svt/oss/encore/EncoreWorkerApplication.kt new file mode 100644 index 00000000..1d916947 --- /dev/null +++ b/encore-worker/src/main/kotlin/se/svt/oss/encore/EncoreWorkerApplication.kt @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore + +import mu.KotlinLogging +import org.springframework.boot.CommandLineRunner +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.runApplication +import org.springframework.context.ApplicationContext +import org.springframework.context.annotation.ImportRuntimeHints +import se.svt.oss.encore.config.EncoreProperties +import se.svt.oss.encore.service.EncoreService +import se.svt.oss.encore.service.queue.QueueService + +@EnableConfigurationProperties(EncoreProperties::class) +@ImportRuntimeHints(EncoreRuntimeHints::class) +@SpringBootApplication +class EncoreWorkerApplication( + private val queueService: QueueService, + private val encoreService: EncoreService, + private val applicationContext: ApplicationContext, + private val encoreProperties: EncoreProperties +) : CommandLineRunner { + private val log = KotlinLogging.logger { } + + override fun run(vararg args: String?) { + try { + poll() + } finally { + log.info { "Stopping" } + SpringApplication.exit(applicationContext) + } + } + + private fun poll() { + val queueNo = encoreProperties.pollQueue ?: 0 + log.info { "Polling queue $queueNo" } + val jobRun = queueService.poll(queueNo, encoreService::encode) + if (encoreProperties.workerDrainQueue && jobRun) { + poll() + } + } +} + +fun main(args: Array) { + runApplication(* args) +} diff --git a/encore-worker/src/main/resources/application.yml b/encore-worker/src/main/resources/application.yml new file mode 100644 index 00000000..45669c0c --- /dev/null +++ b/encore-worker/src/main/resources/application.yml @@ -0,0 +1,13 @@ +spring: + application: + name: encore + banner: + location: classpath:asciilogo.txt + main: + web-application-type: none + cloud: + config: + import-check: + enabled: false +logging: + config: classpath:logback-json.xml \ No newline at end of file diff --git a/encore-worker/src/main/resources/asciilogo.txt b/encore-worker/src/main/resources/asciilogo.txt new file mode 100644 index 00000000..c69322cb --- /dev/null +++ b/encore-worker/src/main/resources/asciilogo.txt @@ -0,0 +1,45 @@ + ``````` + ``.--:/+ossssyysssoo+/:-.``` + ``-:/syhdmmNNNMMMMMMMMMNNNNmddyso:-.` + `.:oydmNMMMMMMNNNmmmmmmmmNNNMMMMMMMNmdho/-.-+/- + `-/ydNNMMMNNmdyso//:--.....--::++oyhmmNMMMNNmmmNy+ + `-:ydNMMMNmdyo:-.``` ```.-/+yhmMMMMMMho + `.ohmMMMNmho/-`` ./hNMMMMMds + `-ohNMMMmd+:.` .:shdddmmhs` + -+dNMMNmo/.` ``...--:-. + `-ydMMMms+`` `...-:/:. + .+dNMMmd:. `-ohhdmmNhs `.-`` + `.ydMMNd+: ./dMMMMMMNd..` `.shdy+:`` .-/:- + -oNMMNd/. `/hNMMMMMMMhyso+/osNMMMNms+. `ohNdh-` + -omMMMy+` `.--` `.+smMMMMMMMMMMMNNNNMMMMMMMNh+` `sdMMNs/` + `sdMMNd:` `:ydds+:/shNNMMMMMMMMNNMMMMMMMMMMMMMho. :oNMMNs: + .:mNMNh/` `ohNMMMNmNNMMMMNmdhyo+++ooyhmNMMMMMMMs/ `-hmMMdy` + :oMMMdo. -+mNMMMMMMMMMMmdo/..``` ````.-oymNMMMMds-` `/hNMNm-. + `ohMMMo: -smNMMMMMMMMNs+. `:sdMMMMNh:-..-.` `odMMMo: + `-yNMMN:. ``/odNMMMMNho`` `.-++o++/.` .-dmMMMMmdhddh+. /yMMMy+ + `/hMMNd.` -yNMMMMh+. `-ohmNNNNNmhs:` /yNMMMMMMMMNy/ -+MMMdo` + `+dMMmy` `:hNMMMN/- +yNMMMMMMMMMNd/. `+hMMMMMMMMMhs .:NMMms- + `omMMds` `-+mMMMMN-. `-hmMMMMMMMMMMMMho -oMMMMMMMMNdy` `-NNMNy-` + .smMMds` `.++ohdMMMMNm-` `:dNMMMMMMMMMMMMds -+NMMMMmhso/: `-NNMNy-` + .smMMds` -/NNMMMMMMMMN-. `-hmMMMMMMMMMMMMho :oMMMMms-`` .-NNMNy-` + `odMMmy` .-mNMMMMMMMMM+- +yNMMMMMMMMMNd/. `+dMMMMd/` ./NMMms-` + `/dMMNd.` ``hmMMMMMMMMMd+. `-ohmNNNNNmhs:` `/yNMMMNh-` :oMMMdo. + `-yNMNN:. oymdddmNMMMMds.` `.-++o++/.` .:dNMMMMNms:. /yMMMy+ + .odMMMo: ..--..+sNMMMMNy+.` ` `:ydMMMMMMMMNmh-` .sdMMMo: + /sMMMdo. `.sdMMMMMNdo+-.```````.-:oyNNMMMMMMMMMMmy.` `/hNMNm-. + .:mNMNh/` `-ymMMMMMMNNdhyssossyhdmNMMMMMNmNNMMNm+- `-hmMMdy` + `ydMMNd-` `+hNMMMMMMMMMMMMMMMMMMMMMMMNms+/osddh/. :oNMMms: + -smMMNo: `:dNMMMMMMMNNNNMMMMMMMMMMMNy+:`` ``--.` `-ymMMNs:` + -+mmm+: `.+ydNMMMNdo++osydmMMMMMMMm/. `:yNMMmh-` + ``:::.` .-oydds:` ```/oMMMMMMMm+. ``ohNMMNs/ + .--. .:NNNmmdhy:. .:sNMMNm+-` + ` `///:-..` .-ydNMMNs+` + .:+//:::--` `./sdMMMNdo-` + /yNNNNNNdy.` `.:oyNNMMNmo:` + :sMMMMMMds-.` `.:oymNMMMNho:` + :+MMMMMMNNdys/:.``` ``.-/+yhmNMMMNdyo.` + -/mdyhdNNMMMNNmdys+//:---------::/+oyhmmNMMMNNdyo-.` + `.:-``./oydmNMMMMMMMNNmmmmmmmmmNNMMMMMMNNmdyo/.` + ``-:/syhddmNNNNMMMMMMMNNNNmdhhso+:-` + ``.--:/++oossyysso++/:-..`` + diff --git a/encore-worker/src/main/resources/logback-json.xml b/encore-worker/src/main/resources/logback-json.xml new file mode 100644 index 00000000..885d39f1 --- /dev/null +++ b/encore-worker/src/main/resources/logback-json.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/encore-worker/src/test/kotlin/se/svt/oss/encore/EncoreWorkerApplicationTest.kt b/encore-worker/src/test/kotlin/se/svt/oss/encore/EncoreWorkerApplicationTest.kt new file mode 100644 index 00000000..60c24a96 --- /dev/null +++ b/encore-worker/src/test/kotlin/se/svt/oss/encore/EncoreWorkerApplicationTest.kt @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: 2020 Sveriges Television AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore + +import io.mockk.every +import io.mockk.impl.annotations.InjectMockKs +import io.mockk.impl.annotations.MockK +import io.mockk.junit5.MockKExtension +import io.mockk.mockkStatic +import io.mockk.unmockkAll +import io.mockk.verify +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.boot.SpringApplication +import org.springframework.context.ApplicationContext +import se.svt.oss.encore.config.EncoreProperties +import se.svt.oss.encore.service.EncoreService +import se.svt.oss.encore.service.queue.QueueService + +@ExtendWith(MockKExtension::class) +class EncoreWorkerApplicationTest { + + @MockK + private lateinit var queueService: QueueService + + @MockK + private lateinit var encoreService: EncoreService + + @MockK + private lateinit var applicationContext: ApplicationContext + + @MockK + private lateinit var encoreProperties: EncoreProperties + + @InjectMockKs + lateinit var application: EncoreWorkerApplication + + @BeforeEach + fun setUp() { + every { queueService.poll(any(), any()) } returns true andThen false + every { encoreProperties.pollQueue } returns 1 + every { encoreProperties.workerDrainQueue } returns false + mockkStatic(SpringApplication::class) + every { SpringApplication.exit(any()) } returns 0 + } + + @AfterEach + fun tearDown() { + unmockkAll() + } + + @Test + fun pollOnce() { + application.run() + verify(exactly = 1) { queueService.poll(1, encoreService::encode) } + verify { SpringApplication.exit(applicationContext) } + } + + @Test + fun defaultsToQueue0() { + every { encoreProperties.pollQueue } returns null + application.run() + verify(exactly = 1) { queueService.poll(0, encoreService::encode) } + verify { SpringApplication.exit(applicationContext) } + } + + @Test + fun drainQueue() { + every { encoreProperties.workerDrainQueue } returns true + application.run() + verify(exactly = 2) { queueService.poll(1, encoreService::encode) } + verify { SpringApplication.exit(applicationContext) } + } +} diff --git a/src/main/resources/encore_logo.png b/encore_logo.png similarity index 100% rename from src/main/resources/encore_logo.png rename to encore_logo.png diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 249e5832..7f93135c 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 8fad3f5a..ac72c34e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index a69d9cb6..0adc8e1a 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +80,11 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +131,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,6 +198,10 @@ if "$cygwin" || "$msys" ; then done fi + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in diff --git a/settings.gradle b/settings.gradle deleted file mode 100644 index a8051409..00000000 --- a/settings.gradle +++ /dev/null @@ -1 +0,0 @@ -rootProject.name = 'encore' \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 00000000..a8ef87a9 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,4 @@ +rootProject.name = "encore" +include("encore-common") +include("encore-web") +include("encore-worker") \ No newline at end of file diff --git a/src/main/kotlin/se/svt/oss/encore/FeignConfiguration.kt b/src/main/kotlin/se/svt/oss/encore/FeignConfiguration.kt deleted file mode 100644 index 3fa4edf7..00000000 --- a/src/main/kotlin/se/svt/oss/encore/FeignConfiguration.kt +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore - -import feign.RequestInterceptor -import org.springframework.beans.factory.annotation.Value -import org.springframework.cloud.openfeign.EnableFeignClients -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.http.HttpHeaders - -@Configuration -@EnableFeignClients -class FeignConfiguration { - - @Bean - fun userAgentInterceptor(@Value("\${service.name:encore}") userAgent: String): RequestInterceptor = - RequestInterceptor { template -> template.header(HttpHeaders.USER_AGENT, userAgent) } -} diff --git a/src/main/kotlin/se/svt/oss/encore/config/EncoreProperties.kt b/src/main/kotlin/se/svt/oss/encore/config/EncoreProperties.kt deleted file mode 100644 index a6ebb001..00000000 --- a/src/main/kotlin/se/svt/oss/encore/config/EncoreProperties.kt +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore.config - -import org.springframework.boot.context.properties.ConfigurationProperties -import org.springframework.boot.context.properties.ConstructorBinding -import java.time.Duration - -@ConfigurationProperties("encore-settings") -@ConstructorBinding -data class EncoreProperties( - val localTemporaryEncode: Boolean = false, - val concurrency: Int = 2, - val pollInitialDelay: Duration = Duration.ofSeconds(10), - val pollDelay: Duration = Duration.ofSeconds(5), - val redisKeyPrefix: String = "encore", - val security: Security = Security(), - val openApi: OpenApi = OpenApi(), - val encoding: EncodingProperties = EncodingProperties() -) { - data class Security( - val enabled: Boolean = false, - val userPassword: String = "", - val adminPassword: String = "" - ) - - data class OpenApi( - val title: String = "Encore OpenAPI", - val description: String = "Endpoints for Encore", - val contactName: String = "", - val contactUrl: String = "", - val contactEmail: String = "" - ) -} diff --git a/src/main/kotlin/se/svt/oss/encore/repository/URIConverters.kt b/src/main/kotlin/se/svt/oss/encore/repository/URIConverters.kt deleted file mode 100644 index 42367e96..00000000 --- a/src/main/kotlin/se/svt/oss/encore/repository/URIConverters.kt +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore.repository - -import java.net.URI -import org.springframework.core.convert.converter.Converter -import org.springframework.data.convert.ReadingConverter -import org.springframework.data.convert.WritingConverter - -@WritingConverter -class URIToByteArrayConverter : Converter { - override fun convert(source: URI): ByteArray? { - return source.toString().toByteArray() - } -} - -@ReadingConverter -class ByteArrayToURIConverter : Converter { - override fun convert(source: ByteArray): URI? { - return URI.create(String(source)) - } -} diff --git a/src/main/kotlin/se/svt/oss/encore/repository/UUIDConverters.kt b/src/main/kotlin/se/svt/oss/encore/repository/UUIDConverters.kt deleted file mode 100644 index fcfeb04f..00000000 --- a/src/main/kotlin/se/svt/oss/encore/repository/UUIDConverters.kt +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore.repository - -import java.util.UUID -import org.springframework.core.convert.converter.Converter -import org.springframework.data.convert.ReadingConverter -import org.springframework.data.convert.WritingConverter - -@WritingConverter -class UUIDToByteArrayConverter : Converter { - override fun convert(source: UUID): ByteArray? { - return source.toString().toByteArray() - } -} - -@ReadingConverter -class ByteArrayToUUIDConverter : Converter { - override fun convert(source: ByteArray): UUID? { - return UUID.fromString(String(source)) - } -} diff --git a/src/main/kotlin/se/svt/oss/encore/service/EncoreService.kt b/src/main/kotlin/se/svt/oss/encore/service/EncoreService.kt deleted file mode 100644 index 050a354b..00000000 --- a/src/main/kotlin/se/svt/oss/encore/service/EncoreService.kt +++ /dev/null @@ -1,163 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore.service - -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.FlowPreview -import kotlinx.coroutines.Job -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.ReceiveChannel -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.conflate -import kotlinx.coroutines.flow.consumeAsFlow -import kotlinx.coroutines.flow.distinctUntilChanged -import kotlinx.coroutines.flow.sample -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.slf4j.MDCContext -import mu.KotlinLogging -import org.redisson.api.RTopic -import org.redisson.api.RedissonClient -import org.springframework.data.redis.core.PartialUpdate -import org.springframework.data.redis.core.RedisKeyValueTemplate -import org.springframework.stereotype.Service -import se.svt.oss.encore.cancellation.CancellationListener -import se.svt.oss.encore.config.EncoreProperties -import se.svt.oss.encore.model.CancelEvent -import se.svt.oss.encore.model.EncoreJob -import se.svt.oss.encore.model.Status -import se.svt.oss.encore.repository.EncoreJobRepository -import se.svt.oss.encore.service.callback.CallbackService -import se.svt.oss.encore.service.localencode.LocalEncodeService -import se.svt.oss.encore.service.mediaanalyzer.MediaAnalyzerService -import se.svt.oss.encore.service.profile.ProfileService -import se.svt.oss.mediaanalyzer.file.MediaContainer -import se.svt.oss.mediaanalyzer.file.MediaFile -import java.util.Locale - -@Service -@ExperimentalCoroutinesApi -@FlowPreview -class EncoreService( - private val callbackService: CallbackService, - private val repository: EncoreJobRepository, - private val profileService: ProfileService, - private val ffmpegExecutor: FfmpegExecutor, - private val redissonClient: RedissonClient, - private val redisKeyValueTemplate: RedisKeyValueTemplate, - private val mediaAnalyzerService: MediaAnalyzerService, - private val localEncodeService: LocalEncodeService, - private val encoreProperties: EncoreProperties -) { - - private val log = KotlinLogging.logger {} - - private val cancelTopicName = "cancel" - - fun encode(encoreJob: EncoreJob) { - val coroutineJob = Job() - val cancelListener = CancellationListener(encoreJob.id, coroutineJob) - var cancelTopic: RTopic? = null - var outputFolder: String? = null - - try { - cancelTopic = redissonClient.getTopic(cancelTopicName) - cancelTopic.addListener(CancelEvent::class.java, cancelListener) - - encoreJob.inputs.forEach { input -> - mediaAnalyzerService.analyzeInput(input) - } - - log.info { "Start $encoreJob" } - encoreJob.status = Status.IN_PROGRESS - repository.save(encoreJob) - - val profile = profileService.getProfile(encoreJob.profile) - - outputFolder = localEncodeService.outputFolder(encoreJob) - - val outputs = profile.encodes.mapNotNull { - it.getOutput( - encoreJob, - encoreProperties.encoding - ) - } - - check(outputs.distinctBy { it.id }.size == outputs.size) { - "Profile ${encoreJob.profile} contains duplicate suffixes: ${outputs.map { it.id }}!" - } - - val start = System.currentTimeMillis() - - var outputFiles = runBlocking(coroutineJob + MDCContext()) { - val progressChannel = Channel() - handleProgress(progressChannel, encoreJob) - ffmpegExecutor.run(encoreJob, profile, outputs, outputFolder, progressChannel) - } - - outputFiles = localEncodeService.localEncodedFilesToCorrectDir(outputFolder, outputFiles, encoreJob) - - val time = (System.currentTimeMillis() - start) / 1000 - val speed = outputFiles.filterIsInstance().firstOrNull()?.let { - "%.3f".format(Locale.US, it.duration / time).toDouble() - } ?: 0.0 - log.info { "Done encoding, time: ${time}s, speed: ${speed}X" } - updateSuccessfulJob(encoreJob, outputFiles, speed) - log.info { "Done with $encoreJob" } - } catch (e: InterruptedException) { - val message = "Job execution interrupted" - log.error(e) { message } - encoreJob.status = Status.QUEUED - encoreJob.message = message - throw e - } catch (e: CancellationException) { - log.error(e) { "Job execution cancelled: $e.message" } - encoreJob.status = Status.CANCELLED - encoreJob.message = e.message - } catch (e: Exception) { - log.error(e) { "Job execution failed: ${e.message}" } - encoreJob.status = Status.FAILED - encoreJob.message = e.message - } finally { - repository.save(encoreJob) - cancelTopic?.removeListener(cancelListener) - callbackService.sendProgressCallback(encoreJob) - localEncodeService.cleanup(outputFolder) - } - } - - private fun CoroutineScope.handleProgress( - progressChannel: ReceiveChannel, - encoreJob: EncoreJob - ) { - launch { - progressChannel.consumeAsFlow() - .conflate() - .distinctUntilChanged() - .sample(10_000) - .collect { - log.info { "Received progress $it" } - try { - encoreJob.progress = it - val partialUpdate = PartialUpdate(encoreJob.id, EncoreJob::class.java) - .set(encoreJob::progress.name, encoreJob.progress) - redisKeyValueTemplate.update(partialUpdate) - callbackService.sendProgressCallback(encoreJob) - } catch (e: Exception) { - log.warn(e) { "Error updating progress!" } - } - } - } - } - - private fun updateSuccessfulJob(encoreJob: EncoreJob, output: List, speed: Double) { - encoreJob.output = output - encoreJob.status = Status.SUCCESSFUL - encoreJob.progress = 100 - encoreJob.speed = speed - } -} diff --git a/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackClient.kt b/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackClient.kt deleted file mode 100644 index 734e71b2..00000000 --- a/src/main/kotlin/se/svt/oss/encore/service/callback/CallbackClient.kt +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore.service.callback - -import java.net.URI -import org.springframework.cloud.openfeign.FeignClient -import org.springframework.web.bind.annotation.PostMapping -import se.svt.oss.encore.model.callback.JobProgress - -@FeignClient("callback") -interface CallbackClient { - - @PostMapping - fun sendProgressCallback(callbackUri: URI, progress: JobProgress) -} diff --git a/src/main/kotlin/se/svt/oss/encore/service/poll/JobPoller.kt b/src/main/kotlin/se/svt/oss/encore/service/poll/JobPoller.kt deleted file mode 100644 index b140386a..00000000 --- a/src/main/kotlin/se/svt/oss/encore/service/poll/JobPoller.kt +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore.service.poll - -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.FlowPreview -import mu.KotlinLogging -import mu.withLoggingContext -import org.springframework.data.repository.findByIdOrNull -import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler -import org.springframework.stereotype.Service -import se.svt.oss.encore.config.EncoreProperties -import se.svt.oss.encore.model.EncoreJob -import se.svt.oss.encore.model.Status -import se.svt.oss.encore.model.queue.QueueItem -import se.svt.oss.encore.repository.EncoreJobRepository -import se.svt.oss.encore.service.EncoreService -import se.svt.oss.encore.service.queue.QueueService -import java.time.Instant -import java.util.UUID -import java.util.concurrent.ScheduledFuture -import javax.annotation.PostConstruct -import javax.annotation.PreDestroy - -@Service -@ExperimentalCoroutinesApi -@FlowPreview -class JobPoller( - private val repository: EncoreJobRepository, - private val queueService: QueueService, - private val encoreService: EncoreService, - private val scheduler: ThreadPoolTaskScheduler, - private val encoreProperties: EncoreProperties, -) { - - private val log = KotlinLogging.logger {} - private var scheduledTasks = emptyList>() - - @PostConstruct - fun init() { - scheduledTasks = (0 until encoreProperties.concurrency).map { queueNo -> - scheduler.scheduleWithFixedDelay( - { - try { - queueService.poll(queueNo)?.let { handleJob(it) } - } catch (e: Throwable) { - log.error(e) { "Error polling queue $queueNo!" } - } - }, - Instant.now().plus(encoreProperties.pollInitialDelay), - encoreProperties.pollDelay - ) - } - } - - @PreDestroy - fun destroy() { - scheduledTasks.forEach { it.cancel(false) } - } - - private fun handleJob(queueItem: QueueItem) { - val id = UUID.fromString(queueItem.id) - log.info { "Handling job $id" } - val job = repository.findByIdOrNull(id) - ?: retry(id) // Sometimes there has been sync issues - ?: throw RuntimeException("Job ${queueItem.id} does not exist") - - withLoggingContext(job.contextMap) { - if (job.status.isCancelled) { - log.info { "Job was cancelled" } - return - } - log.info { "Running job" } - try { - encoreService.encode(job) - } catch (e: InterruptedException) { - repostJob(job) - } - } - } - - private fun repostJob(job: EncoreJob) { - try { - log.info { "Adding job to queue (repost on interrupt)" } - queueService.enqueue(job) - log.info { "Added job to queue (repost on interrupt)" } - } catch (e: Exception) { - val message = "Failed to add interrupted job to queue" - log.error(e) { message } - job.message = message - job.status = Status.FAILED - repository.save(job) - } - } - - private fun retry(id: UUID): EncoreJob? { - Thread.sleep(5000) - log.info { "Retrying read of job from repository " } - return repository.findByIdOrNull(id) - } -} diff --git a/src/main/kotlin/se/svt/oss/encore/service/queue/QueueService.kt b/src/main/kotlin/se/svt/oss/encore/service/queue/QueueService.kt deleted file mode 100644 index c9f36f68..00000000 --- a/src/main/kotlin/se/svt/oss/encore/service/queue/QueueService.kt +++ /dev/null @@ -1,76 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 -package se.svt.oss.encore.service.queue - -import mu.KotlinLogging -import org.redisson.api.RPriorityBlockingQueue -import org.redisson.api.RedissonClient -import org.springframework.stereotype.Component -import se.svt.oss.encore.config.EncoreProperties -import se.svt.oss.encore.model.EncoreJob -import se.svt.oss.encore.model.queue.QueueItem -import se.svt.oss.encore.service.queue.QueueUtil.getQueueNumberByPriority -import java.util.concurrent.ConcurrentSkipListMap -import java.util.concurrent.TimeUnit -import javax.annotation.PostConstruct - -@Component -class QueueService( - encoreProperties: EncoreProperties, - private val redisson: RedissonClient -) { - - private val log = KotlinLogging.logger { } - private val queues = ConcurrentSkipListMap>() - private val concurrency = encoreProperties.concurrency - private val redisKeyPrefix = encoreProperties.redisKeyPrefix - - fun poll(queueNo: Int): QueueItem? = - (0..queueNo) - .asSequence() - .mapNotNull { getQueue(it).poll() } - .firstOrNull() - - fun enqueue(job: EncoreJob) { - val queueItem = QueueItem( - id = job.id.toString(), - priority = job.priority, - created = job.createdDate.toLocalDateTime() - ) - if (!queueByPrio(job.priority).offer(queueItem, 5, TimeUnit.SECONDS)) { - throw RuntimeException("Job could not be added to queue!") - } - } - - fun getQueue(): List { - return (0 until concurrency).flatMap { getQueue(it).toList() } - } - - private fun queueByPrio(priority: Int) = - getQueue(getQueueNumberByPriority(concurrency, priority)) - - private fun getQueue(queueNo: Int) = queues.computeIfAbsent(queueNo) { - redisson.getPriorityBlockingQueue("$redisKeyPrefix-queue-$queueNo") - } - - @PostConstruct - internal fun handleOrphanedQueues() { - try { - val oldConcurrency = - redisson.getAtomicLong("$redisKeyPrefix-concurrency").getAndSet(concurrency.toLong()).toInt() - if (oldConcurrency > concurrency) { - log.info { "Moving orphaned queue items to lowest priority queue. Old concurrency: $oldConcurrency, new concurrency: $concurrency" } - val lowestPrioQueue = getQueue(concurrency - 1) - (concurrency until oldConcurrency).forEach { queueNo -> - val orphanedQueue = getQueue(queueNo) - val transferred = orphanedQueue.drainTo(lowestPrioQueue) - log.info { "Moved $transferred orphaned items from queue $queueNo to lowest priority queue." } - orphanedQueue.delete() - } - } - } catch (e: Exception) { - log.error(e) { "Error checking for concurrency change: ${e.message}" } - } - } -} diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml deleted file mode 100644 index 3e5933fd..00000000 --- a/src/main/resources/application-local.yml +++ /dev/null @@ -1,24 +0,0 @@ -service: - name: encore-local - -spring: - redis: - host: localhost - port: 6379 - -profile: - location: url:https://raw.githubusercontent.com/svt/encore/master/src/test/resources/profile/profiles.yml - -encore-settings: - concurrency: 3 - local-temporary-encode: false - poll-initial-delay: 1s - poll-delaly: 1s - - audio-mix-presets: - default: - pan-mapping: - 6: - 2: stereo|c0=1.0*c0+0.707*c2+0.707*c4|c1=1.0*c1+0.707*c2+0.707*c5 - de: - fallback-to-auto: true diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml deleted file mode 100644 index 1eee7289..00000000 --- a/src/main/resources/application.yml +++ /dev/null @@ -1,49 +0,0 @@ -spring: - application: - name: encore - banner: - location: classpath:asciilogo.txt - cloud: - config: - enabled: false - -health: - config: - enabled: false - -management: - endpoint: - health: - show-details: always - endpoints: - web: - base-path: / - health: - redis: - enabled: true - -service: - name: encore - -feign: - okhttp: - enabled: true - client: - config: - default: - connectTimeout: 2000 - readTimeout: 5000 - loggerLevel: basic - -encore-settings: - redis-key-prefix: ${service.name} - poll-initial-delay: 10s - poll-delaly: 5s - - -springdoc: - paths-to-exclude: /profile/encoreJobs,/profile - swagger-ui: - operations-sorter: alpha - tags-sorter: alpha - disable-swagger-default-url: true diff --git a/src/test/kotlin/se/svt/oss/encore/EncoreEndpointAccessIntegrationTest.kt b/src/test/kotlin/se/svt/oss/encore/EncoreEndpointAccessIntegrationTest.kt deleted file mode 100644 index 65ee1c17..00000000 --- a/src/test/kotlin/se/svt/oss/encore/EncoreEndpointAccessIntegrationTest.kt +++ /dev/null @@ -1,120 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore - -import feign.FeignException -import feign.RequestInterceptor -import feign.auth.BasicAuthRequestInterceptor -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows -import org.springframework.beans.factory.NoSuchBeanDefinitionException -import org.springframework.boot.test.context.SpringBootTest -import org.springframework.boot.test.context.TestConfiguration -import org.springframework.boot.test.context.assertj.AssertableApplicationContext -import org.springframework.boot.test.context.runner.ApplicationContextRunner -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Import -import org.springframework.test.context.ActiveProfiles -import se.svt.oss.encore.Assertions.assertThat -import se.svt.oss.encore.config.EncoreProperties -import java.io.File - -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, - properties = ["encore-settings.security.enabled=true", "encore-settings.security.user-password=upw", "encore-settings.security.admin-password=apw"] -) -@ActiveProfiles("test") -class EncoreEndpointAccessIntegrationTest : EncoreIntegrationTestBase() { - - @Test - fun `security configuration is not loaded in context when security disabled`() { - val contextRunner = ApplicationContextRunner() - contextRunner - .withBean(EncoreProperties::class.java) - .withPropertyValues("encore-settings.security.enabled=false") - .withUserConfiguration(SecurityConfiguration::class.java) - .run { context: AssertableApplicationContext -> - assertThrows { - context.getBean(SecurityConfiguration::class.java) - } - } - } -} - -@Import(UserEndpointAccessIntegrationTest.Conf::class) -class UserEndpointAccessIntegrationTest : EncoreEndpointAccessIntegrationTest() { - - @TestConfiguration - class Conf { - @Bean - fun basicAuthRequestInterceptor(): RequestInterceptor? { - return BasicAuthRequestInterceptor("user", "upw") - } - } - - @Test - fun `User user is allowed GET`() { - encoreClient.jobs() - } - - @Test - fun `User user is Forbidden POST`() { - assertThrows { - encoreClient.createJob(job(File(""))) - } - } -} - -@Import(AdminEndpointAccessIntegrationTest.Conf::class) -class AdminEndpointAccessIntegrationTest : EncoreEndpointAccessIntegrationTest() { - - @TestConfiguration - class Conf { - @Bean - fun basicAuthRequestInterceptor(): RequestInterceptor? { - return BasicAuthRequestInterceptor("admin", "apw") - } - } - - @Test - fun `Admin user is allowed GET`() { - encoreClient.jobs() - } - - @Test - fun `Admin user is allowed POST`() { - encoreClient.createJob(job(File(""))) - } -} - -class NoUserEndpointAccessIntegrationTest : EncoreEndpointAccessIntegrationTest() { - - data class MyHealth( - val status: String, - val components: Map?, - val groups: List? - ) - - @Test - fun `Anonymous user is not authorized GET`() { - assertThrows { - encoreClient.jobs() - } - } - - @Test - fun `Anonymous user is authorized GET health without details`() { - val health = objectMapper.readValue(encoreClient.health(), MyHealth::class.java) - assertThat(health.status).isEqualTo("UP") - assertThat(health.components).isNull() - } - - @Test - fun `Anonymous user is not authorized POST`() { - assertThrows { - encoreClient.createJob(job(File(""))) - } - } -} diff --git a/src/test/kotlin/se/svt/oss/encore/service/poll/JobPollerTest.kt b/src/test/kotlin/se/svt/oss/encore/service/poll/JobPollerTest.kt deleted file mode 100644 index ae8af95f..00000000 --- a/src/test/kotlin/se/svt/oss/encore/service/poll/JobPollerTest.kt +++ /dev/null @@ -1,191 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore.service.poll - -import io.mockk.Runs -import io.mockk.every -import io.mockk.impl.annotations.InjectMockKs -import io.mockk.impl.annotations.MockK -import io.mockk.junit5.MockKExtension -import io.mockk.just -import io.mockk.mockk -import io.mockk.verify -import io.mockk.verifySequence -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.FlowPreview -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.ExtendWith -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.ValueSource -import org.springframework.data.repository.findByIdOrNull -import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler -import se.svt.oss.encore.Assertions.assertThat -import se.svt.oss.encore.config.EncoreProperties -import se.svt.oss.encore.defaultEncoreJob -import se.svt.oss.encore.model.Status -import se.svt.oss.encore.model.queue.QueueItem -import se.svt.oss.encore.repository.EncoreJobRepository -import se.svt.oss.encore.service.EncoreService -import se.svt.oss.encore.service.queue.QueueService -import java.time.Instant -import java.util.concurrent.ScheduledFuture - -@FlowPreview -@ExperimentalCoroutinesApi -@ExtendWith(MockKExtension::class) -class JobPollerTest { - - @MockK - private lateinit var repository: EncoreJobRepository - - @MockK - private lateinit var queueService: QueueService - - private val encoreProperties = EncoreProperties(concurrency = 3) - - @MockK - private lateinit var encoreService: EncoreService - - @MockK - private lateinit var scheduler: ThreadPoolTaskScheduler - - @InjectMockKs - private lateinit var jobPoller: JobPoller - - private val encoreJob = defaultEncoreJob() - - private val queueItem = QueueItem(encoreJob.id.toString()) - - private val capturedRunnables = mutableListOf() - private val scheduledTasks = mutableListOf>() - - @BeforeEach - fun setUp() { - every { scheduler.scheduleWithFixedDelay(capture(capturedRunnables), any(), any()) } answers { - val scheduled = mockk>() - scheduledTasks.add(scheduled) - scheduled - } - every { repository.findByIdOrNull(encoreJob.id) } returns encoreJob - every { encoreService.encode(encoreJob) } just Runs - every { queueService.poll(any()) } returns queueItem - jobPoller.init() - assertThat(capturedRunnables).hasSize(3) - } - - @Test - fun testDestroy() { - assertThat(scheduledTasks).hasSize(3) - scheduledTasks.forEach { - every { it.cancel(false) } returns true - } - jobPoller.destroy() - scheduledTasks.forEach { - verify { it.cancel(false) } - } - } - - @ParameterizedTest - @ValueSource(ints = [0, 1, 2]) - fun poll(thread: Int) { - capturedRunnables[thread].run() - - verifySequence { - queueService.poll(thread) - repository.findByIdOrNull(encoreJob.id) - encoreService.encode(encoreJob) - } - } - - @ParameterizedTest - @ValueSource(ints = [0, 1, 2]) - fun `poll causes exception`(thread: Int) { - every { queueService.poll(thread) } throws Exception("error") - - capturedRunnables[thread].run() - - verifySequence { - queueService.poll(thread) - } - } - - @ParameterizedTest - @ValueSource(ints = [0, 1, 2]) - fun repositoryReturnsNull(thread: Int) { - every { repository.findByIdOrNull(encoreJob.id) } returns null - - capturedRunnables[thread].run() - - verifySequence { - queueService.poll(thread) - repository.findByIdOrNull(encoreJob.id) - repository.findByIdOrNull(encoreJob.id) - } - } - - @ParameterizedTest - @ValueSource(ints = [0, 1, 2]) - fun repositoryRetryWorks(thread: Int) { - every { repository.findByIdOrNull(encoreJob.id) } returns null andThen encoreJob - - capturedRunnables[thread].run() - - verifySequence { - queueService.poll(thread) - repository.findByIdOrNull(encoreJob.id) - repository.findByIdOrNull(encoreJob.id) - encoreService.encode(encoreJob) - } - } - - @ParameterizedTest - @ValueSource(ints = [0, 1, 2]) - fun `cancelled job`(thread: Int) { - encoreJob.status = Status.CANCELLED - - capturedRunnables[thread].run() - assertThat(encoreJob.status).isEqualTo(Status.CANCELLED) - - verifySequence { - queueService.poll(thread) - repository.findByIdOrNull(encoreJob.id) - } - } - - @ParameterizedTest - @ValueSource(ints = [0, 1, 2]) - fun `interrupted job re-enqueues`(thread: Int) { - every { encoreService.encode(encoreJob) } throws InterruptedException() - every { queueService.enqueue(encoreJob) } just Runs - - capturedRunnables[thread].run() - assertThat(encoreJob.status).isEqualTo(Status.NEW) - - verifySequence { - queueService.poll(thread) - repository.findByIdOrNull(encoreJob.id) - queueService.enqueue(encoreJob) - } - } - - @ParameterizedTest - @ValueSource(ints = [0, 1, 2]) - fun `interrupted job re-enqueue fails`(thread: Int) { - every { encoreService.encode(encoreJob) } throws InterruptedException() - every { queueService.enqueue(encoreJob) } throws Exception("error") - every { repository.save(encoreJob) } returns encoreJob - - capturedRunnables[thread].run() - assertThat(encoreJob.status).isEqualTo(Status.FAILED) - - verifySequence { - queueService.poll(thread) - repository.findByIdOrNull(encoreJob.id) - queueService.enqueue(encoreJob) - repository.save(encoreJob) - } - } -} diff --git a/src/test/kotlin/se/svt/oss/encore/service/queue/QueueServiceTest.kt b/src/test/kotlin/se/svt/oss/encore/service/queue/QueueServiceTest.kt deleted file mode 100644 index 2c29c2b6..00000000 --- a/src/test/kotlin/se/svt/oss/encore/service/queue/QueueServiceTest.kt +++ /dev/null @@ -1,199 +0,0 @@ -// SPDX-FileCopyrightText: 2020 Sveriges Television AB -// -// SPDX-License-Identifier: EUPL-1.2 - -package se.svt.oss.encore.service.queue - -import io.mockk.every -import io.mockk.impl.annotations.InjectMockKs -import io.mockk.impl.annotations.MockK -import io.mockk.junit5.MockKExtension -import io.mockk.mockk -import io.mockk.verify -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Nested -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.ExtendWith -import org.redisson.api.RPriorityBlockingQueue -import org.redisson.api.RedissonClient -import se.svt.oss.encore.config.EncoreProperties -import se.svt.oss.encore.defaultEncoreJob -import se.svt.oss.encore.model.EncoreJob -import se.svt.oss.encore.model.queue.QueueItem -import java.util.concurrent.TimeUnit - -@ExtendWith(MockKExtension::class) -internal class QueueServiceTest { - private val highPriorityQueue = mockk>() - private val standardPriorityQueue = mockk>() - private val lowPriorityQueue = mockk>() - - private val encoreProperties = EncoreProperties(concurrency = 3,) - - @MockK - private lateinit var redisson: RedissonClient - - @InjectMockKs - private lateinit var queueService: QueueService - - private val queueItemHighPrio = QueueItem("high", 90) - private val queueItemStandardPrio = QueueItem("standard", 51) - private val queueItemLowPrio = QueueItem("low", 10) - - @BeforeEach - internal fun setUp() { - every { highPriorityQueue.poll() } returns queueItemHighPrio - every { standardPriorityQueue.poll() } returns queueItemStandardPrio - every { lowPriorityQueue.poll() } returns queueItemLowPrio - every { redisson.getPriorityBlockingQueue("${encoreProperties.redisKeyPrefix}-queue-0") } returns highPriorityQueue - every { redisson.getPriorityBlockingQueue("${encoreProperties.redisKeyPrefix}-queue-1") } returns standardPriorityQueue - every { redisson.getPriorityBlockingQueue("${encoreProperties.redisKeyPrefix}-queue-2") } returns lowPriorityQueue - } - - @Nested - inner class Init { - - @Test - fun concurrencyReduced() { - val orphanedQueue1 = mockk>() - val orphanedQueue2 = mockk>() - every { orphanedQueue1.drainTo(any()) } returns 1 - every { orphanedQueue2.drainTo(any()) } returns 1 - every { orphanedQueue1.delete() } returns true - every { orphanedQueue2.delete() } returns true - val concurrency = encoreProperties.concurrency - every { redisson.getAtomicLong("${encoreProperties.redisKeyPrefix}-concurrency").getAndSet(concurrency.toLong()) } returns concurrency.toLong() + 2 - every { redisson.getPriorityBlockingQueue("${encoreProperties.redisKeyPrefix}-queue-$concurrency") } returns orphanedQueue1 - every { redisson.getPriorityBlockingQueue("${encoreProperties.redisKeyPrefix}-queue-${concurrency + 1}") } returns orphanedQueue2 - - queueService.handleOrphanedQueues() - - verify { orphanedQueue1.drainTo(lowPriorityQueue) } - verify { orphanedQueue2.drainTo(lowPriorityQueue) } - verify { orphanedQueue1.delete() } - verify { orphanedQueue2.delete() } - } - } - - @Nested - inner class PollHighPriorityQueue { - - @AfterEach - fun tearDown() { - verify { highPriorityQueue.poll() } - verify(exactly = 0) { standardPriorityQueue.poll() } - } - - @Test - fun `returns item from high priority queue if any present`() { - assertThat(queueService.poll(0)).isSameAs(queueItemHighPrio) - } - - @Test - fun `returns null if high priority queue is empty`() { - every { highPriorityQueue.poll() } returns null - assertThat(queueService.poll(0)).isNull() - } - } - - @Nested - inner class PollHighOrStandardPriorityQueue { - - @Test - fun `returns item from high priority queue if any present`() { - assertThat(queueService.poll(1)).isSameAs(queueItemHighPrio) - verify { highPriorityQueue.poll() } - verify(exactly = 0) { standardPriorityQueue.poll() } - } - - @Test - fun `returns items from standard priority queue if high priority queue is empty`() { - every { highPriorityQueue.poll() } returns null - assertThat(queueService.poll(1)).isSameAs(queueItemStandardPrio) - verify { highPriorityQueue.poll() } - verify { standardPriorityQueue.poll() } - } - - @Test - fun `returns null if both queues empty`() { - every { highPriorityQueue.poll() } returns null - every { standardPriorityQueue.poll() } returns null - assertThat(queueService.poll(1)).isNull() - verify { highPriorityQueue.poll() } - verify { standardPriorityQueue.poll() } - } - } - - @Nested - inner class PollHighOrLowPriorityQueue { - - @Test - fun `returns item from high priority queue if any present`() { - assertThat(queueService.poll(2)).isSameAs(queueItemHighPrio) - verify { highPriorityQueue.poll() } - verify(exactly = 0) { lowPriorityQueue.poll() } - } - - @Test - fun `returns items from low priority queue if present and high priority queue is empty`() { - every { highPriorityQueue.poll() } returns null - every { standardPriorityQueue.poll() } returns null - assertThat(queueService.poll(2)).isSameAs(queueItemLowPrio) - verify { highPriorityQueue.poll() } - verify { lowPriorityQueue.poll() } - } - - @Test - fun `returns null if both queues are empty`() { - every { highPriorityQueue.poll() } returns null - every { standardPriorityQueue.poll() } returns null - every { lowPriorityQueue.poll() } returns null - assertThat(queueService.poll(2)).isNull() - verify { highPriorityQueue.poll() } - verify { lowPriorityQueue.poll() } - } - } - - @Nested - inner class Enqueue { - - @Test - fun `low priority job is enqueued on low priority queue`() { - val job = defaultEncoreJob(10) - every { lowPriorityQueue.offer(any(), any(), any()) } returns true - queueService.enqueue(job) - verify { lowPriorityQueue.offer(expectedQueueItem(job), 5, TimeUnit.SECONDS) } - verify(exactly = 0) { standardPriorityQueue.offer(any(), any(), any()) } - verify(exactly = 0) { highPriorityQueue.offer(any(), any(), any()) } - } - - @Test - fun `standard priority job is enqueued on standard priority queue`() { - val job = defaultEncoreJob(55) - every { standardPriorityQueue.offer(any(), any(), any()) } returns true - queueService.enqueue(job) - verify { standardPriorityQueue.offer(expectedQueueItem(job), 5, TimeUnit.SECONDS) } - verify(exactly = 0) { highPriorityQueue.offer(any(), any(), any()) } - verify(exactly = 0) { lowPriorityQueue.offer(any(), any(), any()) } - } - - @Test - fun `high priority job is enqueued on high priority queue`() { - val job = defaultEncoreJob(priority = 90) - every { highPriorityQueue.offer(any(), any(), any()) } returns true - queueService.enqueue(job) - verify { highPriorityQueue.offer(expectedQueueItem(job), 5, TimeUnit.SECONDS) } - verify(exactly = 0) { standardPriorityQueue.offer(any(), any(), any()) } - verify(exactly = 0) { lowPriorityQueue.offer(any(), any(), any()) } - } - - private fun expectedQueueItem(job: EncoreJob) = - QueueItem( - id = job.id.toString(), - priority = job.priority, - created = job.createdDate.toLocalDateTime() - ) - } -}