diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt index 665ccc0..94eda9c 100644 --- a/encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt @@ -10,6 +10,9 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import se.svt.oss.encore.service.remotefiles.s3.S3Properties import se.svt.oss.encore.service.remotefiles.s3.S3RemoteFileHandler +import se.svt.oss.encore.service.remotefiles.s3.S3UriConverter +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3AsyncClient import software.amazon.awssdk.services.s3.S3Configuration @@ -35,6 +38,13 @@ class S3RemoteFilesConfiguration { .pathStyleAccessEnabled(true) .build() ) + .credentialsProvider( + if (s3Properties.anonymousAccess) { + AnonymousCredentialsProvider.create() + } else { + DefaultCredentialsProvider.create() + } + ) .apply { if (!s3Properties.endpoint.isNullOrBlank()) { endpointOverride(URI.create(s3Properties.endpoint)) @@ -58,6 +68,9 @@ class S3RemoteFilesConfiguration { .build() @Bean - fun s3RemoteFileHandler(s3Client: S3AsyncClient, s3Presigner: S3Presigner, s3Properties: S3Properties) = - S3RemoteFileHandler(s3Client, s3Presigner, s3Properties) + fun s3UriConverter(s3Properties: S3Properties, s3Region: Region) = S3UriConverter(s3Properties, s3Region) + + @Bean + fun s3RemoteFileHandler(s3Client: S3AsyncClient, s3Presigner: S3Presigner, s3Properties: S3Properties, s3UriConverter: S3UriConverter) = + S3RemoteFileHandler(s3Client, s3Presigner, s3Properties, s3UriConverter) } diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt index 06c2d5b..77fb8c9 100644 --- a/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt @@ -8,14 +8,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo import io.swagger.v3.oas.annotations.media.Schema +import jakarta.validation.constraints.Pattern +import jakarta.validation.constraints.PositiveOrZero import se.svt.oss.encore.model.mediafile.toParams import se.svt.oss.encore.model.profile.ChannelLayout import se.svt.oss.mediaanalyzer.file.FractionString import se.svt.oss.mediaanalyzer.file.MediaContainer import se.svt.oss.mediaanalyzer.file.MediaFile import se.svt.oss.mediaanalyzer.file.VideoFile -import jakarta.validation.constraints.Pattern -import jakarta.validation.constraints.PositiveOrZero const val TYPE_AUDIO_VIDEO = "AudioVideo" const val TYPE_AUDIO = "Audio" @@ -255,7 +255,7 @@ fun List.inputParams(readDuration: Double?): List = (readDuration?.let { listOf("-t", "$it") } ?: emptyList()) + (input.seekTo?.let { listOf("-ss", "$it") } ?: emptyList()) + (if (input.copyTs) listOf("-copyts") else emptyList()) + - listOf("-i", input.accessUri ?: input.uri) + listOf("-i", input.accessUri) } fun List.maxDuration(): Double? = maxOfOrNull { diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt index a3c3383..82eb5e3 100644 --- a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt @@ -10,6 +10,7 @@ import java.time.Duration @ConfigurationProperties("remote-files.s3") data class S3Properties( val enabled: Boolean = false, + val anonymousAccess: Boolean = false, val endpoint: String = "", val presignDurationSeconds: Long = Duration.ofHours(12).seconds, val uploadTimeoutSeconds: Long = Duration.ofHours(1).seconds diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3RemoteFileHandler.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3RemoteFileHandler.kt index 43c5b06..52a5e22 100644 --- a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3RemoteFileHandler.kt +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3RemoteFileHandler.kt @@ -17,7 +17,8 @@ import java.util.concurrent.TimeUnit class S3RemoteFileHandler( private val client: S3AsyncClient, private val presigner: S3Presigner, - private val s3Properties: S3Properties + private val s3Properties: S3Properties, + private val s3UriConverter: S3UriConverter ) : RemoteFileHandler { private val log = mu.KotlinLogging.logger {} @@ -25,9 +26,18 @@ class S3RemoteFileHandler( override fun getAccessUri(uri: String): String { val s3Uri = URI.create(uri) + if (s3Properties.anonymousAccess) { + return s3UriConverter.toHttp(s3Uri) + } + return presignUrl(s3Uri) + } + + private fun presignUrl(s3Uri: URI): String { + val (bucket, key) = s3UriConverter.getBucketAndKey(s3Uri) + val objectRequest: GetObjectRequest = GetObjectRequest.builder() - .bucket(s3Uri.host) - .key(s3Uri.path.stripLeadingSlash()) + .bucket(bucket) + .key(key) .build() val presignRequest: GetObjectPresignRequest = GetObjectPresignRequest.builder() .signatureDuration(java.time.Duration.ofSeconds(s3Properties.presignDurationSeconds)) @@ -42,17 +52,14 @@ class S3RemoteFileHandler( override fun upload(localFile: String, remoteFile: String) { log.info { "Uploading $localFile to $remoteFile" } val s3Uri = URI.create(remoteFile) - val bucket = s3Uri.host - val objectName = s3Uri.path.stripLeadingSlash() + val (bucket, key) = s3UriConverter.getBucketAndKey(s3Uri) val putObjectRequest: PutObjectRequest = PutObjectRequest.builder() .bucket(bucket) - .key(objectName) + .key(key) .build() val res = client.putObject(putObjectRequest, Paths.get(localFile)).get(s3Properties.presignDurationSeconds, TimeUnit.SECONDS) log.info { "Upload result: $res" } } - private fun String.stripLeadingSlash() = if (startsWith("/")) substring(1) else this - override val protocols = listOf("s3") } diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3UriConverter.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3UriConverter.kt new file mode 100644 index 0000000..0727cfc --- /dev/null +++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3UriConverter.kt @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2024 Eyevinn Technology AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.service.remotefiles.s3 + +import software.amazon.awssdk.regions.Region +import java.net.URI + +class S3UriConverter( + private val s3Properties: S3Properties, + private val region: Region +) { + + fun toHttp(s3Uri: URI): String { + if (s3Uri.scheme != "s3") { + throw IllegalArgumentException("Invalid URI: $s3Uri") + } + val bucket = s3Uri.host + val key = s3Uri.path.stripLeadingSlash() + + if (s3Properties.endpoint.isNotBlank()) { + return "https://$bucket.${s3Properties.endpoint}/$key" + } + return "https://$bucket.s3.$region.amazonaws.com/$key" + } + + fun getBucketAndKey(s3Uri: URI): Pair { + if (s3Uri.scheme != "s3") { + throw IllegalArgumentException("Invalid URI: $s3Uri") + } + val bucket = s3Uri.host + val key = s3Uri.path.stripLeadingSlash() + return Pair(bucket, key) + } + + private fun String.stripLeadingSlash() = if (startsWith("/")) substring(1) else this +} diff --git a/encore-common/src/test/kotlin/se/svt/oss/encore/service/remotefiles/S3UriConverterTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/service/remotefiles/S3UriConverterTest.kt new file mode 100644 index 0000000..c67b9c9 --- /dev/null +++ b/encore-common/src/test/kotlin/se/svt/oss/encore/service/remotefiles/S3UriConverterTest.kt @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2024 Eyevinn Technology AB +// +// SPDX-License-Identifier: EUPL-1.2 + +package se.svt.oss.encore.service.remotefiles + +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import se.svt.oss.encore.service.remotefiles.s3.S3Properties +import se.svt.oss.encore.service.remotefiles.s3.S3UriConverter +import software.amazon.awssdk.regions.Region +import java.net.URI + +class S3UriConverterTest { + + private val s3Properties = S3Properties(enabled = true, anonymousAccess = true) + private val region = Region.of("eu-west-1") + private val s3Uri = URI.create("s3://my-bucket/test2/test1_x264_3100.mp4") + private val s3UriConverter = S3UriConverter(s3Properties, region) + + @Test + fun toHttpReturnsCorrectUri() { + val httpUri = s3UriConverter.toHttp(s3Uri) + assertThat(httpUri) + .isEqualTo("https://my-bucket.s3.eu-west-1.amazonaws.com/test2/test1_x264_3100.mp4") + } + + @Test + fun differentRegionReturnsCorrectUri() { + val s3UriConverter = S3UriConverter(s3Properties, Region.of("eu-north-1")) + + val httpUri = s3UriConverter.toHttp(s3Uri) + assertThat(httpUri) + .isEqualTo("https://my-bucket.s3.eu-north-1.amazonaws.com/test2/test1_x264_3100.mp4") + } + + @Test + fun toHttpWithCustomEndpointReturnsCorrectUri() { + val endpoint = "some-host:1234" + val s3UriConverter = S3UriConverter(s3Properties.copy(endpoint = endpoint), region) + + val httpUri = s3UriConverter.toHttp(s3Uri) + assertThat(httpUri) + .isEqualTo("https://my-bucket.some-host:1234/test2/test1_x264_3100.mp4") + } + + @Test + fun getBucketAndKeyReturnsCorrectValues() { + val (bucket, key) = s3UriConverter.getBucketAndKey(s3Uri) + assertThat(bucket).isEqualTo("my-bucket") + assertThat(key).isEqualTo("test2/test1_x264_3100.mp4") + } + + @Test + fun toHttpNonS3UriThrowsException() { + val uri = URI.create("https://my-bucket/test2/test1_x264_3100.mp4") + assertThatThrownBy { s3UriConverter.toHttp(uri) } + .isInstanceOf(IllegalArgumentException::class.java) + .hasMessage("Invalid URI: $uri") + } + + @Test + fun getBucketAndKeyNonS3UriThrowsException() { + val uri = URI.create("https://my-bucket/test2/test1_x264_3100.mp4") + assertThatThrownBy { s3UriConverter.getBucketAndKey(uri) } + .isInstanceOf(IllegalArgumentException::class.java) + .hasMessage("Invalid URI: $uri") + } +}