diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt
index 665ccc0..94eda9c 100644
--- a/encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt
+++ b/encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt
@@ -10,6 +10,9 @@ import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import se.svt.oss.encore.service.remotefiles.s3.S3Properties
import se.svt.oss.encore.service.remotefiles.s3.S3RemoteFileHandler
+import se.svt.oss.encore.service.remotefiles.s3.S3UriConverter
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
import software.amazon.awssdk.regions.Region
import software.amazon.awssdk.services.s3.S3AsyncClient
import software.amazon.awssdk.services.s3.S3Configuration
@@ -35,6 +38,13 @@ class S3RemoteFilesConfiguration {
.pathStyleAccessEnabled(true)
.build()
)
+ .credentialsProvider(
+ if (s3Properties.anonymousAccess) {
+ AnonymousCredentialsProvider.create()
+ } else {
+ DefaultCredentialsProvider.create()
+ }
+ )
.apply {
if (!s3Properties.endpoint.isNullOrBlank()) {
endpointOverride(URI.create(s3Properties.endpoint))
@@ -58,6 +68,9 @@ class S3RemoteFilesConfiguration {
.build()
@Bean
- fun s3RemoteFileHandler(s3Client: S3AsyncClient, s3Presigner: S3Presigner, s3Properties: S3Properties) =
- S3RemoteFileHandler(s3Client, s3Presigner, s3Properties)
+ fun s3UriConverter(s3Properties: S3Properties, s3Region: Region) = S3UriConverter(s3Properties, s3Region)
+
+ @Bean
+ fun s3RemoteFileHandler(s3Client: S3AsyncClient, s3Presigner: S3Presigner, s3Properties: S3Properties, s3UriConverter: S3UriConverter) =
+ S3RemoteFileHandler(s3Client, s3Presigner, s3Properties, s3UriConverter)
}
diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt
index 06c2d5b..77fb8c9 100644
--- a/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt
+++ b/encore-common/src/main/kotlin/se/svt/oss/encore/model/input/Input.kt
@@ -8,14 +8,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.annotation.JsonSubTypes
import com.fasterxml.jackson.annotation.JsonTypeInfo
import io.swagger.v3.oas.annotations.media.Schema
+import jakarta.validation.constraints.Pattern
+import jakarta.validation.constraints.PositiveOrZero
import se.svt.oss.encore.model.mediafile.toParams
import se.svt.oss.encore.model.profile.ChannelLayout
import se.svt.oss.mediaanalyzer.file.FractionString
import se.svt.oss.mediaanalyzer.file.MediaContainer
import se.svt.oss.mediaanalyzer.file.MediaFile
import se.svt.oss.mediaanalyzer.file.VideoFile
-import jakarta.validation.constraints.Pattern
-import jakarta.validation.constraints.PositiveOrZero
const val TYPE_AUDIO_VIDEO = "AudioVideo"
const val TYPE_AUDIO = "Audio"
@@ -255,7 +255,7 @@ fun List.inputParams(readDuration: Double?): List =
(readDuration?.let { listOf("-t", "$it") } ?: emptyList()) +
(input.seekTo?.let { listOf("-ss", "$it") } ?: emptyList()) +
(if (input.copyTs) listOf("-copyts") else emptyList()) +
- listOf("-i", input.accessUri ?: input.uri)
+ listOf("-i", input.accessUri)
}
fun List.maxDuration(): Double? = maxOfOrNull {
diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt
index a3c3383..82eb5e3 100644
--- a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt
+++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt
@@ -10,6 +10,7 @@ import java.time.Duration
@ConfigurationProperties("remote-files.s3")
data class S3Properties(
val enabled: Boolean = false,
+ val anonymousAccess: Boolean = false,
val endpoint: String = "",
val presignDurationSeconds: Long = Duration.ofHours(12).seconds,
val uploadTimeoutSeconds: Long = Duration.ofHours(1).seconds
diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3RemoteFileHandler.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3RemoteFileHandler.kt
index 43c5b06..52a5e22 100644
--- a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3RemoteFileHandler.kt
+++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3RemoteFileHandler.kt
@@ -17,7 +17,8 @@ import java.util.concurrent.TimeUnit
class S3RemoteFileHandler(
private val client: S3AsyncClient,
private val presigner: S3Presigner,
- private val s3Properties: S3Properties
+ private val s3Properties: S3Properties,
+ private val s3UriConverter: S3UriConverter
) : RemoteFileHandler {
private val log = mu.KotlinLogging.logger {}
@@ -25,9 +26,18 @@ class S3RemoteFileHandler(
override fun getAccessUri(uri: String): String {
val s3Uri = URI.create(uri)
+ if (s3Properties.anonymousAccess) {
+ return s3UriConverter.toHttp(s3Uri)
+ }
+ return presignUrl(s3Uri)
+ }
+
+ private fun presignUrl(s3Uri: URI): String {
+ val (bucket, key) = s3UriConverter.getBucketAndKey(s3Uri)
+
val objectRequest: GetObjectRequest = GetObjectRequest.builder()
- .bucket(s3Uri.host)
- .key(s3Uri.path.stripLeadingSlash())
+ .bucket(bucket)
+ .key(key)
.build()
val presignRequest: GetObjectPresignRequest = GetObjectPresignRequest.builder()
.signatureDuration(java.time.Duration.ofSeconds(s3Properties.presignDurationSeconds))
@@ -42,17 +52,14 @@ class S3RemoteFileHandler(
override fun upload(localFile: String, remoteFile: String) {
log.info { "Uploading $localFile to $remoteFile" }
val s3Uri = URI.create(remoteFile)
- val bucket = s3Uri.host
- val objectName = s3Uri.path.stripLeadingSlash()
+ val (bucket, key) = s3UriConverter.getBucketAndKey(s3Uri)
val putObjectRequest: PutObjectRequest = PutObjectRequest.builder()
.bucket(bucket)
- .key(objectName)
+ .key(key)
.build()
val res = client.putObject(putObjectRequest, Paths.get(localFile)).get(s3Properties.presignDurationSeconds, TimeUnit.SECONDS)
log.info { "Upload result: $res" }
}
- private fun String.stripLeadingSlash() = if (startsWith("/")) substring(1) else this
-
override val protocols = listOf("s3")
}
diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3UriConverter.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3UriConverter.kt
new file mode 100644
index 0000000..0727cfc
--- /dev/null
+++ b/encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3UriConverter.kt
@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: 2024 Eyevinn Technology AB
+//
+// SPDX-License-Identifier: EUPL-1.2
+
+package se.svt.oss.encore.service.remotefiles.s3
+
+import software.amazon.awssdk.regions.Region
+import java.net.URI
+
+class S3UriConverter(
+ private val s3Properties: S3Properties,
+ private val region: Region
+) {
+
+ fun toHttp(s3Uri: URI): String {
+ if (s3Uri.scheme != "s3") {
+ throw IllegalArgumentException("Invalid URI: $s3Uri")
+ }
+ val bucket = s3Uri.host
+ val key = s3Uri.path.stripLeadingSlash()
+
+ if (s3Properties.endpoint.isNotBlank()) {
+ return "https://$bucket.${s3Properties.endpoint}/$key"
+ }
+ return "https://$bucket.s3.$region.amazonaws.com/$key"
+ }
+
+ fun getBucketAndKey(s3Uri: URI): Pair {
+ if (s3Uri.scheme != "s3") {
+ throw IllegalArgumentException("Invalid URI: $s3Uri")
+ }
+ val bucket = s3Uri.host
+ val key = s3Uri.path.stripLeadingSlash()
+ return Pair(bucket, key)
+ }
+
+ private fun String.stripLeadingSlash() = if (startsWith("/")) substring(1) else this
+}
diff --git a/encore-common/src/test/kotlin/se/svt/oss/encore/service/remotefiles/S3UriConverterTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/service/remotefiles/S3UriConverterTest.kt
new file mode 100644
index 0000000..c67b9c9
--- /dev/null
+++ b/encore-common/src/test/kotlin/se/svt/oss/encore/service/remotefiles/S3UriConverterTest.kt
@@ -0,0 +1,70 @@
+// SPDX-FileCopyrightText: 2024 Eyevinn Technology AB
+//
+// SPDX-License-Identifier: EUPL-1.2
+
+package se.svt.oss.encore.service.remotefiles
+
+import org.assertj.core.api.Assertions.assertThat
+import org.assertj.core.api.Assertions.assertThatThrownBy
+import org.junit.jupiter.api.Test
+import se.svt.oss.encore.service.remotefiles.s3.S3Properties
+import se.svt.oss.encore.service.remotefiles.s3.S3UriConverter
+import software.amazon.awssdk.regions.Region
+import java.net.URI
+
+class S3UriConverterTest {
+
+ private val s3Properties = S3Properties(enabled = true, anonymousAccess = true)
+ private val region = Region.of("eu-west-1")
+ private val s3Uri = URI.create("s3://my-bucket/test2/test1_x264_3100.mp4")
+ private val s3UriConverter = S3UriConverter(s3Properties, region)
+
+ @Test
+ fun toHttpReturnsCorrectUri() {
+ val httpUri = s3UriConverter.toHttp(s3Uri)
+ assertThat(httpUri)
+ .isEqualTo("https://my-bucket.s3.eu-west-1.amazonaws.com/test2/test1_x264_3100.mp4")
+ }
+
+ @Test
+ fun differentRegionReturnsCorrectUri() {
+ val s3UriConverter = S3UriConverter(s3Properties, Region.of("eu-north-1"))
+
+ val httpUri = s3UriConverter.toHttp(s3Uri)
+ assertThat(httpUri)
+ .isEqualTo("https://my-bucket.s3.eu-north-1.amazonaws.com/test2/test1_x264_3100.mp4")
+ }
+
+ @Test
+ fun toHttpWithCustomEndpointReturnsCorrectUri() {
+ val endpoint = "some-host:1234"
+ val s3UriConverter = S3UriConverter(s3Properties.copy(endpoint = endpoint), region)
+
+ val httpUri = s3UriConverter.toHttp(s3Uri)
+ assertThat(httpUri)
+ .isEqualTo("https://my-bucket.some-host:1234/test2/test1_x264_3100.mp4")
+ }
+
+ @Test
+ fun getBucketAndKeyReturnsCorrectValues() {
+ val (bucket, key) = s3UriConverter.getBucketAndKey(s3Uri)
+ assertThat(bucket).isEqualTo("my-bucket")
+ assertThat(key).isEqualTo("test2/test1_x264_3100.mp4")
+ }
+
+ @Test
+ fun toHttpNonS3UriThrowsException() {
+ val uri = URI.create("https://my-bucket/test2/test1_x264_3100.mp4")
+ assertThatThrownBy { s3UriConverter.toHttp(uri) }
+ .isInstanceOf(IllegalArgumentException::class.java)
+ .hasMessage("Invalid URI: $uri")
+ }
+
+ @Test
+ fun getBucketAndKeyNonS3UriThrowsException() {
+ val uri = URI.create("https://my-bucket/test2/test1_x264_3100.mp4")
+ assertThatThrownBy { s3UriConverter.getBucketAndKey(uri) }
+ .isInstanceOf(IllegalArgumentException::class.java)
+ .hasMessage("Invalid URI: $uri")
+ }
+}