forked from svt/encore
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: support for enabling anonymous access for s3 urls
By setting env variable REMOTEFILES_S3_ANONYMOUSACCESS to true, s3 urls will be accessed in anonymous mode, corresponding to using the '--no-sign-request' flag with the aws cli. Any s3 access key or secrets key configured will be ignored. Signed-off-by: Gustav Grusell <[email protected]>
- Loading branch information
Showing
6 changed files
with
142 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
38 changes: 38 additions & 0 deletions
38
encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3UriConverter.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| // SPDX-FileCopyrightText: 2024 Eyevinn Technology AB | ||
| // | ||
| // SPDX-License-Identifier: EUPL-1.2 | ||
|
|
||
| package se.svt.oss.encore.service.remotefiles.s3 | ||
|
|
||
| import software.amazon.awssdk.regions.Region | ||
| import java.net.URI | ||
|
|
||
| class S3UriConverter( | ||
| private val s3Properties: S3Properties, | ||
| private val region: Region | ||
| ) { | ||
|
|
||
| fun toHttp(s3Uri: URI): String { | ||
| if (s3Uri.scheme != "s3") { | ||
| throw IllegalArgumentException("Invalid URI: $s3Uri") | ||
| } | ||
| val bucket = s3Uri.host | ||
| val key = s3Uri.path.stripLeadingSlash() | ||
|
|
||
| if (s3Properties.endpoint.isNotBlank()) { | ||
| return "https://$bucket.${s3Properties.endpoint}/$key" | ||
| } | ||
| return "https://$bucket.s3.$region.amazonaws.com/$key" | ||
| } | ||
|
|
||
| fun getBucketAndKey(s3Uri: URI): Pair<String, String> { | ||
| if (s3Uri.scheme != "s3") { | ||
| throw IllegalArgumentException("Invalid URI: $s3Uri") | ||
| } | ||
| val bucket = s3Uri.host | ||
| val key = s3Uri.path.stripLeadingSlash() | ||
| return Pair(bucket, key) | ||
| } | ||
|
|
||
| private fun String.stripLeadingSlash() = if (startsWith("/")) substring(1) else this | ||
| } |
70 changes: 70 additions & 0 deletions
70
encore-common/src/test/kotlin/se/svt/oss/encore/service/remotefiles/S3UriConverterTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| // SPDX-FileCopyrightText: 2024 Eyevinn Technology AB | ||
| // | ||
| // SPDX-License-Identifier: EUPL-1.2 | ||
|
|
||
| package se.svt.oss.encore.service.remotefiles | ||
|
|
||
| import org.assertj.core.api.Assertions.assertThat | ||
| import org.assertj.core.api.Assertions.assertThatThrownBy | ||
| import org.junit.jupiter.api.Test | ||
| import se.svt.oss.encore.service.remotefiles.s3.S3Properties | ||
| import se.svt.oss.encore.service.remotefiles.s3.S3UriConverter | ||
| import software.amazon.awssdk.regions.Region | ||
| import java.net.URI | ||
|
|
||
| class S3UriConverterTest { | ||
|
|
||
| private val s3Properties = S3Properties(enabled = true, anonymousAccess = true) | ||
| private val region = Region.of("eu-west-1") | ||
| private val s3Uri = URI.create("s3://my-bucket/test2/test1_x264_3100.mp4") | ||
| private val s3UriConverter = S3UriConverter(s3Properties, region) | ||
|
|
||
| @Test | ||
| fun toHttpReturnsCorrectUri() { | ||
| val httpUri = s3UriConverter.toHttp(s3Uri) | ||
| assertThat(httpUri) | ||
| .isEqualTo("https://my-bucket.s3.eu-west-1.amazonaws.com/test2/test1_x264_3100.mp4") | ||
| } | ||
|
|
||
| @Test | ||
| fun differentRegionReturnsCorrectUri() { | ||
| val s3UriConverter = S3UriConverter(s3Properties, Region.of("eu-north-1")) | ||
|
|
||
| val httpUri = s3UriConverter.toHttp(s3Uri) | ||
| assertThat(httpUri) | ||
| .isEqualTo("https://my-bucket.s3.eu-north-1.amazonaws.com/test2/test1_x264_3100.mp4") | ||
| } | ||
|
|
||
| @Test | ||
| fun toHttpWithCustomEndpointReturnsCorrectUri() { | ||
| val endpoint = "some-host:1234" | ||
| val s3UriConverter = S3UriConverter(s3Properties.copy(endpoint = endpoint), region) | ||
|
|
||
| val httpUri = s3UriConverter.toHttp(s3Uri) | ||
| assertThat(httpUri) | ||
| .isEqualTo("https://my-bucket.some-host:1234/test2/test1_x264_3100.mp4") | ||
| } | ||
|
|
||
| @Test | ||
| fun getBucketAndKeyReturnsCorrectValues() { | ||
| val (bucket, key) = s3UriConverter.getBucketAndKey(s3Uri) | ||
| assertThat(bucket).isEqualTo("my-bucket") | ||
| assertThat(key).isEqualTo("test2/test1_x264_3100.mp4") | ||
| } | ||
|
|
||
| @Test | ||
| fun toHttpNonS3UriThrowsException() { | ||
| val uri = URI.create("https://my-bucket/test2/test1_x264_3100.mp4") | ||
| assertThatThrownBy { s3UriConverter.toHttp(uri) } | ||
| .isInstanceOf(IllegalArgumentException::class.java) | ||
| .hasMessage("Invalid URI: $uri") | ||
| } | ||
|
|
||
| @Test | ||
| fun getBucketAndKeyNonS3UriThrowsException() { | ||
| val uri = URI.create("https://my-bucket/test2/test1_x264_3100.mp4") | ||
| assertThatThrownBy { s3UriConverter.getBucketAndKey(uri) } | ||
| .isInstanceOf(IllegalArgumentException::class.java) | ||
| .hasMessage("Invalid URI: $uri") | ||
| } | ||
| } |