From ccec888654c53af118a7bb4651e76828a44316dd Mon Sep 17 00:00:00 2001 From: Benjamin Gaidioz Date: Fri, 15 Nov 2024 11:51:44 +0100 Subject: [PATCH] Revert "Fixed RD-12458: Add awsCredential parameter to S3.Build (#531)" (#536) This reverts commit 34ddcf546138763cc696522ceb06b3936919c3f5. --- project/Dependencies.scala | 2 +- .../compiler/tests/SnapiTestContext.scala | 40 +++++++++-- .../tests/builtin/S3PackageTest.scala | 22 +++--- .../credentials/LocationPackageTest.scala | 33 +++------ .../builtin/credentials/S3PackageTest.scala | 66 +++-------------- .../regressions/credentials/RD5932Test.scala | 6 +- .../extensions/LocationDescription.scala | 33 +++++++-- .../snapi/extensions/builtin/S3Package.scala | 9 +-- .../LocationFromHttpNode.java | 3 +- .../location_package/LocationFromS3Node.java | 70 ++++++++++--------- .../s3_extension/TruffleS3BuildEntry.java | 3 +- 11 files changed, 131 insertions(+), 156 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b551497da..95f465337 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,7 +20,7 @@ object Dependencies { val protocolRaw = "com.raw-labs" %% "protocol-raw" % "0.50.0" - val protocolCompiler = "com.raw-labs" %% "protocol-compiler" % "0.51.0" + val protocolCompiler = "com.raw-labs" %% "protocol-compiler" % "0.50.1" val scalaLogging = "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5" diff --git a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/SnapiTestContext.scala b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/SnapiTestContext.scala index 847507c70..32dc4b4d6 100644 --- a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/SnapiTestContext.scala +++ b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/SnapiTestContext.scala @@ -44,13 +44,14 @@ import com.rawlabs.snapi.compiler._ import com.rawlabs.snapi.frontend.base.source.{BaseProgram, Type} import com.rawlabs.snapi.frontend.inferrer.local.LocalInferrerTestContext import com.rawlabs.protocol.compiler.{ - AwsConfig, DropboxAccessTokenConfig, HttpHeadersConfig, LocationConfig, MySqlConfig, OracleConfig, PostgreSQLConfig, + S3AccessSecretKey, + S3Config, SQLServerConfig, SnowflakeConfig } @@ -85,17 +86,42 @@ object TestCredentials { // Bucket with public access val UnitTestPublicBucket = "rawlabs-public-test-data" + val UnitTestPublicBucketCred = S3Config.newBuilder().setRegion("eu-west-1").build() + // IAM user 'unit-test-private-bucket', which only has permissions only to access bucket 'rawlabs-private-test-data' val UnitTestPrivateBucket = "rawlabs-private-test-data" + val UnitTestPrivateBucketCred = S3Config + .newBuilder() + .setAccessSecretKey(S3AccessSecretKey.newBuilder().setAccessKey(accessKeyId).setSecretKey(secretKeyId)) + .setRegion("eu-west-1") + .build() + val UnitTestPrivateBucket2 = "rawlabs-unit-tests" + val UnitTestPrivateBucket2Cred = S3Config + .newBuilder() + .setAccessSecretKey(S3AccessSecretKey.newBuilder().setAccessKey(accessKeyId).setSecretKey(secretKeyId)) + .setRegion("eu-west-1") + .build() + val UnitTestEmptyBucketPrivateBucket = "rawlabs-unit-test-empty-bucket" + val UnitTestEmptyBucketPrivateBucketCred = S3Config + .newBuilder() + .setAccessSecretKey(S3AccessSecretKey.newBuilder().setAccessKey(accessKeyId).setSecretKey(secretKeyId)) + .setRegion("eu-west-1") + .build() + val UnitTestListRootPrivateBucket = "rawlabs-unit-test-list-root" - val unitTestPrivateBucketUsEast1 = "rawlabs-unit-tests-us-east-1" + val UnitTestListRootPrivateBucketCred = S3Config + .newBuilder() + .setAccessSecretKey(S3AccessSecretKey.newBuilder().setAccessKey(accessKeyId).setSecretKey(secretKeyId)) + .setRegion("eu-west-1") + .build() - val rawAwsCredentials = AwsConfig + val unitTestPrivateBucketUsEast1 = "rawlabs-unit-tests-us-east-1" + val unitTestPrivateBucketUsEast1Cred = S3Config .newBuilder() - .setAccessKey(accessKeyId) - .setSecretKey(secretKeyId) + .setAccessSecretKey(S3AccessSecretKey.newBuilder().setAccessKey(accessKeyId).setSecretKey(secretKeyId)) + .setRegion("us-east-1") .build() /////////////////////////////////////////////////////////////////////////// @@ -207,8 +233,8 @@ trait SnapiTestContext locationConfigs.put(name, locationConfig) } - def awsCreds(name: String, awsConfig: AwsConfig): Unit = { - locationConfig(name, LocationConfig.newBuilder().setAws(awsConfig).build()) + def s3Bucket(name: String, bucket: S3Config): Unit = { + locationConfig(name, LocationConfig.newBuilder().setS3(bucket).build()) } def rdbms(name: String, db: MySqlConfig): Unit = { diff --git a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/S3PackageTest.scala b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/S3PackageTest.scala index 9e9311bd4..bda1862d7 100644 --- a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/S3PackageTest.scala +++ b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/S3PackageTest.scala @@ -18,7 +18,7 @@ class S3PackageTest extends SnapiTestContext { import com.rawlabs.snapi.compiler.tests.TestCredentials._ - awsCreds("raw-aws", rawAwsCredentials) + s3Bucket(UnitTestPrivateBucket, UnitTestPrivateBucketCred) // Reading a public bucket without credentials test(s"""let | data = Csv.InferAndRead( @@ -50,16 +50,16 @@ class S3PackageTest extends SnapiTestContext { | S3.Build( | "$UnitTestPrivateBucket", | "/students.csv", - | region = "eu-west-1", - | accessKey = "${rawAwsCredentials.getAccessKey}", - | secretKey = "${rawAwsCredentials.getSecretKey}" + | region = "${UnitTestPrivateBucketCred.getRegion}", + | accessKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getAccessKey}", + | secretKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getSecretKey}" | ) | ) |in | Collection.Count(data) |""".stripMargin)(it => it should evaluateTo("7")) - // Reading a private bucket without credential + // Reading a private bucket using a registered credential test(s"""let | data = Csv.InferAndRead( | S3.Build( @@ -69,17 +69,11 @@ class S3PackageTest extends SnapiTestContext { | ) |in | Collection.Count(data) - |""".stripMargin)(it => it should runErrorAs("path not authorized")) + |""".stripMargin)(it => it should evaluateTo("7")) - // Reading a private bucket using a registered credential + // Using the automatic casting from url to S3Location using credentials test(s"""let - | data = Csv.InferAndRead( - | S3.Build( - | "$UnitTestPrivateBucket", - | "/students.csv", - | awsCredential = "raw-aws" - | ) - | ) + | data = Csv.InferAndRead("s3://$UnitTestPrivateBucket/students.csv") |in | Collection.Count(data) |""".stripMargin)(it => it should evaluateTo("7")) diff --git a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/LocationPackageTest.scala b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/LocationPackageTest.scala index 6af6d910d..7c7512387 100644 --- a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/LocationPackageTest.scala +++ b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/LocationPackageTest.scala @@ -19,7 +19,8 @@ class LocationPackageTest extends SnapiTestContext { import com.rawlabs.snapi.compiler.tests.TestCredentials._ httpHeaders("dropbox-token", Map("Authorization" -> ("Bearer " + dropboxLongLivedAccessToken))) - awsCreds("raw-aws", rawAwsCredentials) + + s3Bucket(UnitTestPrivateBucket2, UnitTestPrivateBucket2Cred) property("raw.utils.sources.dropbox.clientId", dropboxClientId) @@ -31,22 +32,6 @@ class LocationPackageTest extends SnapiTestContext { | ) |)""".stripMargin)(it => it should run) - test(""" - |String.Read( - | Http.Post( - | "https://api.dropboxapi.com/2/users/get_space_usage", - | authCredentialName = "dropboxToken" // wrong: doesn't exist - | ) - |)""".stripMargin)(it => it should runErrorAs("unknown credential: dropboxToken")) - - test(""" - |String.Read( - | Http.Post( - | "https://api.dropboxapi.com/2/users/get_space_usage", - | authCredentialName = "raw-aws" // wrong: AWS credential - | ) - |)""".stripMargin)(it => it should runErrorAs("credential is not an HTTP headers")) - // reading a public s3 bucket without registering or passing credentials test(s"""let | data = Csv.InferAndRead("s3://$UnitTestPublicBucket/students.csv") @@ -101,9 +86,9 @@ class LocationPackageTest extends SnapiTestContext { | S3.Build( | "$UnitTestPrivateBucket", | "students.csv", - | region = "eu-west-1", - | accessKey = "${rawAwsCredentials.getAccessKey}", - | secretKey = "${rawAwsCredentials.getSecretKey}" + | region = "${UnitTestPrivateBucketCred.getRegion}", + | accessKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getAccessKey}", + | secretKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getSecretKey}" | ) | ) |in @@ -111,14 +96,14 @@ class LocationPackageTest extends SnapiTestContext { |""".stripMargin)(it => it should evaluateTo("7")) // using a private bucket registered in the credentials server - test(s"""String.Read(S3.Build("$UnitTestPrivateBucket2", "/file1.csv", awsCredential = "raw-aws")) + test(s"""String.Read(S3.Build("$UnitTestPrivateBucket2", "/file1.csv")) |""".stripMargin)(it => it should evaluateTo(""" "foobar" """)) test(s"""let dir = S3.Build( | "$UnitTestPrivateBucket", "/publications/publications-hjson/*.json", - | region = "eu-west-1", - | accessKey = "${rawAwsCredentials.getAccessKey}", - | secretKey = "${rawAwsCredentials.getSecretKey}" + | region = "${UnitTestPrivateBucketCred.getRegion}", + | accessKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getAccessKey}", + | secretKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getSecretKey}" | ), | files = Location.Ls(dir), | lines = List.Unnest(files, f -> List.From(String.ReadLines(f))) diff --git a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/S3PackageTest.scala b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/S3PackageTest.scala index 44776decf..a1c5339fe 100644 --- a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/S3PackageTest.scala +++ b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/builtin/credentials/S3PackageTest.scala @@ -19,7 +19,7 @@ class S3PackageTest extends SnapiTestContext { import TestCredentials._ - awsCreds("raw-aws", rawAwsCredentials) + s3Bucket(UnitTestPrivateBucket2, UnitTestPrivateBucket2Cred) // reading a non public s3 bucket passing credentials in the location settings test(s"""let @@ -27,9 +27,9 @@ class S3PackageTest extends SnapiTestContext { | S3.Build( | "$UnitTestPrivateBucket", | "/students.csv", - | region = "eu-west-1", - | accessKey = "${rawAwsCredentials.getAccessKey}", - | secretKey = "${rawAwsCredentials.getSecretKey}" + | region = "${UnitTestPrivateBucketCred.getRegion}", + | accessKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getAccessKey}", + | secretKey = "${UnitTestPrivateBucketCred.getAccessSecretKey.getSecretKey}" | ) | ) |in @@ -37,7 +37,7 @@ class S3PackageTest extends SnapiTestContext { |""".stripMargin)(it => it should evaluateTo("7")) // using a private bucket registered in the credentials server - test(s"""String.Read(S3.Build("$UnitTestPrivateBucket2", "/file1.csv", awsCredential = "raw-aws")) + test(s"""String.Read(S3.Build("$UnitTestPrivateBucket2", "/file1.csv")) |""".stripMargin)(it => it should evaluateTo(""" "foobar" """)) // listing a s3 bucket from us-east-1 (non default region) @@ -46,26 +46,9 @@ class S3PackageTest extends SnapiTestContext { | S3.Build( | "$unitTestPrivateBucketUsEast1", | "/csvs/01", - | region = "us-east-1", - | accessKey = "${rawAwsCredentials.getAccessKey}", - | secretKey = "${rawAwsCredentials.getSecretKey}" - | ) - | ) - |in - | data.url - |""".stripMargin)(it => it should evaluateTo("""[ - | "s3://rawlabs-unit-tests-us-east-1/csvs/01/data2.csv", - | "s3://rawlabs-unit-tests-us-east-1/csvs/01/data1.csv" - |]""".stripMargin)) - - // listing a s3 bucket from us-east-1 (non default region, using the credentials name) - test(s"""let - | data = Location.Ll( - | S3.Build( - | "$unitTestPrivateBucketUsEast1", - | "/csvs/01", - | region = "us-east-1", - | awsCredential = "raw-aws" + | region = "${unitTestPrivateBucketUsEast1Cred.getRegion}", + | accessKey = "${unitTestPrivateBucketUsEast1Cred.getAccessSecretKey.getAccessKey}", + | secretKey = "${unitTestPrivateBucketUsEast1Cred.getAccessSecretKey.getSecretKey}" | ) | ) |in @@ -81,24 +64,8 @@ class S3PackageTest extends SnapiTestContext { | S3.Build( | "$unitTestPrivateBucketUsEast1", | "/csvs/01", - | accessKey = "${rawAwsCredentials.getAccessKey}", - | secretKey = "${rawAwsCredentials.getSecretKey}" - | ) - | ) - |in - | data.url - |""".stripMargin)(it => it should evaluateTo("""[ - | "s3://rawlabs-unit-tests-us-east-1/csvs/01/data2.csv", - | "s3://rawlabs-unit-tests-us-east-1/csvs/01/data1.csv" - |]""".stripMargin)) - - // listing a s3 bucket from us-east-1 without passing the region (using the credentials name) - test(s"""let - | data = Location.Ll( - | S3.Build( - | "$unitTestPrivateBucketUsEast1", - | "/csvs/01", - | awsCredential = "raw-aws" + | accessKey = "${unitTestPrivateBucketUsEast1Cred.getAccessSecretKey.getAccessKey}", + | secretKey = "${unitTestPrivateBucketUsEast1Cred.getAccessSecretKey.getSecretKey}" | ) | ) |in @@ -108,17 +75,4 @@ class S3PackageTest extends SnapiTestContext { | "s3://rawlabs-unit-tests-us-east-1/csvs/01/data1.csv" |]""".stripMargin)) - // error with wrong credential name - test(s"""let - | data = Location.Ll( - | S3.Build( - | "$unitTestPrivateBucketUsEast1", - | "/csvs/01", - | awsCredential = "private.data.us" // a typo - | ) - | ) - |in - | data.url - |""".stripMargin)(it => it should runErrorAs("unknown credential: private.data.us")) - } diff --git a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/regressions/credentials/RD5932Test.scala b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/regressions/credentials/RD5932Test.scala index fd171a1b3..590edfbdd 100644 --- a/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/regressions/credentials/RD5932Test.scala +++ b/snapi-compiler/src/test/scala/com/rawlabs/snapi/compiler/tests/regressions/credentials/RD5932Test.scala @@ -17,11 +17,9 @@ import com.rawlabs.snapi.compiler.tests.SnapiTestContext class RD5932Test extends SnapiTestContext { - awsCreds("private-creds", TestCredentials.rawAwsCredentials) + s3Bucket(TestCredentials.UnitTestPrivateBucket, TestCredentials.UnitTestPrivateBucketCred) - test( - """Json.InferAndRead(S3.Build("rawlabs-private-test-data", "rd-5932.json", awsCredential = "private-creds"))""" - ) { + test("""Json.InferAndRead("s3://rawlabs-private-test-data/rd-5932.json")""") { _ should run } diff --git a/snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/LocationDescription.scala b/snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/LocationDescription.scala index 302ceb392..634fdef8c 100644 --- a/snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/LocationDescription.scala +++ b/snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/LocationDescription.scala @@ -604,19 +604,42 @@ object LocationDescription { val userInfoParts = uriUserInfo.split(":") maybeAccessKey = Some(userInfoParts(0)) if (maybeAccessKey.get.isEmpty) { - return Left("missing AWS access key") + return Left("missing S3 access key") } if (userInfoParts.length > 1) { maybeSecretKey = Some(userInfoParts(1)) if (maybeSecretKey.get.isEmpty) { - return Left("missing AWS secret key") + return Left("missing S3 secret key") } } else { - return Left("missing AWS secret key") + return Left("missing S3 secret key") } } - // TODO (msb): There is no way to specify the region when using a direct URL... - Right(S3PathLocationDescription(bucketName, None, maybeAccessKey, maybeSecretKey, objectKey)) + + if (maybeAccessKey.isEmpty) { + // If the access key/secret key are not defined, check if credential exists. + programEnvironment.locationConfigs.get(bucketName) match { + case Some(l) if l.hasS3 => + val s3Credential = l.getS3 + Right( + S3PathLocationDescription( + bucketName, + if (s3Credential.hasRegion) Some(s3Credential.getRegion) else None, + if (s3Credential.hasAccessSecretKey) Some(s3Credential.getAccessSecretKey.getAccessKey) else None, + if (s3Credential.hasAccessSecretKey) Some(s3Credential.getAccessSecretKey.getSecretKey) else None, + objectKey + ) + ) + case Some(l) if l.hasError => Left(l.getError.getMessage) + case Some(_) => Left("not a S3 credential") + case None => + // Anonymous access. + Right(S3PathLocationDescription(bucketName, None, None, None, objectKey)) + } + } else { + // TODO (msb): There is no way to specify the region when using a direct URL... + Right(S3PathLocationDescription(bucketName, None, maybeAccessKey, maybeSecretKey, objectKey)) + } case "dropbox" => // In Dropbox, the host is the name of the credential val DROPBOX_REGEX(name, path) = url diff --git a/snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/builtin/S3Package.scala b/snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/builtin/S3Package.scala index 97393e8a6..4e5e19804 100644 --- a/snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/builtin/S3Package.scala +++ b/snapi-frontend/src/main/scala/com/rawlabs/snapi/frontend/snapi/extensions/builtin/S3Package.scala @@ -42,12 +42,6 @@ class S3BuildEntry extends EntryExtension { params = List( ParamDoc("bucket", TypeDoc(List("string")), "The name of the bucket."), ParamDoc("path", TypeDoc(List("string")), "The path to the file in the bucket."), - ParamDoc( - "awsCredential", - TypeDoc(List("string")), - "The name of the AWS credential registered in the credentials storage.", - isOptional = true - ), ParamDoc("region", TypeDoc(List("string")), "The region of the bucket, e.g. 'eu-west-1'.", isOptional = true), ParamDoc("accessKey", TypeDoc(List("string")), "The AWS access key.", isOptional = true), ParamDoc("secretKey", TypeDoc(List("string")), "The AWS secret key.", isOptional = true) @@ -66,10 +60,9 @@ class S3BuildEntry extends EntryExtension { Right(ExpParam(SnapiStringType())) } - override def optionalParams: Option[Set[String]] = Some(Set("region", "accessKey", "secretKey", "awsCredential")) + override def optionalParams: Option[Set[String]] = Some(Set("region", "accessKey", "secretKey")) override def getOptionalParam(prevMandatoryArgs: Seq[Arg], idn: String): Either[String, Param] = idn match { - case "awsCredential" => Right(ExpParam(SnapiStringType())) case "region" => Right(ExpParam(SnapiStringType())) case "accessKey" => Right(ExpParam(SnapiStringType())) case "secretKey" => Right(ExpParam(SnapiStringType())) diff --git a/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationFromHttpNode.java b/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationFromHttpNode.java index a83cea403..e83d0a5d5 100644 --- a/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationFromHttpNode.java +++ b/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationFromHttpNode.java @@ -23,7 +23,6 @@ import com.rawlabs.snapi.truffle.SnapiContext; import com.rawlabs.snapi.truffle.ast.ExpressionNode; import com.rawlabs.snapi.truffle.runtime.exceptions.TruffleInternalErrorException; -import com.rawlabs.snapi.truffle.runtime.exceptions.TruffleRuntimeException; import com.rawlabs.snapi.truffle.runtime.list.ListNodes; import com.rawlabs.snapi.truffle.runtime.list.ListNodesFactory; import com.rawlabs.snapi.truffle.runtime.primitives.*; @@ -180,7 +179,7 @@ public Object executeGeneric(VirtualFrame frame) { headersBuilder.$plus$eq(Tuple2.apply(entry.getKey(), entry.getValue())); } } else { - throw new TruffleRuntimeException("credential is not an HTTP headers"); + throw new TruffleInternalErrorException("credential is not an HTTP headers"); } } diff --git a/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationFromS3Node.java b/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationFromS3Node.java index 8f8135bb8..0d16e2027 100644 --- a/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationFromS3Node.java +++ b/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/ast/expressions/builtin/location_package/LocationFromS3Node.java @@ -14,11 +14,10 @@ import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.NodeInfo; -import com.rawlabs.protocol.compiler.AwsConfig; import com.rawlabs.protocol.compiler.LocationConfig; +import com.rawlabs.protocol.compiler.S3Config; import com.rawlabs.snapi.truffle.SnapiContext; import com.rawlabs.snapi.truffle.ast.ExpressionNode; -import com.rawlabs.snapi.truffle.runtime.exceptions.TruffleRuntimeException; import com.rawlabs.snapi.truffle.runtime.primitives.*; import com.rawlabs.utils.sources.filesystem.s3.S3Path; import scala.None$; @@ -30,7 +29,6 @@ public class LocationFromS3Node extends ExpressionNode { @Child private ExpressionNode bucket; @Child private ExpressionNode path; - @Child private ExpressionNode awsCredential; @Child private ExpressionNode accessKey; @Child private ExpressionNode secretKey; @Child private ExpressionNode region; @@ -38,13 +36,11 @@ public class LocationFromS3Node extends ExpressionNode { public LocationFromS3Node( ExpressionNode bucket, ExpressionNode path, - ExpressionNode awsCredential, ExpressionNode accessKey, ExpressionNode secretKey, ExpressionNode region) { this.bucket = bucket; this.path = path; - this.awsCredential = awsCredential; this.accessKey = accessKey; this.secretKey = secretKey; this.region = region; @@ -55,36 +51,44 @@ public Object executeGeneric(VirtualFrame frame) { String bucket = (String) this.bucket.executeGeneric(frame); String path = (String) this.path.executeGeneric(frame); + // The docs say: + // "If the S3 bucket is not registered in the credentials storage, then the region, accessKey + // and secretKey must be provided as arguments." + // However, if the access key/secret key are passed, they should be used. SnapiContext context = SnapiContext.get(this); - - Option maybeAccessKey = None$.empty(); - Option maybeSecretKey = None$.empty(); - Option maybeRegion = None$.empty(); - if (this.awsCredential != null) { - String credName = (String) this.awsCredential.executeGeneric(frame); - // getLocationConfig throws if the credential isn't found. - LocationConfig locationConfig = context.getLocationConfig(credName); - if (!locationConfig.hasAws()) { - throw new TruffleRuntimeException("credential is not an AWS credential"); - } - AwsConfig awsConfig = locationConfig.getAws(); - maybeAccessKey = new Some(awsConfig.getAccessKey()); - maybeSecretKey = new Some(awsConfig.getSecretKey()); - } - - if (this.accessKey != null) { - maybeAccessKey = new Some(this.accessKey.executeGeneric(frame)); + S3Path location; + if (this.accessKey == null + && this.secretKey == null + && context.existsLocationConfig(bucket) + && context.getLocationConfig(bucket).hasS3()) { + LocationConfig l = context.getLocationConfig(bucket); + S3Config s3Config = l.getS3(); + Option maybeAccessKey = + (s3Config.hasAccessSecretKey()) + ? new Some(s3Config.getAccessSecretKey().getAccessKey()) + : None$.empty(); + Option maybeSecretKey = + (s3Config.hasAccessSecretKey()) + ? new Some(s3Config.getAccessSecretKey().getSecretKey()) + : None$.empty(); + Option maybeRegion = + (s3Config.hasRegion()) ? new Some(s3Config.getRegion()) : None$.empty(); + location = + new S3Path( + bucket, maybeRegion, maybeAccessKey, maybeSecretKey, path, context.getSettings()); + } else { + // We actually do NOT throw an exception if the accessKey is not passed. + // Instead, we go without it, which triggers anonymous access to the S3 bucket. + Option maybeAccessKey = + (this.accessKey != null) ? new Some(this.accessKey.executeGeneric(frame)) : None$.empty(); + Option maybeSecretKey = + (this.secretKey != null) ? new Some(this.secretKey.executeGeneric(frame)) : None$.empty(); + Option maybeRegion = + (this.region != null) ? new Some(this.region.executeGeneric(frame)) : None$.empty(); + location = + new S3Path( + bucket, maybeRegion, maybeAccessKey, maybeSecretKey, path, context.getSettings()); } - if (this.secretKey != null) { - maybeSecretKey = new Some(this.secretKey.executeGeneric(frame)); - } - if (this.region != null) { - maybeRegion = new Some(this.region.executeGeneric(frame)); - } - - S3Path location = - new S3Path( - bucket, maybeRegion, maybeAccessKey, maybeSecretKey, path, context.getSettings()); StringBuilder url = new StringBuilder(); url.append("s3://"); diff --git a/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/emitter/builtin/s3_extension/TruffleS3BuildEntry.java b/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/emitter/builtin/s3_extension/TruffleS3BuildEntry.java index 58c04b65a..f47f5736b 100644 --- a/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/emitter/builtin/s3_extension/TruffleS3BuildEntry.java +++ b/snapi-truffle/src/main/java/com/rawlabs/snapi/truffle/emitter/builtin/s3_extension/TruffleS3BuildEntry.java @@ -27,11 +27,10 @@ public class TruffleS3BuildEntry extends S3BuildEntry implements TruffleEntryExt public ExpressionNode toTruffle(Type type, List args, SnapiLanguage rawLanguage) { ExpressionNode bucket = args.get(0).exprNode(); ExpressionNode path = args.get(1).exprNode(); - ExpressionNode awsCredential = arg(args, "awsCredential").orElse(null); ExpressionNode accessKey = arg(args, "accessKey").orElse(null); ExpressionNode secretKey = arg(args, "secretKey").orElse(null); ExpressionNode region = arg(args, "region").orElse(null); - return new LocationFromS3Node(bucket, path, awsCredential, accessKey, secretKey, region); + return new LocationFromS3Node(bucket, path, accessKey, secretKey, region); } }