maybe support aws instance profile auth?

airbytehq · Jan 3, 2025 · 77b3b60 · 77b3b60
1 parent 73cefa5
commit 77b3b60
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 11 deletions.
diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml b/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml
@@ -16,7 +16,7 @@ data:
             alias: airbyte-connector-testing-secret-store
   connectorType: destination
   definitionId: 37a928c1-2d5c-431a-a97d-ae236bd1ea0c
-  dockerImageTag: 0.2.5
+  dockerImageTag: 0.2.6
   dockerRepository: airbyte/destination-iceberg-v2
   documentationUrl: https://docs.airbyte.com/integrations/destinations/s3
   githubIssueLabel: destination-iceberg-v2

diff --git a/...eberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt b/...eberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt
@@ -186,22 +186,15 @@ class IcebergUtil(private val tableIdGenerator: TableIdGenerator) {
     fun toCatalogProperties(config: IcebergV2Configuration): Map<String, String> {
         val icebergCatalogConfig = config.icebergCatalogConfiguration
         val catalogConfig = icebergCatalogConfig.catalogConfiguration
-        val awsAccessKeyId =
-            requireNotNull(config.awsAccessKeyConfiguration.accessKeyId) {
-                "AWS Access Key ID cannot be null"
-            }
-        val awsSecretAccessKey =
-            requireNotNull(config.awsAccessKeyConfiguration.secretAccessKey) {
-                "AWS Secret Access Key cannot be null"
-            }
+        val awsAccessKeyId = config.awsAccessKeyConfiguration.accessKeyId
+        val awsSecretAccessKey = config.awsAccessKeyConfiguration.secretAccessKey
 
         // Common S3/Iceberg properties shared across all catalog types.
         // The S3 endpoint is optional; if provided, it will be included.
         val s3CommonProperties =
             mutableMapOf<String, String>(
                     CatalogProperties.FILE_IO_IMPL to S3FileIO::class.java.name,
-                    S3FileIOProperties.ACCESS_KEY_ID to awsAccessKeyId,
-                    S3FileIOProperties.SECRET_ACCESS_KEY to awsSecretAccessKey,
+
                     // Required for MinIO or other S3-compatible stores using path-style access.
                     S3FileIOProperties.PATH_STYLE_ACCESS to "true"
                 )
@@ -211,6 +204,15 @@ class IcebergUtil(private val tableIdGenerator: TableIdGenerator) {
                     }
                 }
 
+        // If we don't have explicit S3 creds, fall back to the default creds provider chain.
+        // For example, this should allow us to use AWS instance profiles.
+        // TODO add support for our IAM assume role feature
+        //   (https://github.com/airbytehq/airbyte-internal-issues/issues/11082)
+        if (awsAccessKeyId != null && awsSecretAccessKey != null) {
+            s3CommonProperties[S3FileIOProperties.ACCESS_KEY_ID] = awsAccessKeyId
+            s3CommonProperties[S3FileIOProperties.SECRET_ACCESS_KEY] = awsSecretAccessKey
+        }
+
         return when (catalogConfig) {
             is NessieCatalogConfiguration -> {
                 // Nessie relies on the AWS region being set as a system property.