diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml b/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml index 338f56ee16d7..9228111f342b 100644 --- a/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml +++ b/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml @@ -16,7 +16,7 @@ data: alias: airbyte-connector-testing-secret-store connectorType: destination definitionId: 37a928c1-2d5c-431a-a97d-ae236bd1ea0c - dockerImageTag: 0.2.5 + dockerImageTag: 0.2.6 dockerRepository: airbyte/destination-iceberg-v2 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3 githubIssueLabel: destination-iceberg-v2 diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt b/airbyte-integrations/connectors/destination-iceberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt index c3b3d627f827..4288204341e9 100644 --- a/airbyte-integrations/connectors/destination-iceberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt +++ b/airbyte-integrations/connectors/destination-iceberg-v2/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt @@ -186,22 +186,15 @@ class IcebergUtil(private val tableIdGenerator: TableIdGenerator) { fun toCatalogProperties(config: IcebergV2Configuration): Map { val icebergCatalogConfig = config.icebergCatalogConfiguration val catalogConfig = icebergCatalogConfig.catalogConfiguration - val awsAccessKeyId = - requireNotNull(config.awsAccessKeyConfiguration.accessKeyId) { - "AWS Access Key ID cannot be null" - } - val awsSecretAccessKey = - requireNotNull(config.awsAccessKeyConfiguration.secretAccessKey) { - "AWS Secret Access Key cannot be null" - } + val awsAccessKeyId = config.awsAccessKeyConfiguration.accessKeyId + val awsSecretAccessKey = config.awsAccessKeyConfiguration.secretAccessKey // Common S3/Iceberg properties shared across all catalog types. // The S3 endpoint is optional; if provided, it will be included. val s3CommonProperties = mutableMapOf( CatalogProperties.FILE_IO_IMPL to S3FileIO::class.java.name, - S3FileIOProperties.ACCESS_KEY_ID to awsAccessKeyId, - S3FileIOProperties.SECRET_ACCESS_KEY to awsSecretAccessKey, + // Required for MinIO or other S3-compatible stores using path-style access. S3FileIOProperties.PATH_STYLE_ACCESS to "true" ) @@ -211,6 +204,15 @@ class IcebergUtil(private val tableIdGenerator: TableIdGenerator) { } } + // If we don't have explicit S3 creds, fall back to the default creds provider chain. + // For example, this should allow us to use AWS instance profiles. + // TODO add support for our IAM assume role feature + // (https://github.com/airbytehq/airbyte-internal-issues/issues/11082) + if (awsAccessKeyId != null && awsSecretAccessKey != null) { + s3CommonProperties[S3FileIOProperties.ACCESS_KEY_ID] = awsAccessKeyId + s3CommonProperties[S3FileIOProperties.SECRET_ACCESS_KEY] = awsSecretAccessKey + } + return when (catalogConfig) { is NessieCatalogConfiguration -> { // Nessie relies on the AWS region being set as a system property.