Skip to content

Commit

Permalink
Merge cockroachdb#125745
Browse files Browse the repository at this point in the history 
125745: changefeedccl: add support for IAM role-based authentication to Kafka sink r=asg0451,rharding6373 a=cty123

This patch adds support to the go Kafka client(Sarama) to authenticate
with AWS IAM roles. AWS IAM role authentication is the recommended
way to connect with AWS MSK cluster as it's more secure than the
SASL username + password method.

AWS now provides a library in a repository under their official account
https://github.com/aws/aws-msk-iam-sasl-signer-go that is able to
connect to AWS MSK with IAM roles. This PR introduces the new
dependency on the library and provides option in Kafka DialConfig
to enable the IAM role authentcaition.

Resolves: cockroachdb#124431

Release note (enterprise change): Add support to the Kafka sink
to authenticate with MSK via AWS IAM roles


Co-authored-by: cty123 <[email protected]>
  • Loading branch information
@cty123
craig[bot] and cty123 committed Jul 9, 2024
2 parents 43a80c2 + 09a2b4a commit f87a559
Show file tree
Hide file tree
Showing 9 changed files with 201 additions and 92 deletions.
80 changes: 50 additions & 30 deletions DEPS.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,16 @@ def go_deps():
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-lambda-go/com_github_aws_aws_lambda_go-v1.13.3.zip",
],
)
go_repository(
name = "com_github_aws_aws_msk_iam_sasl_signer_go",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-msk-iam-sasl-signer-go",
sha256 = "b5f99e40aae3664b1a58b312efda28e432b4e976dd3296e24520cc79b9651a14",
strip_prefix = "github.com/aws/[email protected]",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-msk-iam-sasl-signer-go/com_github_aws_aws_msk_iam_sasl_signer_go-v1.0.0.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go",
build_file_proto_mode = "disable_global",
Expand All @@ -473,70 +483,70 @@ def go_deps():
name = "com_github_aws_aws_sdk_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2",
sha256 = "db9f1deea94002f8a40a665ab86b58d9c713fda9358997c75b4859668d66802a",
strip_prefix = "github.com/aws/aws-sdk-go-v2@v1.17.3",
sha256 = "a000599cb80b0466affe6baf2fa98d79258ee7a4fe05c12c7819bcd44a349bd0",
strip_prefix = "github.com/aws/aws-sdk-go-v2@v1.19.0",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/com_github_aws_aws_sdk_go_v2-v1.17.3.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/com_github_aws_aws_sdk_go_v2-v1.19.0.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go_v2_config",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/config",
sha256 = "9a3c9618ee70f5981194af7436a9054b35102d4f1bfa0fce78d60667d648b6f6",
strip_prefix = "github.com/aws/aws-sdk-go-v2/config@v1.15.3",
sha256 = "0c8ba4bf7ac47ac043998c3bdcd76289cf62dcae24e72cbef1bdf6cf2da7b922",
strip_prefix = "github.com/aws/aws-sdk-go-v2/config@v1.18.28",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/config/com_github_aws_aws_sdk_go_v2_config-v1.15.3.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/config/com_github_aws_aws_sdk_go_v2_config-v1.18.28.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go_v2_credentials",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/credentials",
sha256 = "88fc13b9995c5f862299280d9a0dbca54661149500c04ab8e14f543353974a25",
strip_prefix = "github.com/aws/aws-sdk-go-v2/credentials@v1.11.2",
sha256 = "cdeb51df11f42e1b979add3cf290b01682b271e720fb7e06e8f0aad42680cff7",
strip_prefix = "github.com/aws/aws-sdk-go-v2/credentials@v1.13.27",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/credentials/com_github_aws_aws_sdk_go_v2_credentials-v1.11.2.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/credentials/com_github_aws_aws_sdk_go_v2_credentials-v1.13.27.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go_v2_feature_ec2_imds",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/feature/ec2/imds",
sha256 = "5561425aaec7bca20634d8055bddd2894975d043bcc4f159f1862b14ea3dd8d5",
strip_prefix = "github.com/aws/aws-sdk-go-v2/feature/ec2/imds@v1.12.3",
sha256 = "b3b0740921b7b2832aaa7525d4c1f2580de3601c05dc0198aff4970046fd4fd0",
strip_prefix = "github.com/aws/aws-sdk-go-v2/feature/ec2/imds@v1.13.5",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/feature/ec2/imds/com_github_aws_aws_sdk_go_v2_feature_ec2_imds-v1.12.3.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/feature/ec2/imds/com_github_aws_aws_sdk_go_v2_feature_ec2_imds-v1.13.5.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go_v2_internal_configsources",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/internal/configsources",
sha256 = "e7f6ed262a89b7de55994c3534734660cae21f07e138ca1ed48796048708f896",
strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/[email protected].27",
sha256 = "5c794460a1ac545ea1283ee4bf214465d208082375c682f32d440399f182960a",
strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/[email protected].35",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/configsources/com_github_aws_aws_sdk_go_v2_internal_configsources-v1.1.27.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/configsources/com_github_aws_aws_sdk_go_v2_internal_configsources-v1.1.35.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go_v2_internal_endpoints_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/internal/endpoints/v2",
sha256 = "4c1baeacdb316aa7d37e16045c343e53157a1a8b5209d9cbf0bcb9b0b94781bc",
strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/endpoints/[email protected].21",
sha256 = "b234d84e6759f7e47c0c7ce6608f428acb69fd008468e885cc2393be0bdbf557",
strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/endpoints/[email protected].29",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/com_github_aws_aws_sdk_go_v2_internal_endpoints_v2-v2.4.21.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/com_github_aws_aws_sdk_go_v2_internal_endpoints_v2-v2.4.29.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go_v2_internal_ini",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/internal/ini",
sha256 = "353a01da6df6ec898baeed901980429745decb28aff68ef8b85eefe5ae515ec9",
strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/[email protected].10",
sha256 = "3cc3aa1dbe66b81bf189276f680ec6f6a1fc7668e8e828dae50764c58a72e4a8",
strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/[email protected].36",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/ini/com_github_aws_aws_sdk_go_v2_internal_ini-v1.3.10.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/ini/com_github_aws_aws_sdk_go_v2_internal_ini-v1.3.36.zip",
],
)
go_repository(
Expand Down Expand Up @@ -577,10 +587,10 @@ def go_deps():
name = "com_github_aws_aws_sdk_go_v2_service_internal_presigned_url",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url",
sha256 = "a4748c20c2dd79d2a7503439778d2a52e6fdd657be31324b27caf18288968d92",
strip_prefix = "github.com/aws/aws-sdk-go-v2/service/internal/[email protected].3",
sha256 = "cdd979528b2fdddbd9b0eaec969f33f6407d1ac9388631df7525cc9434d4dac3",
strip_prefix = "github.com/aws/aws-sdk-go-v2/service/internal/[email protected].29",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/com_github_aws_aws_sdk_go_v2_service_internal_presigned_url-v1.9.3.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/com_github_aws_aws_sdk_go_v2_service_internal_presigned_url-v1.9.29.zip",
],
)
go_repository(
Expand All @@ -607,20 +617,30 @@ def go_deps():
name = "com_github_aws_aws_sdk_go_v2_service_sso",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/service/sso",
sha256 = "49f04cbd2199e505307269240d3fd3e587ca3e4ba2cbd7f8d62262475a4992c1",
strip_prefix = "github.com/aws/aws-sdk-go-v2/service/[email protected]",
sha256 = "8ca0d1458087d636596bfb9182ad4a18f52aa5a960e8600e8aeeb0d52d2a33eb",
strip_prefix = "github.com/aws/aws-sdk-go-v2/service/[email protected]",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sso/com_github_aws_aws_sdk_go_v2_service_sso-v1.12.13.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go_v2_service_ssooidc",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/service/ssooidc",
sha256 = "fb6d9039c24f3448afdb516d44b4730ba1279ca91bd10c2edfea26a020a8385b",
strip_prefix = "github.com/aws/aws-sdk-go-v2/service/[email protected]",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sso/com_github_aws_aws_sdk_go_v2_service_sso-v1.11.3.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/ssooidc/com_github_aws_aws_sdk_go_v2_service_ssooidc-v1.14.13.zip",
],
)
go_repository(
name = "com_github_aws_aws_sdk_go_v2_service_sts",
build_file_proto_mode = "disable_global",
importpath = "github.com/aws/aws-sdk-go-v2/service/sts",
sha256 = "a17741c144807b854a48db54b08d5929a65ff8bc1728ac3cbbff12266d8bcb27",
strip_prefix = "github.com/aws/aws-sdk-go-v2/service/sts@v1.16.3",
sha256 = "c7904f761ff6f3d00a4e544a9427aba19032239a8053ee03309bdbed0cd4ffd5",
strip_prefix = "github.com/aws/aws-sdk-go-v2/service/sts@v1.19.3",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sts/com_github_aws_aws_sdk_go_v2_service_sts-v1.16.3.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sts/com_github_aws_aws_sdk_go_v2_service_sts-v1.19.3.zip",
],
)
go_repository(
Expand Down
22 changes: 12 additions & 10 deletions build/bazelutil/distdir_files.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,21 +241,23 @@ DISTDIR_FILES = {
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/asaskevich/govalidator/com_github_asaskevich_govalidator-v0.0.0-20230301143203-a9d515a09cc2.zip": "0f8ec67bbc585d29ec115c0885cef6f2431a422cc1cc10008e466ebe8be5dc37",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/atotto/clipboard/com_github_atotto_clipboard-v0.1.4.zip": "d67b2c36c662751309fd2ec351df3651584bea840bd27be9a90702c3a238b43f",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-lambda-go/com_github_aws_aws_lambda_go-v1.13.3.zip": "8cfc5400798abd2840f456c75265f8fba4ae488e32ca2af9a5c8073fb219ea82",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/com_github_aws_aws_sdk_go_v2-v1.17.3.zip": "db9f1deea94002f8a40a665ab86b58d9c713fda9358997c75b4859668d66802a",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/config/com_github_aws_aws_sdk_go_v2_config-v1.15.3.zip": "9a3c9618ee70f5981194af7436a9054b35102d4f1bfa0fce78d60667d648b6f6",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/credentials/com_github_aws_aws_sdk_go_v2_credentials-v1.11.2.zip": "88fc13b9995c5f862299280d9a0dbca54661149500c04ab8e14f543353974a25",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/feature/ec2/imds/com_github_aws_aws_sdk_go_v2_feature_ec2_imds-v1.12.3.zip": "5561425aaec7bca20634d8055bddd2894975d043bcc4f159f1862b14ea3dd8d5",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/configsources/com_github_aws_aws_sdk_go_v2_internal_configsources-v1.1.27.zip": "e7f6ed262a89b7de55994c3534734660cae21f07e138ca1ed48796048708f896",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/com_github_aws_aws_sdk_go_v2_internal_endpoints_v2-v2.4.21.zip": "4c1baeacdb316aa7d37e16045c343e53157a1a8b5209d9cbf0bcb9b0b94781bc",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/ini/com_github_aws_aws_sdk_go_v2_internal_ini-v1.3.10.zip": "353a01da6df6ec898baeed901980429745decb28aff68ef8b85eefe5ae515ec9",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-msk-iam-sasl-signer-go/com_github_aws_aws_msk_iam_sasl_signer_go-v1.0.0.zip": "b5f99e40aae3664b1a58b312efda28e432b4e976dd3296e24520cc79b9651a14",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/com_github_aws_aws_sdk_go_v2-v1.19.0.zip": "a000599cb80b0466affe6baf2fa98d79258ee7a4fe05c12c7819bcd44a349bd0",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/config/com_github_aws_aws_sdk_go_v2_config-v1.18.28.zip": "0c8ba4bf7ac47ac043998c3bdcd76289cf62dcae24e72cbef1bdf6cf2da7b922",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/credentials/com_github_aws_aws_sdk_go_v2_credentials-v1.13.27.zip": "cdeb51df11f42e1b979add3cf290b01682b271e720fb7e06e8f0aad42680cff7",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/feature/ec2/imds/com_github_aws_aws_sdk_go_v2_feature_ec2_imds-v1.13.5.zip": "b3b0740921b7b2832aaa7525d4c1f2580de3601c05dc0198aff4970046fd4fd0",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/configsources/com_github_aws_aws_sdk_go_v2_internal_configsources-v1.1.35.zip": "5c794460a1ac545ea1283ee4bf214465d208082375c682f32d440399f182960a",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/com_github_aws_aws_sdk_go_v2_internal_endpoints_v2-v2.4.29.zip": "b234d84e6759f7e47c0c7ce6608f428acb69fd008468e885cc2393be0bdbf557",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/ini/com_github_aws_aws_sdk_go_v2_internal_ini-v1.3.36.zip": "3cc3aa1dbe66b81bf189276f680ec6f6a1fc7668e8e828dae50764c58a72e4a8",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/databasemigrationservice/com_github_aws_aws_sdk_go_v2_service_databasemigrationservice-v1.18.3.zip": "c69b0b562c314708e5d1542bed72e0ba068f910d3cae4909a0d80219e6b497de",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/ec2/com_github_aws_aws_sdk_go_v2_service_ec2-v1.34.0.zip": "b24b82535334bd7716000ba1af24acc03fcbbcb8817b8e229e9368c1fbbe6c3e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/iam/com_github_aws_aws_sdk_go_v2_service_iam-v1.18.3.zip": "efb7b199ce0ae1dbea275fa3f8d131e874cc27d92c55ba7a007ad89762a88ed8",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/com_github_aws_aws_sdk_go_v2_service_internal_presigned_url-v1.9.3.zip": "a4748c20c2dd79d2a7503439778d2a52e6fdd657be31324b27caf18288968d92",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/com_github_aws_aws_sdk_go_v2_service_internal_presigned_url-v1.9.29.zip": "cdd979528b2fdddbd9b0eaec969f33f6407d1ac9388631df7525cc9434d4dac3",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/rds/com_github_aws_aws_sdk_go_v2_service_rds-v1.18.4.zip": "f5de5a435c3c31b14e853e9a4348ce80646db030c9a99a178e2a12fc00585f7e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/secretsmanager/com_github_aws_aws_sdk_go_v2_service_secretsmanager-v1.18.2.zip": "44dcf0add18d221042e6a709eed9beae974e5eebfe18dd37003944b7abefb271",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sso/com_github_aws_aws_sdk_go_v2_service_sso-v1.11.3.zip": "49f04cbd2199e505307269240d3fd3e587ca3e4ba2cbd7f8d62262475a4992c1",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sts/com_github_aws_aws_sdk_go_v2_service_sts-v1.16.3.zip": "a17741c144807b854a48db54b08d5929a65ff8bc1728ac3cbbff12266d8bcb27",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sso/com_github_aws_aws_sdk_go_v2_service_sso-v1.12.13.zip": "8ca0d1458087d636596bfb9182ad4a18f52aa5a960e8600e8aeeb0d52d2a33eb",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/ssooidc/com_github_aws_aws_sdk_go_v2_service_ssooidc-v1.14.13.zip": "fb6d9039c24f3448afdb516d44b4730ba1279ca91bd10c2edfea26a020a8385b",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sts/com_github_aws_aws_sdk_go_v2_service_sts-v1.19.3.zip": "c7904f761ff6f3d00a4e544a9427aba19032239a8053ee03309bdbed0cd4ffd5",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.40.37.zip": "c0c481d28af88f621fb3fdeacc1e5d32f69a1bb83d0ee959f95ce89e4e2d0494",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/smithy-go/com_github_aws_smithy_go-v1.13.5.zip": "864667edd8459bff1f750d65f40922263b995a06dcb85240536539e239e911e4",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/axiomhq/hyperloglog/com_github_axiomhq_hyperloglog-v0.0.0-20181223111420-4b99d0c2c99e.zip": "812834322ee2ca50dc36f91f9ac3f2cde4631af2f9c330b1271c78b46024a540",
Expand Down
Loading

0 comments on commit f87a559

Please sign in to comment.