From b1d0d18b4f97a77adc7d015cdfe183fbc9f72d70 Mon Sep 17 00:00:00 2001 From: maz Date: Wed, 11 Dec 2024 09:40:33 +0900 Subject: [PATCH] feat(glue): support AWS Glue 5.0 --- packages/@aws-cdk/aws-glue-alpha/README.md | 12 +- .../aws-glue-alpha/lib/job-executable.ts | 8 + .../aws-glue-job.assets.json | 4 +- .../aws-glue-job.template.json | 414 ++++++++++++ .../test/integ.job.js.snapshot/manifest.json | 57 +- .../test/integ.job.js.snapshot/tree.json | 622 +++++++++++++++++- .../@aws-cdk/aws-glue-alpha/test/integ.job.ts | 2 +- .../test/job-executable.test.ts | 2 + 8 files changed, 1094 insertions(+), 27 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index ecb0401122955..8b5b2362eb594 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -41,7 +41,7 @@ An ETL job processes data in batches using Apache Spark. declare const bucket: s3.Bucket; new glue.Job(this, 'ScalaSparkEtlJob', { executable: glue.JobExecutable.scalaEtl({ - glueVersion: glue.GlueVersion.V4_0, + glueVersion: glue.GlueVersion.V5_0, script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'), className: 'com.example.HelloWorld', extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')], @@ -58,7 +58,7 @@ A Streaming job is similar to an ETL job, except that it performs ETL on data st ```ts new glue.Job(this, 'PythonSparkStreamingJob', { executable: glue.JobExecutable.pythonStreaming({ - glueVersion: glue.GlueVersion.V4_0, + glueVersion: glue.GlueVersion.V5_0, pythonVersion: glue.PythonVersion.THREE, script: glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')), }), @@ -94,7 +94,7 @@ These jobs run in a Ray environment managed by AWS Glue. ```ts new glue.Job(this, 'RayJob', { executable: glue.JobExecutable.pythonRay({ - glueVersion: glue.GlueVersion.V4_0, + glueVersion: glue.GlueVersion.V5_0, pythonVersion: glue.PythonVersion.THREE_NINE, runtime: glue.Runtime.RAY_TWO_FOUR, script: glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')), @@ -137,7 +137,7 @@ Enable job run queuing by setting the `jobRunQueuingEnabled` property to `true`. new glue.Job(this, 'EnableRunQueuing', { jobName: 'EtlJobWithRunQueuing', executable: glue.JobExecutable.pythonEtl({ - glueVersion: glue.GlueVersion.V4_0, + glueVersion: glue.GlueVersion.V5_0, pythonVersion: glue.PythonVersion.THREE, script: glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')), }), @@ -488,7 +488,7 @@ new glue.S3Table(this, 'MyTable', { declare const myDatabase: glue.Database; // KMS key is created automatically new glue.S3Table(this, 'MyTable', { - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, // ... database: myDatabase, columns: [{ @@ -546,7 +546,7 @@ new glue.S3Table(this, 'MyTable', { // ... database: myDatabase, dataFormat: glue.DataFormat.JSON, -}); +}); ``` ### Primitives diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts index 4bee0a054bcd8..875190742a66a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts @@ -11,16 +11,19 @@ import { Code } from './code'; export class GlueVersion { /** * Glue version using Spark 2.2.1 and Python 2.7 + * @deprecated Reached end of support */ public static readonly V0_9 = new GlueVersion('0.9'); /** * Glue version using Spark 2.4.3, Python 2.7 and Python 3.6 + * @deprecated Reached end of support */ public static readonly V1_0 = new GlueVersion('1.0'); /** * Glue version using Spark 2.4.3 and Python 3.7 + * @deprecated Reached end of support */ public static readonly V2_0 = new GlueVersion('2.0'); @@ -34,6 +37,11 @@ export class GlueVersion { */ public static readonly V4_0 = new GlueVersion('4.0'); + /** + * Glue version using Spark 3.5.2 and Python 3.11 + */ + public static readonly V5_0 = new GlueVersion('5.0'); + /** * Custom Glue version * @param version custom version diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index 63338c564caae..8cbc419433c87 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -40,7 +40,7 @@ } } }, - "bbfa96ff49629391347bf6f622ed288ee10ced63216750a5a1805dc22b3bcb5a": { + "472ef683512322adcc5fad4705de91c6ee394b671ca18bc5fd2a09b663d879ea": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -48,7 +48,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "bbfa96ff49629391347bf6f622ed288ee10ced63216750a5a1805dc22b3bcb5a.json", + "objectKey": "472ef683512322adcc5fad4705de91c6ee394b671ca18bc5fd2a09b663d879ea.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index c8321e59f15ca..c838912311f18 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -1242,6 +1242,420 @@ "WorkerType": "G.025X" } }, + "EtlJob50ServiceRole3F950524": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "EtlJob50ServiceRoleDefaultPolicyD67CA798": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "EtlJob50SparkUIBucket28E6EA39", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "EtlJob50SparkUIBucket28E6EA39", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "EtlJob50ServiceRoleDefaultPolicyD67CA798", + "Roles": [ + { + "Ref": "EtlJob50ServiceRole3F950524" + } + ] + } + }, + "EtlJob50SparkUIBucket28E6EA39": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "EtlJob502E3517AF": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "glueetl", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "DefaultArguments": { + "--job-language": "python", + "--enable-continuous-cloudwatch-log": "true", + "--enable-continuous-log-filter": "true", + "--continuous-log-logStreamPrefix": "EtlJob", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "EtlJob50SparkUIBucket28E6EA39" + }, + "/" + ] + ] + }, + "arg1": "value1", + "arg2": "value2", + "--conf": "valueConf" + }, + "ExecutionClass": "STANDARD", + "ExecutionProperty": { + "MaxConcurrentRuns": 2 + }, + "GlueVersion": "5.0", + "MaxRetries": 2, + "Name": "EtlJob5.0", + "NotificationProperty": { + "NotifyDelayAfter": 1 + }, + "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob50ServiceRole3F950524", + "Arn" + ] + }, + "Tags": { + "key": "value" + }, + "Timeout": 5, + "WorkerType": "G.1X" + } + }, + "EtlJob50SuccessMetricRule2BFA7538": { + "Type": "AWS::Events::Rule", + "Properties": { + "Description": { + "Fn::Join": [ + "", + [ + "Rule triggered when Glue job ", + { + "Ref": "EtlJob502E3517AF" + }, + " is in SUCCEEDED state" + ] + ] + }, + "EventPattern": { + "source": [ + "aws.glue" + ], + "detail-type": [ + "Glue Job State Change", + "Glue Job Run Status" + ], + "detail": { + "jobName": [ + { + "Ref": "EtlJob502E3517AF" + } + ], + "state": [ + "SUCCEEDED" + ] + } + }, + "State": "ENABLED" + } + }, + "StreamingJob50ServiceRoleE8945CAE": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "StreamingJob50ServiceRoleDefaultPolicy815A506E": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob50SparkUIBucket31964232", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob50SparkUIBucket31964232", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StreamingJob50ServiceRoleDefaultPolicy815A506E", + "Roles": [ + { + "Ref": "StreamingJob50ServiceRoleE8945CAE" + } + ] + } + }, + "StreamingJob50SparkUIBucket31964232": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "StreamingJob506D8CB337": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "gluestreaming", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "DefaultArguments": { + "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob50SparkUIBucket31964232" + }, + "/" + ] + ] + }, + "arg1": "value1", + "arg2": "value2" + }, + "GlueVersion": "5.0", + "Name": "StreamingJob5.0", + "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob50ServiceRoleE8945CAE", + "Arn" + ] + }, + "Tags": { + "key": "value" + }, + "WorkerType": "G.025X" + } + }, "ShellJobServiceRoleCF97BC4B": { "Type": "AWS::IAM::Role", "Properties": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index 30381c71e9dd5..b3b01901d9c1a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -16,10 +16,9 @@ "templateFile": "aws-glue-job.template.json", "terminationProtection": false, "validateOnSynth": false, - "notificationArns": [], "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/bbfa96ff49629391347bf6f622ed288ee10ced63216750a5a1805dc22b3bcb5a.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/472ef683512322adcc5fad4705de91c6ee394b671ca18bc5fd2a09b663d879ea.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -197,6 +196,60 @@ "data": "StreamingJob40E284A782" } ], + "/aws-glue-job/EtlJob5.0/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob50ServiceRole3F950524" + } + ], + "/aws-glue-job/EtlJob5.0/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob50ServiceRoleDefaultPolicyD67CA798" + } + ], + "/aws-glue-job/EtlJob5.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob50SparkUIBucket28E6EA39" + } + ], + "/aws-glue-job/EtlJob5.0/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob502E3517AF" + } + ], + "/aws-glue-job/EtlJob5.0/SuccessMetricRule/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob50SuccessMetricRule2BFA7538" + } + ], + "/aws-glue-job/StreamingJob5.0/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob50ServiceRoleE8945CAE" + } + ], + "/aws-glue-job/StreamingJob5.0/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob50ServiceRoleDefaultPolicy815A506E" + } + ], + "/aws-glue-job/StreamingJob5.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob50SparkUIBucket31964232" + } + ], + "/aws-glue-job/StreamingJob5.0/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob506D8CB337" + } + ], "/aws-glue-job/ShellJob/ServiceRole/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 4f2101649acb2..4e48a0c267bca 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -203,13 +203,13 @@ "version": "0.0.0" } }, - "Code0afe53ab7e601c42a4b87c9022cbe434": { - "id": "Code0afe53ab7e601c42a4b87c9022cbe434", - "path": "aws-glue-job/EtlJob2.0/Code0afe53ab7e601c42a4b87c9022cbe434", + "Codecc31fe5213ff3688a52934fe07b8224f": { + "id": "Codecc31fe5213ff3688a52934fe07b8224f", + "path": "aws-glue-job/EtlJob2.0/Codecc31fe5213ff3688a52934fe07b8224f", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/EtlJob2.0/Code0afe53ab7e601c42a4b87c9022cbe434/Stage", + "path": "aws-glue-job/EtlJob2.0/Codecc31fe5213ff3688a52934fe07b8224f/Stage", "constructInfo": { "fqn": "aws-cdk-lib.AssetStaging", "version": "0.0.0" @@ -217,7 +217,7 @@ }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/EtlJob2.0/Code0afe53ab7e601c42a4b87c9022cbe434/AssetBucket", + "path": "aws-glue-job/EtlJob2.0/Codecc31fe5213ff3688a52934fe07b8224f/AssetBucket", "constructInfo": { "fqn": "aws-cdk-lib.aws_s3.BucketBase", "version": "0.0.0" @@ -1804,6 +1804,596 @@ "version": "0.0.0" } }, + "EtlJob5.0": { + "id": "EtlJob5.0", + "path": "aws-glue-job/EtlJob5.0", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job/EtlJob5.0/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/EtlJob5.0/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob5.0/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job/EtlJob5.0/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob5.0/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "EtlJob50SparkUIBucket28E6EA39", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "EtlJob50SparkUIBucket28E6EA39", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "EtlJob50ServiceRoleDefaultPolicyD67CA798", + "roles": [ + { + "Ref": "EtlJob50ServiceRole3F950524" + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/EtlJob5.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob5.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob5.0/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "glueetl", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "defaultArguments": { + "--job-language": "python", + "--enable-continuous-cloudwatch-log": "true", + "--enable-continuous-log-filter": "true", + "--continuous-log-logStreamPrefix": "EtlJob", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "EtlJob50SparkUIBucket28E6EA39" + }, + "/" + ] + ] + }, + "arg1": "value1", + "arg2": "value2", + "--conf": "valueConf" + }, + "executionClass": "STANDARD", + "executionProperty": { + "maxConcurrentRuns": 2 + }, + "glueVersion": "5.0", + "maxRetries": 2, + "name": "EtlJob5.0", + "notificationProperty": { + "notifyDelayAfter": 1 + }, + "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob50ServiceRole3F950524", + "Arn" + ] + }, + "tags": { + "key": "value" + }, + "timeout": 5, + "workerType": "G.1X" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" + } + }, + "SuccessMetricRule": { + "id": "SuccessMetricRule", + "path": "aws-glue-job/EtlJob5.0/SuccessMetricRule", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob5.0/SuccessMetricRule/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Events::Rule", + "aws:cdk:cloudformation:props": { + "description": { + "Fn::Join": [ + "", + [ + "Rule triggered when Glue job ", + { + "Ref": "EtlJob502E3517AF" + }, + " is in SUCCEEDED state" + ] + ] + }, + "eventPattern": { + "source": [ + "aws.glue" + ], + "detail-type": [ + "Glue Job State Change", + "Glue Job Run Status" + ], + "detail": { + "jobName": [ + { + "Ref": "EtlJob502E3517AF" + } + ], + "state": [ + "SUCCEEDED" + ] + } + }, + "state": "ENABLED" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" + } + }, + "StreamingJob5.0": { + "id": "StreamingJob5.0", + "path": "aws-glue-job/StreamingJob5.0", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job/StreamingJob5.0/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/StreamingJob5.0/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob5.0/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job/StreamingJob5.0/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob5.0/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob50SparkUIBucket31964232", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob50SparkUIBucket31964232", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "StreamingJob50ServiceRoleDefaultPolicy815A506E", + "roles": [ + { + "Ref": "StreamingJob50ServiceRoleE8945CAE" + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/StreamingJob5.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob5.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob5.0/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "gluestreaming", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "defaultArguments": { + "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob50SparkUIBucket31964232" + }, + "/" + ] + ] + }, + "arg1": "value1", + "arg2": "value2" + }, + "glueVersion": "5.0", + "name": "StreamingJob5.0", + "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob50ServiceRoleE8945CAE", + "Arn" + ] + }, + "tags": { + "key": "value" + }, + "workerType": "G.025X" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" + } + }, "ShellJob": { "id": "ShellJob", "path": "aws-glue-job/ShellJob", @@ -2314,13 +2904,13 @@ "version": "0.0.0" } }, - "Code7cb6f530344016d269e3d4643b46e988": { - "id": "Code7cb6f530344016d269e3d4643b46e988", - "path": "aws-glue-job/RayJob/Code7cb6f530344016d269e3d4643b46e988", + "Code50e1efc20448ce2ab46601fe58b428a6": { + "id": "Code50e1efc20448ce2ab46601fe58b428a6", + "path": "aws-glue-job/RayJob/Code50e1efc20448ce2ab46601fe58b428a6", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/RayJob/Code7cb6f530344016d269e3d4643b46e988/Stage", + "path": "aws-glue-job/RayJob/Code50e1efc20448ce2ab46601fe58b428a6/Stage", "constructInfo": { "fqn": "aws-cdk-lib.AssetStaging", "version": "0.0.0" @@ -2328,7 +2918,7 @@ }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/RayJob/Code7cb6f530344016d269e3d4643b46e988/AssetBucket", + "path": "aws-glue-job/RayJob/Code50e1efc20448ce2ab46601fe58b428a6/AssetBucket", "constructInfo": { "fqn": "aws-cdk-lib.aws_s3.BucketBase", "version": "0.0.0" @@ -2340,13 +2930,13 @@ "version": "0.0.0" } }, - "Code7fed4fdaae1c2239a27034492f66d212": { - "id": "Code7fed4fdaae1c2239a27034492f66d212", - "path": "aws-glue-job/RayJob/Code7fed4fdaae1c2239a27034492f66d212", + "Code56ba4602155dc0a716e99f3ef481df41": { + "id": "Code56ba4602155dc0a716e99f3ef481df41", + "path": "aws-glue-job/RayJob/Code56ba4602155dc0a716e99f3ef481df41", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/RayJob/Code7fed4fdaae1c2239a27034492f66d212/Stage", + "path": "aws-glue-job/RayJob/Code56ba4602155dc0a716e99f3ef481df41/Stage", "constructInfo": { "fqn": "aws-cdk-lib.AssetStaging", "version": "0.0.0" @@ -2354,7 +2944,7 @@ }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/RayJob/Code7fed4fdaae1c2239a27034492f66d212/AssetBucket", + "path": "aws-glue-job/RayJob/Code56ba4602155dc0a716e99f3ef481df41/AssetBucket", "constructInfo": { "fqn": "aws-cdk-lib.aws_s3.BucketBase", "version": "0.0.0" @@ -2827,7 +3417,7 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.3.0" + "version": "10.4.2" } } }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index d2195eb4fc7eb..4736af72483a7 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -25,7 +25,7 @@ const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_wor const scriptResolveOptions = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'resolve_options.py')); const moduleUtils = glue.Code.fromAsset(path.join(__dirname, 'module', 'utils.zip')); -[glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { +[glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0, glue.GlueVersion.V5_0].forEach((glueVersion) => { const etlJob = new glue.Job(stack, 'EtlJob' + glueVersion.name, { jobName: 'EtlJob' + glueVersion.name, executable: glue.JobExecutable.pythonEtl({ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts index f749cfaf90f5d..8d5e57ddd6211 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts @@ -13,6 +13,8 @@ describe('GlueVersion', () => { test('.V4_0 should set the name correctly', () => expect(glue.GlueVersion.V4_0.name).toEqual('4.0')); + test('.V5_0 should set the name correctly', () => expect(glue.GlueVersion.V5_0.name).toEqual('5.0')); + test('of(customVersion) should set the name correctly', () => expect(glue.GlueVersion.of('CustomVersion').name).toEqual('CustomVersion')); });