Merge pull request #93 from Sage-Bionetworks/etl-580

[ETL-580] JSON to Parquet - Write record count of each export to S3
Sage-Bionetworks · Dec 18, 2023 · b19936b · b19936b
2 parents 64eff88 + 1adf6d8
commit b19936b
Show file tree

Hide file tree

Showing 9 changed files with 836 additions and 194 deletions.
diff --git a/config/develop/namespaced/glue-job-JSONToParquet.yaml b/config/develop/namespaced/glue-job-JSONToParquet.yaml
@@ -11,6 +11,7 @@ parameters:
   S3ScriptBucket: {{ stack_group_config.template_bucket_name }}
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/json_to_parquet.py'
   GlueVersion: "{{ stack_group_config.json_to_parquet_glue_version }}"
+  AdditionalPythonModules: ecs_logging~=2.0
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data:

diff --git a/config/prod/namespaced/glue-job-JSONToParquet.yaml b/config/prod/namespaced/glue-job-JSONToParquet.yaml
@@ -11,6 +11,7 @@ parameters:
   S3ScriptBucket: {{ stack_group_config.template_bucket_name }}
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/json_to_parquet.py'
   GlueVersion: "{{ stack_group_config.json_to_parquet_glue_version }}"
+  AdditionalPythonModules: ecs_logging~=2.0
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data: