From a581dc3a93b9be7d7c700f6aed2f17ddb3326d2c Mon Sep 17 00:00:00 2001 From: k8 Date: Thu, 12 Oct 2023 15:46:17 -0400 Subject: [PATCH] chore: refactor to call secrets manager JIRA:CLOUDSEC-12 --- dataeng/resources/secrets-manager.sh | 21 +++++++++++++++++++ .../resources/snowflake-collect-metrics.sh | 6 ++++++ .../snowflake-demographics-cleanup.sh | 5 +++++ .../snowflake-public-grants-cleaner.sh | 5 +++++ .../resources/snowflake-refresh-snowpipe.sh | 11 ++++++++++ ...nowflake-user-retirement-status-cleanup.sh | 5 +++++ .../resources/snowflake-validate-stitch.sh | 5 +++++ .../resources/stitch-snowflake-lag-monitor.sh | 5 +++++ 8 files changed, 63 insertions(+) create mode 100755 dataeng/resources/secrets-manager.sh mode change 100644 => 100755 dataeng/resources/snowflake-user-retirement-status-cleanup.sh diff --git a/dataeng/resources/secrets-manager.sh b/dataeng/resources/secrets-manager.sh new file mode 100755 index 000000000..880a1010f --- /dev/null +++ b/dataeng/resources/secrets-manager.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +secret_to_call="$1" +secret_name="$2" +set +x + +SECRET_JSON=$(aws secretsmanager get-secret-value --secret-id $secret_to_call --region "us-east-1" --output json) +# Check the exit status of the AWS CLI command + +echo "$SECRET_JSON" +extract_and_store_secret_value() { + + value=$(echo "$SECRET_JSON" | jq -r ".SecretString | fromjson.$secret_name" 2>/dev/null) + eval "$secret_name"='$value' +} + +if [ $? -eq 0 ]; then + # Use jq to extract the values from the JSON response + extract_and_store_secret_value $SECRET_JSON $secret_name +else + echo "AWS CLI command failed" +fi diff --git a/dataeng/resources/snowflake-collect-metrics.sh b/dataeng/resources/snowflake-collect-metrics.sh index 0f18f753a..6b3dbd677 100644 --- a/dataeng/resources/snowflake-collect-metrics.sh +++ b/dataeng/resources/snowflake-collect-metrics.sh @@ -10,6 +10,12 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS ACCOUNT +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS METRIC_NAME + python collect-metrics.py \ --metric_name $METRIC_NAME \ --key_path $WORKSPACE/analytics-secure/snowflake/rsa_key_snowflake_task_automation_user.p8 \ diff --git a/dataeng/resources/snowflake-demographics-cleanup.sh b/dataeng/resources/snowflake-demographics-cleanup.sh index 7f86c757f..aba33c64b 100644 --- a/dataeng/resources/snowflake-demographics-cleanup.sh +++ b/dataeng/resources/snowflake-demographics-cleanup.sh @@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS ACCOUNT + python demographics_cleanup.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ diff --git a/dataeng/resources/snowflake-public-grants-cleaner.sh b/dataeng/resources/snowflake-public-grants-cleaner.sh index 4fb013ff6..6e6374229 100644 --- a/dataeng/resources/snowflake-public-grants-cleaner.sh +++ b/dataeng/resources/snowflake-public-grants-cleaner.sh @@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS ACCOUNT + python snowflake_public_grants_cleaner.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ diff --git a/dataeng/resources/snowflake-refresh-snowpipe.sh b/dataeng/resources/snowflake-refresh-snowpipe.sh index d288c83e3..ff3985c7a 100644 --- a/dataeng/resources/snowflake-refresh-snowpipe.sh +++ b/dataeng/resources/snowflake-refresh-snowpipe.sh @@ -10,6 +10,17 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS ACCOUNT +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS SCHEMA +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PIPE_NAME +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS TABLE_NAME +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS DELAY +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS LIMIT + + python refresh_snowpipe.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ diff --git a/dataeng/resources/snowflake-user-retirement-status-cleanup.sh b/dataeng/resources/snowflake-user-retirement-status-cleanup.sh old mode 100644 new mode 100755 index 7f8c526df..b4507b93e --- a/dataeng/resources/snowflake-user-retirement-status-cleanup.sh +++ b/dataeng/resources/snowflake-user-retirement-status-cleanup.sh @@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS ACCOUNT + python retirement_cleanup.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ diff --git a/dataeng/resources/snowflake-validate-stitch.sh b/dataeng/resources/snowflake-validate-stitch.sh index 2017e1d94..5f75ec293 100644 --- a/dataeng/resources/snowflake-validate-stitch.sh +++ b/dataeng/resources/snowflake-validate-stitch.sh @@ -14,6 +14,11 @@ COMPARISON_START_TIME=$(date --utc --iso=minutes -d "${COMPARISON_END_TIME} - 15 cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS ACCOUNT + python stitch_vs_sqoop_validation.py \ --key_path $WORKSPACE/analytics-secure/${SNOWFLAKE_KEY_PATH} \ --passphrase_path $WORKSPACE/analytics-secure/${SNOWFLAKE_PASSPHRASE_PATH} \ diff --git a/dataeng/resources/stitch-snowflake-lag-monitor.sh b/dataeng/resources/stitch-snowflake-lag-monitor.sh index 1cdd7804e..d21581374 100644 --- a/dataeng/resources/stitch-snowflake-lag-monitor.sh +++ b/dataeng/resources/stitch-snowflake-lag-monitor.sh @@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS ACCOUNT + python stitch-snowflake-monitoring.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \