chore: fix script + test

dataeng/resources/secrets-manager.sh

@@ -1,20 +1,38 @@
-#!/usr/bin/env bash
-secret_to_call="$1"
-secret_name="$2"
-set +x
+#!/bin/bash
 
-SECRET_JSON=$(aws secretsmanager get-secret-value --secret-id $secret_to_call --region "us-east-1" --output json)
-# Check the exit status of the AWS CLI command
+extract_value_from_json() {
+    local json="$1"
+    local key="$2"
+    local value=$(echo "$json" | jq -r ".$key")
+}
 
-extract_and_store_secret_value() {
+fetch_whole_secret() {
+    local secret_name="$1"
+    local variable_name="$2"
+    local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
+    #set whole file as env var
+    declare "${secret_name%=*}=${secret_value}"
+}
 
-    value=$(echo "$SECRET_JSON" | jq -r ".SecretString | fromjson.$secret_name" 2>/dev/null)
-    eval "$secret_name"='$value'
+fetch_specific_key() {
+    local secret_name="$1"
+    local key="$2"
+    local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
+    local extracted_value=$(extract_value_from_json "$secret_value" "$key")
+    declare "${key%=*}=${extracted_value}"
 }
 
-if [ $? -eq 0 ]; then
-    # Use jq to extract the values from the JSON response
-    extract_and_store_secret_value $SECRET_JSON $secret_name
+# Main script
+if [[ "$1" == "-w" ]]; then
+    if [ $# -ne 3 ]; then
+        echo "Usage: $0 -w <name_of_file> <name_of_variable>"
+        exit 1
+    fi
+    fetch_whole_secret "$2" "$3"
 else
-    echo "AWS CLI command failed"
+    if [ $# -ne 2 ]; then
+        echo "Usage: $0 <name_of_file> <name_of_key>"
+        exit 1
+    fi
+    fetch_specific_key "$1" "$2"
 fi

dataeng/resources/snowflake-refresh-snowpipe.sh

@@ -14,19 +14,20 @@ make requirements
 source $WORKSPACE/secrets-manager.sh
 # Fetch the secrets from AWS
 set +x
-get_secret_value analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS KEY_PATH
-get_secret_value analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PASSPHRASE_PATH
-get_secret_value analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS USER
-get_secret_value analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS ACCOUNT
+
+
+secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_snowpipe_user.p8
+secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_passphrase_snowpipe_user
+
 set -x
 
 python refresh_snowpipe.py \
-    --key_path $WORKSPACE/analytics-secure/$KEY_PATH \
-    --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \
-    --user $USER \
+    --user 'SNOWPIPE' \
     --schema $SCHEMA \
-    --account $ACCOUNT \
+    --account 'edx.us-east-1' \
     --pipe_name $PIPE_NAME \
     --table_name $TABLE_NAME \
     --delay $DELAY \
     --limit $LIMIT
+    --key_file $KEY_PATH \
+    --passphrase_file $PASSPHRASE_PATH