Update for mono repo

.env.example

@@ -9,5 +9,9 @@ STATE_BUCKET_NAME=<Fill Me>
 STATE_BUCKET_KEY=<Fill Me>
 STATE_DYNAMO_TABLE=<Fill Me>
 ASSUME_ROLE_ARNS='["<Read role>", "<Write role>"]'
-COGNITO_APP_SECRET=<Fill Me>
-STAC_INGESTOR_API_URL=<Fill Me>
+VEDA_WORKFLOWS_CLIENT_SECRET_ID=<Fill Me>
+VEDA_STAC_INGESTOR_API_URL=<Fill Me>
+VEDA_RASTER_URL=<Fill Me>
+VEDA_DATA_ACCESS_ROLE_ARN=<Fill Me>
+VEDA_STAC_URL=<Fill Me>
+WORKFLOW_ROOT_PATH=<Fill Me>

.flake8

@@ -1,3 +1,3 @@
 [flake8]
 # taken from github actions ignore
-ignore = E1, E2, E3, E5, W1, W2, W3, W5
+ignore = E1, E2, E3, E5, W1, W2, W3, W5

.github/actions/terraform-deploy/action.yml

@@ -0,0 +1,64 @@
+name: Deploy
+
+inputs:
+  env_aws_secret_name:
+    required: true
+    type: string
+  env-file:
+    type: string
+    default: ".env"
+  dir:
+    required: false
+    type: string
+    default: "."
+  script_path:
+    type: string
+  backend_stack_name:
+    type: string
+  auth_stack_name:
+    type: string
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+        cache: "pip"
+
+    - name: Install python dependencies
+      shell: bash
+      working-directory: ${{ inputs.dir }}
+      run: pip install -r deploy_requirements.txt
+
+    - name: Get relevant environment configuration from aws secrets
+      shell: bash
+      working-directory: ${{ inputs.dir }}
+      env:
+        SECRET_SSM_NAME: ${{ inputs.env_aws_secret_name }}
+        AWS_DEFAULT_REGION: us-west-2
+      run: |
+        if [[ -z "${{ inputs.script_path }}" ]]; then
+        ./scripts/sync-env.sh ${{ inputs.env_aws_secret_name }}
+        else
+        echo ${{ inputs.auth_stack_name}}
+        echo ${{ inputs.backend_stack_name}}
+        python ${{ inputs.script_path }} --secret-id ${{ inputs.env_aws_secret_name }} --stack-names ${{ inputs.auth_stack_name}},${{ inputs.backend_stack_name}}
+        source .env
+        echo "PREFIX=data-pipeline-$STAGE" >> ${{ inputs.env-file }}
+        cat .env
+        fi
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v3
+      with:
+        terraform_version: 1.3.3
+
+    - name: Deploy
+      shell: bash
+      working-directory: ${{ inputs.dir }}
+      run: |
+        ./scripts/deploy.sh ${{ inputs.env-file }} <<< init
+        ./scripts/deploy.sh ${{ inputs.env-file }} <<< deploy

.github/workflows/cicd.yml

@@ -1,91 +1,77 @@
-name: CI/CD
+name: CICD 🚀
+
+permissions:
+  id-token: write
+  contents: read
 
 on:
   push:
+    branches:
+      - main
+      - dev
+      - production
   pull_request:
+    branches:
+      - main
+      - dev
+      - production
     types: [ opened, reopened, edited, synchronize ]
 
 jobs:
   gitflow-enforcer:
-      runs-on: ubuntu-latest
-      steps:
-        - name: Check branch
-          run: |
-            if [[ $GITHUB_BASE_REF == "main" ]]; then
-               if [[ $GITHUB_HEAD_REF != "dev" && $GITHUB_HEAD_REF != "revert-"* ]]; then
-                  echo "ERROR: You can only merge to 'main' from 'dev' or a 'revert-*' branch"
-                  exit 1
-               fi
-            elif [[ $GITHUB_BASE_REF == "production" ]]; then
-               if [[ $GITHUB_HEAD_REF != "main" && $GITHUB_HEAD_REF != "revert-"* ]]; then
-                  echo "ERROR: You can only merge to 'production' from 'main' or a 'revert-*' branch"
-                  exit 1
-               fi
-            fi
-
-
-  run-linters:
-    name: Run linters
+    name: GitFlow Enforcer 👮‍
     runs-on: ubuntu-latest
-    needs: gitflow-enforcer
+    steps:
+      - name: Check branch
+        if: github.base_ref == 'main' && github.head_ref != 'dev' || github.base_ref == 'production' && github.head_ref != 'main'
+        run: |
+          echo "ERROR: You can only merge to main from dev and to production from main"
+          exit 1
 
+  define-environment:
+    name: Set ✨ environment ✨
+    needs: gitflow-enforcer
+    runs-on: ubuntu-latest
     steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
+      - name: Set the environment based on the branch
+        id: define_environment
+        run: |
+          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+            echo "env_name=staging" >> $GITHUB_OUTPUT
+          elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then
+            echo "env_name=development" >> $GITHUB_OUTPUT
+          elif [ "${{ github.ref }}" = "refs/heads/production" ]; then
+            echo "env_name=production" >> $GITHUB_OUTPUT
+          fi
+      - name: Print the environment
+        run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}"
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
+    outputs:
+      env_name: ${{ steps.define_environment.outputs.env_name }}
 
-      - name: Install Python dependencies
-        run: pip install black flake8
+  deploy:
+    name: Deploy to ${{ needs.define-environment.outputs.env_name }} 🚀
+    runs-on: ubuntu-latest
+    if: ${{ needs.define-environment.outputs.env_name }}
+    needs: [gitflow-enforcer, define-environment]
+    environment: ${{ needs.define-environment.outputs.env_name }}
+    concurrency: ${{ needs.define-environment.outputs.env_name }}
 
-      - name: Run linters
-        uses: wearerequired/lint-action@v2
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
         with:
-          continue_on_error: false
-          black: true
-          flake8: true
-          flake8_args: "--ignore E1,E2,E3,E5,W1,W2,W3,W5" # black already handles formatting, this prevents conflicts
-
-  deploy-to-dev:
-    needs: run-linters
-    if: github.ref_name == 'dev'
-    concurrency: development
-    uses: "./.github/workflows/deploy.yml"
-    with:
-      environment: development
-      env-file: ".env_dev"
-      stage: "dev"
-      role-session-name: "veda-data-airflow-github-development-deployment"
-      aws-region: "us-west-2"
-
-    secrets: inherit
-
-  deploy-to-staging:
-    needs: run-linters
-    if: github.ref_name == 'main'
-    concurrency: staging
-    uses: "./.github/workflows/deploy.yml"
-    with:
-      environment: staging
-      env-file: ".env_staging"
-      stage: "staging"
-      role-session-name: "veda-data-airflow-github-staging-deployment"
-      aws-region: "us-west-2"
-    secrets: inherit
-
-  deploy-to-production:
-    needs: run-linters
-    if: github.ref_name == 'production'
-    concurrency: production
-    uses: "./.github/workflows/deploy.yml"
-    with:
-      environment: production
-      env-file: ".env_prod"
-      stage: "production"
-      role-session-name: "veda-data-airflow-github-production-deployment"
-      aws-region: "us-west-2"
+          lfs: "true"
+          submodules: "recursive"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
+          role-session-name: "veda-airflow-github-${{ needs.define-environment.outputs.env_name }}-deployment"
+          aws-region: "us-west-2"
 
-    secrets: inherit
+      - name: Run deployment
+        uses: "./.github/actions/terraform-deploy"
+        with:
+          env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }}

dags/requirements-constraints.txt

@@ -631,4 +631,3 @@ zipp==3.10.0
 zope.event==4.5.0
 zope.interface==5.5.1
 zstandard==0.19.0
-

dags/veda_data_pipeline/groups/discover_group.py

@@ -20,7 +20,7 @@ def discover_from_s3_task(ti):
     config = ti.dag_run.conf
     # (event, chunk_size=2800, role_arn=None, bucket_output=None):
     MWAA_STAC_CONF = Variable.get("MWAA_STACK_CONF", deserialize_json=True)
-    read_assume_arn = Variable.get("ASSUME_ROLE_READ_ARN")
+    read_assume_arn = Variable.get("ASSUME_ROLE_READ_ARN", default_var=None)
     return s3_discovery_handler(
         event=config,
         role_arn=read_assume_arn,

dags/veda_data_pipeline/groups/processing_group.py

@@ -59,7 +59,7 @@ def subdag_process():
                         "environment": [
                             {
                                 "name": "EXTERNAL_ROLE_ARN",
-                                "value": Variable.get("ASSUME_ROLE_READ_ARN"),
+                                "value": Variable.get("ASSUME_ROLE_READ_ARN", default_var=''),
                             },
                             {
                                 "name": "BUCKET",

dags/veda_data_pipeline/groups/transfer_group.py

@@ -25,7 +25,7 @@ def cogify_choice(ti):
 def transfer_data(ti):
     """Transfer data from one S3 bucket to another; s3 copy, no need for docker"""
     config = ti.dag_run.conf
-    role_arn = Variable.get("ASSUME_ROLE_READ_ARN")
+    role_arn = Variable.get("ASSUME_ROLE_READ_ARN", default_var="")
     # (event, chunk_size=2800, role_arn=None, bucket_output=None):
     return data_transfer_handler(event=config, role_arn=role_arn)
 
@@ -66,7 +66,9 @@ def subdag_transfer():
                         "environment": [
                             {
                                 "name": "EXTERNAL_ROLE_ARN",
-                                "value": Variable.get("ASSUME_ROLE_READ_ARN"),
+                                "value": Variable.get(
+                                    "ASSUME_ROLE_READ_ARN", default_var=""
+                                ),
                             },
                         ],
                         "memory": 2048,

dags/veda_data_pipeline/veda_process_raster_pipeline.py

@@ -20,7 +20,7 @@
     "discovery": "s3",
     "datetime_range": "month",
     "discovered": 33,
-    "payload": "s3://veda-uah-sit-mwaa-853558080719/events/geoglam/s3_discover_output_6c46b57a-7474-41fe-977a-19d164531cdc.json"
+    "payload": "s3://veda-uah-sit-mwaa-853558080719/events/geoglam/s3_discover_output_6c46b57a-7474-41fe-977a-.json"
 }	
 ```
 - [Supports linking to external content](https://github.com/NASA-IMPACT/veda-data-pipelines)
@@ -36,6 +36,7 @@
     "payload": "<s3_uri_event_payload",
 }
 dag_args = {
+    "max_active_runs": 20,
     "start_date": pendulum.today("UTC").add(days=-1),
     "schedule_interval": None,
     "catchup": False,