Improve deploy process using custom github action (#463)

* update dev variabe to test * Add if else condition * use different branch instead of master to test * use input name in tests typo * Update actual deploy code * revert the image * remove prod workflow * Update .github/workflows/deploy.yml Co-authored-by: chowbao <[email protected]> --------- Co-authored-by: chowbao <[email protected]>
stellar · Jul 26, 2024 · c3170a3 · c3170a3
1 parent 4e86886
commit c3170a3
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 119 deletions.
diff --git a/.github/workflows/ci-cd-prod.yml b/.github/workflows/ci-cd-prod.yml
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -13,33 +13,6 @@ on:
           - prod
 
 jobs:
-  log:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Display Environment Name
-        run: echo "Hello ${{ inputs.envName }}"
-
-  pre-commit:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-
-      - id: file_changes
-        uses: trilom/[email protected]
-        with:
-          output: " "
-
-      - uses: pre-commit/[email protected]
-        env:
-          extra_args: --color=always --files ${{ steps.file_changes.outputs.files}}
-
   tests:
     runs-on: ubuntu-latest
 
@@ -53,7 +26,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json
+          cat airflow_variables_${{ inputs.envName }}.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json
           python -m pip install --upgrade pip
           pip install -r requirements-ci.txt
 
@@ -78,6 +51,7 @@ jobs:
         run: pytest dags/
 
   deploy-to-dev:
+    if: ${{ github.event.inputs.envName == 'dev' }}
     runs-on: ubuntu-latest
     needs: [tests]
     steps:
@@ -116,3 +90,45 @@ jobs:
             components install kubectl && gcloud composer environments run
             $COMPOSER_ENVIRONMENT --location $LOCATION variables import
             -- gcsfuse/actual_mount_path/variables.json
+
+  deploy-to-prod:
+    if: ${{ github.event.inputs.envName == 'prod' && github.ref == 'refs/heads/master' }}
+    runs-on: ubuntu-latest
+    needs: [tests]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install google-cloud-storage==2.1.0
+
+      - name: Authenticate to hubble GCP
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}"
+
+      - name: Upload files to prod GCS bucket
+        run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod
+        env:
+          GOOGLE_CLOUD_PROJECT: hubble-261722
+          BUCKET: us-central1-hubble-14c4ca64-bucket
+
+      - name: Update Airflow variables
+        uses: actions-hub/gcloud@master
+        env:
+          PROJECT_ID: hubble-261722
+          APPLICATION_CREDENTIALS: "${{ secrets.CREDS_PROD_HUBBLE }}"
+          COMPOSER_ENVIRONMENT: hubble
+          LOCATION: us-central1
+        with:
+          args: >
+            components install kubectl && gcloud composer environments run
+            $COMPOSER_ENVIRONMENT --location $LOCATION variables import
+            -- gcsfuse/actual_mount_path/variables.json