diff --git a/.github/workflows/ci-cd-prod.yml b/.github/workflows/ci-cd-prod.yml deleted file mode 100644 index 7cb4a97a..00000000 --- a/.github/workflows/ci-cd-prod.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: CI-CD-PROD - -on: - pull_request: - types: - - closed - branches: - - release - -jobs: - tests: - runs-on: ubuntu-latest - if: github.event.pull_request.merged == true - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - cat airflow_variables_prod.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json - python -m pip install --upgrade pip - pip install -r requirements-ci.txt - - - name: Init Airflow SQLite database - run: airflow db init - - - name: Import Airflow variables - run: airflow variables import airflow_variables_ci.json - - - name: Authenticate to test-hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - - id: "get-credentials" - uses: "google-github-actions/get-gke-credentials@v2" - with: - cluster_name: "us-central1-test-hubble-43c3e190-gke" - location: "us-central1" - - - name: Pytest - run: pytest dags/ - - release: - runs-on: ubuntu-latest - needs: [tests] - # deploy only occurs when pr is merged - if: >- - github.event.pull_request.merged == true && - github.repository == 'stellar/stellar-etl-airflow' - - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install google-cloud-storage==2.1.0 - - - name: Authenticate to hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}" - - - name: Upload files to prod GCS bucket - run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod - env: - GOOGLE_CLOUD_PROJECT: hubble-261722 - BUCKET: us-central1-hubble-14c4ca64-bucket - - - name: Update Airflow variables - uses: actions-hub/gcloud@master - env: - PROJECT_ID: hubble-261722 - APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: hubble - LOCATION: us-central1 - with: - args: > - components install kubectl && gcloud composer environments run - $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/actual_mount_path/variables.json diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7df35645..aaf75150 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,33 +13,6 @@ on: - prod jobs: - log: - runs-on: ubuntu-latest - - steps: - - name: Display Environment Name - run: echo "Hello ${{ inputs.envName }}" - - pre-commit: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - id: file_changes - uses: trilom/file-changes-action@v1.2.4 - with: - output: " " - - - uses: pre-commit/action@v3.0.0 - env: - extra_args: --color=always --files ${{ steps.file_changes.outputs.files}} - tests: runs-on: ubuntu-latest @@ -53,7 +26,7 @@ jobs: - name: Install dependencies run: | - cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json + cat airflow_variables_${{ inputs.envName }}.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json python -m pip install --upgrade pip pip install -r requirements-ci.txt @@ -78,6 +51,7 @@ jobs: run: pytest dags/ deploy-to-dev: + if: ${{ github.event.inputs.envName == 'dev' }} runs-on: ubuntu-latest needs: [tests] steps: @@ -116,3 +90,45 @@ jobs: components install kubectl && gcloud composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import -- gcsfuse/actual_mount_path/variables.json + + deploy-to-prod: + if: ${{ github.event.inputs.envName == 'prod' && github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + needs: [tests] + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install google-cloud-storage==2.1.0 + + - name: Authenticate to hubble GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}" + + - name: Upload files to prod GCS bucket + run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod + env: + GOOGLE_CLOUD_PROJECT: hubble-261722 + BUCKET: us-central1-hubble-14c4ca64-bucket + + - name: Update Airflow variables + uses: actions-hub/gcloud@master + env: + PROJECT_ID: hubble-261722 + APPLICATION_CREDENTIALS: "${{ secrets.CREDS_PROD_HUBBLE }}" + COMPOSER_ENVIRONMENT: hubble + LOCATION: us-central1 + with: + args: > + components install kubectl && gcloud composer environments run + $COMPOSER_ENVIRONMENT --location $LOCATION variables import + -- gcsfuse/actual_mount_path/variables.json