test_illumina_genotyping_array.yml

name: Test Illumina Genotyping Array

# Controls when the workflow will run
on:
  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
  # push:
  #   branches:
  #     - kp_GHA_Terra_auth_PD-2682
  pull_request:
    branches: [ "develop", "staging", "master" ]
    # Only run if files in these paths changed:
    ####################################
    # SET PIPELINE SPECIFIC PATHS HERE #
    ####################################
    paths:
      - 'pipelines/broad/genotyping/illumina/**'
      - 'tasks/broad/IlluminaGenotypingArrayTasks.wdl'
      - 'tasks/broad/Qc.wdl'
      - 'verification/VerifyIlluminaGenotypingArray.wdl'
      - 'verification/test-wdls/TestIlluminaGenotypingArray.wdl'
      - 'tasks/broad/Utilities.wdl'
      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
      - '.github/workflows/test_illumina_genotyping_array.yml'


  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
    inputs:
      useCallCache:
        description: 'Use call cache (default: true)'
        required: false
        default: "true"
      updateTruth:
        description: 'Update truth files (default: false)'
        required: false
        default: "false"
      testType:
        description: 'Specify the type of test (Plumbing or Scientific)'
        required: true
      truthBranch:
        description: 'Specify the branch for truth files (default: master)'
        required: false
        default: "master"


env:
  PROJECT_NAME: WARP
  REPOSITORY_NAME: ${{ github.event.repository.name }}
  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
  PIPELINE_NAME: IlluminaGenotypingArray
  DOCKSTORE_PIPELINE_NAME: IlluminaGenotypingArray


jobs:
  run_pipeline:
    runs-on: ubuntu-latest
    # Add "id-token" with the intended permissions.
    permissions:
      contents: 'read'
      id-token: 'write'

    steps:
      # actions/checkout MUST come before auth
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.ref }}

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          pwd
          cd scripts/firecloud_api/
          pip install -r requirements.txt

      - name: Set Branch Name
        id: set_branch
        run: |
          if [ -z "${{ github.head_ref }}" ]; then
            echo "Branch name is missing, using ${GITHUB_REF##*/}"
            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
          else
            echo "Branch name from PR: ${{ github.head_ref }}"
            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
          fi

      - name: Fetch Dockstore Workflow CommitID
        run: |
          # Wait for Dockstore to update
          sleep 1
          # Capture the output of the script (commit ID)
          DOCKSTORE_COMMIT_HASH_FROM_FETCH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \
            $DOCKSTORE_TOKEN \
            $DOCKSTORE_PIPELINE_NAME \
            $branch_name)
          # Export the commit hash as an environment variable
          echo "DOCKSTORE_COMMIT_HASH=$DOCKSTORE_COMMIT_HASH_FROM_FETCH" >> $GITHUB_ENV
          echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH_FROM_FETCH"

        env:
          ## TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT ##
          DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
          DOCKSTORE_PIPELINE_NAME: ${{ env.DOCKSTORE_PIPELINE_NAME }}
          branch_name: ${{ env.branch_name }}

      - name: Fetch Github Commit Hash
        id: fetch_github_commit_hash
        run: |
          echo "GITHUB_COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
          echo "GitHub Commit Hash: ${{ github.sha }}"

  #    - name: Compare Dockstore and Commit Hashes
  #      id: compare_hashes
  #      run: |
  #        echo "Comparing hashes..."
  #        echo "Dockstore Commit Hash: $DOCKSTORE_COMMIT_HASH"
  #        echo "GitHub Commit Hash: $GITHUB_COMMIT_HASH"
#
  #        if [ "$DOCKSTORE_COMMIT_HASH" != "$GITHUB_COMMIT_HASH" ]; then
  #          echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash!"
  #          echo "Mismatch found: $DOCKSTORE_COMMIT_HASH != $GITHUB_COMMIT_HASH"
  #          exit 1
  #        else
  #          echo "Success: The Dockstore Commit Hash matches the GitHub Commit Hash."
  #        fi
  #      env:
  #        DOCKSTORE_COMMIT_HASH: ${{ env.DOCKSTORE_COMMIT_HASH }}
  #        GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}

      - name: Check gcloud auth list manually
        run: |
          # Decode the Base64 service account key and save it to a temporary file
          echo "${{ secrets.PDT_TESTER_SA_B64 }}" | base64 --decode > /tmp/service-account-key.json
    
          gcloud auth activate-service-account pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com --key-file=/tmp/service-account-key.json
          gcloud auth list
          
          rm /tmp/service-account-key.json
        env:
          SA_JSON_B64:
            ${{ secrets.PDT_TESTER_SA_B64 }}

      - name: Check gcloud auth list
        run: |
          python scripts/firecloud_api/gcloudauth.py --key-file="${{ secrets.PDT_TESTER_SA_B64 }}"
        env:
          SA_JSON_B64:
            ${{ secrets.PDT_TESTER_SA_B64 }}

      - name: Perform File Copy with gsutil
        run: |
          python scripts/firecloud_api/firecloud_api.py gsutil_copy \
            --sa-json-b64 "$SA_JSON_B64" \
            --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
            --source "gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt" \
            --destination . \
            --workspace-namespace warp-pipelines \
            --workspace-name "WARP Tests"
        env:
          SA_JSON_B64:
            ${{ secrets.PDT_TESTER_SA_B64 }}

      - name: Set Test Type for PRs
        if: ${{ github.event_name == 'pull_request' }}
        id: set_test_type
        run: |
          # Default to "Scientific" if targeting master
          if [ "${{ github.base_ref }}" == "master" ]; then
            echo "testType=Scientific" >> $GITHUB_ENV
          else
            echo "testType=Plumbing" >> $GITHUB_ENV
          fi

      - name: Use Provided Test Type
        if: ${{ github.event_name == 'workflow_dispatch' }}
        id: use_provided_test_type
        run: |
          # Use the testType provided by the user
          echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV

      - name: Update test inputs, Upload to Terra, Submit, Monitor and Retrieve Outputs
        run: |
          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
          MAX_RETRIES=2
          RETRY_DELAY=300  # 300 seconds = 5 minutes
          # Initialize variables to aggregate statuses and outputs
          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
          ALL_OUTPUTS=""
          # Initialize arrays to track submission and workflow statuses
          declare -a SUBMISSION_IDS
          declare -A WORKFLOW_STATUSES
          
          
          # Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
          if [ "$UPDATE_TRUTH" = "true" ]; then
              UPDATE_TRUTH_BOOL=true
          else
              UPDATE_TRUTH_BOOL=false
          fi
          
          if [ "$USE_CALL_CACHE" == "true" ]; then
              USE_CALL_CACHE_BOOL=true
          else
              USE_CALL_CACHE_BOOL=false
          fi
          
          PIPELINE_NAME="TestIlluminaGenotypingArray"
          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
          TEST_TYPE="${{ env.testType }}"
          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
          echo "Running tests with test type: $TEST_TYPE"
          
          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
          echo "Truth path: $TRUTH_PATH"
          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
          
          # Create the submission_data.json file which will be the same for all inputs
          SUBMISSION_DATA_FILE="submission_data.json"
          
          # Use a heredoc to generate the JSON file content dynamically
          cat <<EOF > "$SUBMISSION_DATA_FILE"
          {
            "methodConfigurationNamespace": "warp-pipelines",
            "methodConfigurationName": "$PIPELINE_NAME",
            "useCallCache": $USE_CALL_CACHE_BOOL,
            "deleteIntermediateOutputFiles": false,
            "useReferenceDisks": true,
            "memoryRetryMultiplier": 1.2,
            "workflowFailureMode": "NoNewCalls",
            "userComment": "Automated submission",
            "ignoreEmptyOutputs": false
          }
          EOF
          
          echo "Created submission data file: $SUBMISSION_DATA_FILE"
          
          # 1. Submit all jobs first and store their submission IDs
          for input_file in "$INPUTS_DIR"/*.json; do
            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                --results_path "$RESULTS_PATH" \
                --inputs_json "$input_file" \
                --update_truth "$UPDATE_TRUTH_BOOL" \
                --commit_hash "$GITHUB_COMMIT_HASH" )
            echo "Uploading the test input file: $test_input_file"
            python3 scripts/firecloud_api/firecloud_api.py \
                upload_test_inputs \
                --workspace-namespace warp-pipelines \
                --workspace-name "WARP Tests" \
                --pipeline_name "$PIPELINE_NAME" \
                --test_input_file "$test_input_file" \
                --branch_name "$branch_name" \
                --sa-json-b64 "$SA_JSON_B64" \
                --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com"
      
            attempt=1
            while [ $attempt -le $MAX_RETRIES ]; do
              SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py submit_job \
               --workspace-namespace "warp-pipelines" \
               --workspace-name "WARP Tests" \
               --sa-json-b64 "$SA_JSON_B64" \
               --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
               --submission_data_file "$SUBMISSION_DATA_FILE")
      
              echo "Submission ID: $SUBMISSION_ID"
            
              if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
                  echo "Error in submission, retrying in $RETRY_DELAY seconds..."
                  ((attempt++))
                  if [ $attempt -gt $MAX_RETRIES ]; then
                      echo "Max retries reached. Exiting..."
                      exit 1
                  fi
                  sleep $RETRY_DELAY
                  continue
              fi
            
              echo "Submission successful. Submission ID: $SUBMISSION_ID"
              SUBMISSION_IDS+=("$SUBMISSION_ID")
              break
            done
          done
          
          echo "All jobs have been submitted. Starting to poll for statuses..."
      
          # 2. After all submissions are done, start polling for statuses of all jobs
          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
            attempt=1
            while [ $attempt -le $MAX_RETRIES ]; do
              echo "Polling for Submission ID: $SUBMISSION_ID"
              RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py poll_job_status \
                  --submission_id "$SUBMISSION_ID" \
                  --sa-json-b64 "$SA_JSON_B64" \
                  --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                  --workspace-namespace "warp-pipelines" \
                  --workspace-name "WARP Tests")
              
              if [ -z "$RESPONSE" ]; then
                  echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
                  ((attempt++))
                  if [ $attempt -gt $MAX_RETRIES ]; then
                      echo "Max retries reached. Exiting..."
                      exit 1
                  fi
                  sleep $RETRY_DELAY
                  continue
              fi
      
              WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
              WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
          
              # retrieve workflow outputs
              echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
              for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
                   WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py get_workflow_outputs \
                    --user "pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com" \
                    --sa-json-b64 "$SA_JSON_B64" \
                    --submission_id "$SUBMISSION_ID" \
                    --workspace-namespace warp-pipelines \
                    --workspace-name "WARP Tests" \
                    --workflow_id "$WORKFLOW_ID" \
                    --pipeline_name "$PIPELINE_NAME")
                    ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
                done
              break
            done
            # Generate final summary tables with hyperlinks for Submission IDs
            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
              # Generate the Terra URL for the submission
              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
          
              # Add the Submission ID as a hyperlink
              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
          
              # Add the workflows and statuses for this submission
              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
          
              # Add a blank line for separation
              echo "" >> $GITHUB_STEP_SUMMARY
            done
          done
        env:
          PIPELINE_NAME: TestIlluminaGenotypingArray
          NAMESPACE: warp-pipelines
          WORKSPACE: WARP Tests

      - name: Print Summary on Success
        if: success()
        run: |
          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY

      - name: Print Summary on Failure
        if: failure()
        run: |
          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY