Np jw test illumina genotyping arrays 2

.github/workflows/test_illumina_genotyping_array.yml

@@ -1,4 +1,3 @@
-
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -10,23 +9,38 @@ on:
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
+
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
       useCallCache:
         description: 'Use call cache (default: true)'
         required: false
         default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: true
+
+
 env:
   PROJECT_NAME: WARP
   # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
 
+
 jobs:
   run_pipeline:
     runs-on: ubuntu-latest
@@ -36,156 +50,224 @@ jobs:
       id-token: 'write'
 
     steps:
-    # actions/checkout MUST come before auth
-    - uses: 'actions/checkout@v3'
-
-    - id: 'auth'
-      name: 'Authenticate to Google Cloud'
-      uses: 'google-github-actions/auth@v2'
-      with:
-        token_format: 'access_token'
-        # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-        # This is provided by the DevOps team - do not change!
-        workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-        # This is our tester service account
-        service_account: '[email protected]'
-        access_token_lifetime: '3600' #seconds, default is 3600
-        access_token_scopes: 'profile, email, openid'
-
-    # ... further steps are automatically authenticated
-    - name: Check working directory
-      run: |
-        echo "Current directory:"
-        pwd
-        ls -lht
-
-    - name: Submit job, poll status, and get outputs
-      id: pipeline_run
-      run: |
-          # Set these environment variables
+      # actions/checkout MUST come before auth
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - id: 'auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          token_format: 'access_token'
+          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+          # This is provided by the DevOps team - do not change!
+          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          # This is our tester service account
+          service_account: '[email protected]'
+          access_token_lifetime: '3600' # seconds, default is 3600
+          access_token_scopes: 'profile, email, openid'
+
+      # Set branch name.
+      # github.head_ref contains the name of the branch in the context of a pull request
+      # if github.head_ref is empty, it implies the workflow was triggered manually
+      # ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
+      # The ##*/ is a parameter expansion that removes the refs/heads/ prefix, leaving just the branch name.
+      - name: Set Branch Name
+        id: set_branch
+        run: |
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Check if Branch has been Pushed Before
+        id: check_branch_push
+        run: |
+          BRANCH_NAME="${{ env.branch_name }}"
+          REPO_OWNER="${{ github.repository_owner }}"
+          REPO_NAME="${{ github.event.repository.name }}"
+
+          # Check if the branch exists remotely
+          BRANCH_EXISTS=$(curl -s "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/branches/${BRANCH_NAME}" | jq -r '.name')
+
+          # If the branch does not exist, it's the first time the branch is being pushed.
+          if [ "$BRANCH_EXISTS" == "$BRANCH_NAME" ]; then
+            # Branch exists, check the last commit timestamp
+            COMMITS=$(curl -s "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/commits?sha=${BRANCH_NAME}")
+
+            # Get the timestamp of the most recent commit
+            LATEST_COMMIT_DATE=$(echo "$COMMITS" | jq -r '.[0].commit.committer.date')
+
+            # Convert the commit date to Unix timestamp
+            LATEST_COMMIT_TIMESTAMP=$(date -d "$LATEST_COMMIT_DATE" +%s)
+
+            # Get the current timestamp
+            CURRENT_TIMESTAMP=$(date +%s)
+
+            # Calculate the time difference in seconds
+            TIME_DIFF=$((CURRENT_TIMESTAMP - LATEST_COMMIT_TIMESTAMP))
+
+            # If the branch was pushed within the last 60 seconds, delay the pipeline by 5 minutes (300 seconds)
+            if [ "$TIME_DIFF" -le 60 ]; then
+              echo "Branch was pushed within the last 60 seconds. Delaying pipeline by 5 minutes."
+              sleep 300
+            else
+              echo "Branch was not pushed in the last 60 seconds. Proceeding with pipeline."
+            fi
+          else
+            echo "Branch has not been pushed before. Proceeding with pipeline."
+          fi
+
+
+      - name: Submit job, poll status, and get outputs
+        id: pipeline_run
+        run: |
+          # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
-          PIPELINE_NAME="IlluminaGenotypingArray"
-          USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
+          PIPELINE_NAME="TestIlluminaGenotypingArray"
+          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
+          # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
+          # We may want to keep the truth and resuts buckets separate for TTL reasons
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
+          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script
           firecloud_action() {
-              python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+            python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
           }
 
-          # Create the submission_data.json file
-          SUBMISSION_DATA_FILE="submission_data.json"
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$USE_CALL_CACHE" = "true" ]; then
-              USE_CALL_CACHE_BOOL=true
+            if [ "$USE_CALL_CACHE" == "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+            else
+                USE_CALL_CACHE_BOOL=false
+            fi
+
+
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
           else
-              USE_CALL_CACHE_BOOL=false
+              UPDATE_TRUTH_BOOL=false
           fi
+
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
             "methodConfigurationName": "$PIPELINE_NAME",
             "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": true,
+            "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,
             "workflowFailureMode": "NoNewCalls",
             "userComment": "Automated submission",
             "ignoreEmptyOutputs": false
           }
           EOF
-
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
-
-          # 1. Submit a new workflow using the generated submission_data.json
-          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-
-          # Check if submission was successful
-          if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed."  # Log failure to stdout
-              echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
-              exit 1
-          fi
-
-          echo "Submission ID: $SUBMISSION_ID"
-          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
 
-          # 2. Poll submission status and get workflow IDs and statuses
-          echo "Polling submission status..."
-          RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
 
-          # Parse the JSON response to get the workflow ID and statuses
-          echo "Workflows and their statuses:"
-          echo "$RESPONSE" | jq
+          # Loop through each file in the appropriate test inputs directory
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+
+          for input_file in "$INPUTS_DIR"/*.json; do
+            echo "Processing input file: $input_file"
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                                                                                --results_path "$RESULTS_PATH" \
+                                                                                --inputs_json "$input_file" \
+                                                                                --update_truth "$UPDATE_TRUTH_BOOL")
+            echo "Uploading the test input file: $test_input_file"
+            echo "Branch name: $branch_name"
+
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
 
-          # Check if RESPONSE is empty
-          if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
-              exit 1
-          fi
+            echo "Submitting job for input file: $input_file"
+            cat "$SUBMISSION_DATA_FILE"
+            SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
 
-          # Extract workflows and their statuses
-          WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-          echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
+            if [ -z "$SUBMISSION_ID" ]; then
+              echo "Submission failed for input file: $input_file"
+              continue
+            fi
 
-          # Generate markdown summary table for workflows and statuses
-          WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
+            echo "Submission ID: $SUBMISSION_ID"
+            SUBMISSION_IDS+=("$SUBMISSION_ID")
+          done
 
-          # Print workflow table to stdout
-          echo "$WORKFLOW_TABLE"
+          echo "Monitoring the status of submitted workflows..."
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 
-          # 3. Iterate over the Workflow IDs to get outputs
-          OUTPUTS=""
-          echo "Retrieving workflow outputs..."
-          for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
+
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+
+            # retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
               WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-          done
-          echo "Workflow outputs retrieved successfully."
-          echo "Raw output before jq:"
-          echo "$OUTPUTS"
-          echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
-
-          # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
-          OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
-          #print outputs table to stdout
-          echo "$OUTPUTS_TABLE"
-
-    - name: Print Summary on Success
-      if: success()
-      run: |
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+            done
+
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+
+      - name: Print Summary on Success
+        if: success()
+        run: |
           echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo " :shipit: " >> $GITHUB_STEP_SUMMARY
-
-    - name: Print Summary on Failure
-      if: failure()
-      run: |
-          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY