Skip to content

Test Illumina Genotyping Array #599

Test Illumina Genotyping Array

Test Illumina Genotyping Array #599

name: Test Illumina Genotyping Array
# Controls when the workflow will run
on:
#run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
# push:
# branches:
# - kp_GHA_Terra_auth_PD-2682
pull_request:
branches: [ "develop", "staging", "master" ]
# Only run if files in these paths changed:
####################################
# SET PIPELINE SPECIFIC PATHS HERE #
####################################
paths:
- 'pipelines/broad/genotyping/illumina/**'
- 'tasks/broad/IlluminaGenotypingArrayTasks.wdl'
- 'tasks/broad/Qc.wdl'
- 'verification/VerifyIlluminaGenotypingArray.wdl'
- 'verification/test-wdls/TestIlluminaGenotypingArray.wdl'
- 'tasks/broad/Utilities.wdl'
- 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
- '.github/workflows/test_illumina_genotyping_array.yml'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
inputs:
useCallCache:
description: 'Use call cache (default: true)'
required: false
default: "true"
updateTruth:
description: 'Update truth files (default: false)'
required: false
default: "false"
testType:
description: 'Specify the type of test (Plumbing or Scientific)'
required: true
truthBranch:
description: 'Specify the branch for truth files (default: master)'
required: false
default: "master"
env:
PROJECT_NAME: WARP
# Github repo name
REPOSITORY_NAME: ${{ github.event.repository.name }}
SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
jobs:
run_pipeline:
runs-on: ubuntu-latest
# Add "id-token" with the intended permissions.
permissions:
contents: 'read'
id-token: 'write'
steps:
# Add a step to wait to account for github -> dockstore -> terra delays
- name: Wait Before Starting
run: |
echo "Waiting for 5 minutes before starting..."
sleep 1 # time in seconds, update this when we really want a delay
# actions/checkout MUST come before auth
- uses: actions/checkout@v3
with:
ref: ${{ github.ref }}
# id: 'auth'
# name: 'Authenticate to Google Cloud'
# uses: 'google-github-actions/auth@v2'
# with:
# token_format: 'access_token'
# # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
# # This is provided by the DevOps team - do not change!
# workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
# # This is our tester service account
# service_account: '[email protected]'
# access_token_lifetime: '3600' # seconds, default is 3600
# access_token_scopes: 'profile, email, openid'
- name: Set up python
id: setup-python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: |
pwd
cd scripts/firecloud_api/
pip install -r requirements.txt
- name: Set Commit Hash
id: set_commit_hash
run: echo "COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
# Set the branch name.
# github.head_ref contains the name of the branch in the context of a pull request
# if github.head_ref is empty, it implies the workflow was triggered manually
# ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
# The ##*/ is a parameter expansion that removes the refs/heads/ prefix, leaving just the branch name.
- name: Set Branch Name
id: set_branch
run: |
if [ -z "${{ github.head_ref }}" ]; then
echo "Branch name is missing, using ${GITHUB_REF##*/}"
echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
else
echo "Branch name from PR: ${{ github.head_ref }}"
echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
fi
- name: Set Test Type for PRs
if: ${{ github.event_name == 'pull_request' }}
id: set_test_type
run: |
# Default to "Scientific" if targeting master
if [ "${{ github.base_ref }}" == "master" ]; then
echo "testType=Scientific" >> $GITHUB_ENV
else
echo "testType=Plumbing" >> $GITHUB_ENV
fi
- name: Use Provided Test Type
if: ${{ github.event_name == 'workflow_dispatch' }}
id: use_provided_test_type
run: |
# Use the testType provided by the user
echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
- name: Update test inputs and Upload to Terra
run: |
UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
MAX_RETRIES=2
RETRY_DELAY=300 # 300 seconds = 5 minutes
# Initialize variables to aggregate statuses and outputs
ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
ALL_OUTPUTS=""
# Initialize arrays to track submission and workflow statuses
declare -a SUBMISSION_IDS
declare -A WORKFLOW_STATUSES
# Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
if [ "$UPDATE_TRUTH" = "true" ]; then
UPDATE_TRUTH_BOOL=true
else
UPDATE_TRUTH_BOOL=false
fi
if [ "$USE_CALL_CACHE" == "true" ]; then
USE_CALL_CACHE_BOOL=true
else
USE_CALL_CACHE_BOOL=false
fi
PIPELINE_NAME="TestIlluminaGenotypingArray"
PIPELINE_DIR="pipelines/broad/genotyping/illumina"
TEST_TYPE="${{ env.testType }}"
INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
echo "Running tests with test type: $TEST_TYPE"
TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
echo "Truth path: $TRUTH_PATH"
RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
# Create the submission_data.json file which will be the same for all inputs
SUBMISSION_DATA_FILE="submission_data.json"
# Use a heredoc to generate the JSON file content dynamically
cat <<EOF > "$SUBMISSION_DATA_FILE"
{
"methodConfigurationNamespace": "warp-pipelines",
"methodConfigurationName": "$PIPELINE_NAME",
"useCallCache": $USE_CALL_CACHE_BOOL,
"deleteIntermediateOutputFiles": false,
"useReferenceDisks": true,
"memoryRetryMultiplier": 1.2,
"workflowFailureMode": "NoNewCalls",
"userComment": "Automated submission",
"ignoreEmptyOutputs": false
}
EOF
echo "Created submission data file: $SUBMISSION_DATA_FILE"
# 1. Submit all jobs first and store their submission IDs
for input_file in "$INPUTS_DIR"/*.json; do
test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
--results_path "$RESULTS_PATH" \
--inputs_json "$input_file" \
--update_truth "$UPDATE_TRUTH_BOOL" \
--commit_hash "$COMMIT_HASH" )
echo "Uploading the test input file: $test_input_file"
python3 scripts/firecloud_api/firecloud_api2.py \
upload_test_inputs \
--workspace-namespace warp-pipelines \
--workspace-name "WARP Tests" \
--pipeline_name "$PIPELINE_NAME" \
--test_input_file "$test_input_file" \
--branch_name "$branch_name" \
--sa-json-b64 "$SA_JSON_B64" \
--user "[email protected]"
attempt=1
while [ $attempt -le $MAX_RETRIES ]; do
SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
--workspace-namespace "warp-pipelines" \
--workspace-name "WARP Tests" \
--sa-json-b64 "$SA_JSON_B64" \
--user "[email protected]" \
--submission_data_file "$SUBMISSION_DATA_FILE")
echo "Submission ID: $SUBMISSION_ID"
if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
echo "Error in submission, retrying in $RETRY_DELAY seconds..."
((attempt++))
if [ $attempt -gt $MAX_RETRIES ]; then
echo "Max retries reached. Exiting..."
exit 1
fi
sleep $RETRY_DELAY
continue
fi
echo "Submission successful. Submission ID: $SUBMISSION_ID"
SUBMISSION_IDS+=("$SUBMISSION_ID")
break
done
done
echo "All jobs have been submitted. Starting to poll for statuses..."
# 2. After all submissions are done, start polling for statuses of all jobs
for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
attempt=1
while [ $attempt -le $MAX_RETRIES ]; do
echo "Polling for Submission ID: $SUBMISSION_ID"
RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
--submission_id "$SUBMISSION_ID" \
--sa-json-b64 "$SA_JSON_B64" \
--user "[email protected]" \
--workspace-namespace "warp-pipelines" \
--workspace-name "WARP Tests")
if [ -z "$RESPONSE" ]; then
echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
((attempt++))
if [ $attempt -gt $MAX_RETRIES ]; then
echo "Max retries reached. Exiting..."
exit 1
fi
sleep $RETRY_DELAY
continue
fi
WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
# retrieve workflow outputs
echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_workflow_outputs \
--user "[email protected]" \
--sa-json-b64 "$SA_JSON_B64" \
--submission_id "$SUBMISSION_ID" \
--workspace-namespace warp-pipelines \
--workspace-name "WARP Tests" \
--workflow_id "$WORKFLOW_ID" \
--pipeline_name "$PIPELINE_NAME")
ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
done
break
done
# Generate final summary tables with hyperlinks for Submission IDs
echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
# Generate the Terra URL for the submission
SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
# Add the Submission ID as a hyperlink
echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
# Add the workflows and statuses for this submission
echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
# Add a blank line for separation
echo "" >> $GITHUB_STEP_SUMMARY
done
done
# - name: Monitor Workflow Status
# run: |
# echo "Monitoring the status of submitted workflows..."
# # Convert the space-separated string into an array
# IFS=' ' read -r -a submission_ids_array <<< "$submission_ids"
#
# # Ensure the array is populated
# if [ ${#submission_ids_array[@]} -eq 0 ]; then
# echo "No submission IDs found. Exiting..."
# exit 1
# fi
#
# declare -A WORKFLOW_STATUSES
#
# echo "submission_ids_array: ${submission_ids_array[@]}"
#
# for SUBMISSION_ID in "${submission_ids_array[@]}"; do
# echo "Polling submission status for Submission ID: $SUBMISSION_ID"
# RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
# --submission_id "$SUBMISSION_ID" \
# --sa-json-b64 "$SA_JSON_B64" \
# --user "[email protected]" \
# --workspace-namespace "warp-pipelines" \
# --workspace-name "WARP Tests")
#
# if [ -z "$RESPONSE" ]; then
# echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
# continue
# fi
#
# # Debug raw response
# echo "Raw Workflow Statuses:"
# echo "$RESPONSE"
#
# # Parse and store workflow statuses
# echo "Parsing workflow statuses for submissions"
# WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
# echo "Raw Workflow Statuses: $WORKFLOW_STATUSES_FOR_SUBMISSION"
# #up until here there are no issues with the script
#
# # Remove duplicates and sanitize
# CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | awk '!seen[$0]++')
# # Debug cleaned statuses
# echo "Cleaned Workflow Statuses: $CLEANED_WORKFLOW_STATUSES"
#
# # Reset and assign to associative array
# WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
#
# #Debug stored status
# echo "Stored Workflow Statuses for $SUBMISSION_ID: ${WORKFLOW_STATUSES[$SUBMISSION_ID]}"
#
# # Append to aggregate statuses
# echo "Appending to aggregate statuses"
# CLEANED_WORKFLOW_STATUSES=$(echo "$WORKFLOW_STATUSES_FOR_SUBMISSION" | tr -d '\n' | tr -d '\r')
# echo "CLEANED_WORKFLOW_STATUSES: $CLEANED_WORKFLOW_STATUSES"
# WORKFLOW_STATUSES["$SUBMISSION_ID"]="$CLEANED_WORKFLOW_STATUSES"
# done
# # # Retrieve workflow outputs
# # echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
# # for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
# # WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_outputs \
# # --submission_id "$SUBMISSION_ID" \
# # --workflow_id "$WORKFLOW_ID" \
# # --pipeline_name "$PIPELINE_NAME")
# # ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
# # done
# # done
# # Generate summary for Submission IDs
# echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
# for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
# # Generate the Terra URL for the submission
# SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
#
# # Add the Submission ID as a hyperlink
# echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
#
# # Add the workflows and statuses for this submission
# echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
#
# # Add a blank line for separation
# echo "" >> $GITHUB_STEP_SUMMARY
# done
env:
#SUBMISSION_IDS: ${{ steps.submit_jobs.outputs.submission_ids }} # Pass IDs from a previous step
PIPELINE_NAME: TestIlluminaGenotypingArray
NAMESPACE: warp-pipelines
WORKSPACE: WARP Tests
#- name: Download Commit Hash from GCP
# run: |
# gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
#- name: Check Commit Hash
# id: check_commit_hash
# run: |
# # Read the commit hash from the downloaded file
# COMMIT_HASH_FROM_WDL=$(cat commit_hash.txt)
#
# # Compare the two commit hashes
# if [ "$COMMIT_HASH_FROM_WDL" != "${{ env.COMMIT_HASH }}" ]; then
# echo "Error: The commit hash from the WDL output does not match the expected commit hash."
# exit 1
# else
# echo "Commit hash match successful: $COMMIT_HASH_FROM_WDL"
# fi
- name: Print Summary on Success
if: success()
run: |
echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
- name: Print Summary on Failure
if: failure()
run: |
echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY