testing pr - Np jw test illumina genotyping arrays #182
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Illumina Genotyping Array | |
| # Workflow triggers | |
| on: | |
| pull_request: | |
| branches: [ "develop", "staging", "master" ] | |
| paths: | |
| - 'pipelines/broad/genotyping/illumina/**' | |
| - 'tasks/**' | |
| - 'verification/**' | |
| - '.github/workflows/test_illumina_genotyping_array.yml' | |
| workflow_dispatch: | |
| inputs: | |
| useCallCache: | |
| description: 'Use call cache (default: true)' | |
| required: false | |
| default: "true" | |
| updateTruth: | |
| description: 'Update truth files (default: false)' | |
| required: false | |
| default: "false" | |
| env: | |
| PROJECT_NAME: WARP | |
| REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| jobs: | |
| run_pipeline: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - uses: 'actions/checkout@v3' | |
| - name: Check working directory | |
| run: | | |
| echo "Current working directory:" | |
| pwd | |
| echo "Listing files in the working directory:" | |
| ls -alh | |
| - id: 'auth' | |
| name: 'Authenticate to Google Cloud' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider' | |
| service_account: '[email protected]' | |
| access_token_lifetime: '3600' | |
| access_token_scopes: 'profile, email, openid' | |
| - name: Submit job, poll status, and get outputs | |
| id: pipeline_run | |
| run: | | |
| TOKEN="${{ steps.auth.outputs.access_token }}" | |
| NAMESPACE="warp-pipelines" | |
| WORKSPACE="WARP Tests" | |
| USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}" | |
| UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}" | |
| TEST_TYPE="Plumbing" | |
| CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S") | |
| PIPELINE_NAME="IlluminaGenotypingArray" | |
| PIPELINE_DIR="pipelines/broad/genotyping/illumina" | |
| TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth" | |
| RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME" | |
| # Create JSON for submission data | |
| SUBMISSION_DATA_FILE="submission_data.json" | |
| CALL_CACHE_VALUE="true" | |
| if [ "$USE_CALL_CACHE" = "false" ]; then | |
| CALL_CACHE_VALUE="false" | |
| fi | |
| cat <<EOF > "$SUBMISSION_DATA_FILE" | |
| { | |
| "methodConfigurationNamespace": "warp-pipelines", | |
| "methodConfigurationName": "$PIPELINE_NAME", | |
| "useCallCache": $CALL_CACHE_VALUE, | |
| "deleteIntermediateOutputFiles": false, | |
| "useReferenceDisks": true, | |
| "memoryRetryMultiplier": 1.2, | |
| "workflowFailureMode": "NoNewCalls", | |
| "userComment": "Automated submission", | |
| "ignoreEmptyOutputs": false | |
| } | |
| EOF | |
| INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE | |
| for input_file in $INPUTS_DIR/*; do | |
| test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \ | |
| --truth_path $TRUTH_PATH \ | |
| --results_path $RESULTS_PATH \ | |
| --inputs_json $input_file \ | |
| --update_truth $UPDATE_TRUTH) | |
| python3 scripts/firecloud_api/firecloud_api.py \ | |
| --token "$TOKEN" \ | |
| --namespace "$NAMESPACE" \ | |
| --workspace "$WORKSPACE" \ | |
| --action upload_file \ | |
| --pipeline_name "$PIPELINE_NAME" \ | |
| --test_input_file "$test_input_file" | |
| SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py \ | |
| --token "$TOKEN" \ | |
| --namespace "$NAMESPACE" \ | |
| --workspace "$WORKSPACE" \ | |
| --action submit \ | |
| --submission_data_file "$SUBMISSION_DATA_FILE") | |
| if [ -z "$SUBMISSION_ID" ]; then | |
| echo "Submission failed for input: $input_file" | |
| continue | |
| fi | |
| echo "Submission ID for input $input_file: $SUBMISSION_ID" | |
| RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py \ | |
| --token "$TOKEN" \ | |
| --namespace "$NAMESPACE" \ | |
| --workspace "$WORKSPACE" \ | |
| --action poll_status \ | |
| --submission_id "$SUBMISSION_ID") | |
| echo "Workflows for input $input_file and their statuses:" | |
| echo "$RESPONSE" | jq | |
| for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do | |
| OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py \ | |
| --token "$TOKEN" \ | |
| --namespace "$NAMESPACE" \ | |
| --workspace "$WORKSPACE" \ | |
| --action get_outputs \ | |
| --submission_id "$SUBMISSION_ID" \ | |
| --workflow_id "$WORKFLOW_ID" \ | |
| --pipeline_name "$PIPELINE_NAME") | |
| echo "Output for workflow $WORKFLOW_ID: $OUTPUT" | |
| done | |
| done |