-
Notifications
You must be signed in to change notification settings - Fork 98
337 lines (295 loc) · 13.9 KB
/
test_illumina_genotyping_array.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
name: Test Illumina Genotyping Array
# Controls when the workflow will run
on:
#run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
# push:
# branches:
# - kp_GHA_Terra_auth_PD-2682
pull_request:
branches: [ "develop", "staging", "master" ]
# Only run if files in these paths changed:
####################################
# SET PIPELINE SPECIFIC PATHS HERE #
####################################
paths:
- 'pipelines/broad/genotyping/illumina/**'
- 'tasks/broad/IlluminaGenotypingArrayTasks.wdl'
- 'tasks/broad/Qc.wdl'
- 'verification/VerifyIlluminaGenotypingArray.wdl'
- 'verification/test-wdls/TestIlluminaGenotypingArray.wdl'
- 'tasks/broad/Utilities.wdl'
- 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
- '.github/workflows/test_illumina_genotyping_array.yml'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
inputs:
useCallCache:
description: 'Use call cache (default: true)'
required: false
default: "true"
updateTruth:
description: 'Update truth files (default: false)'
required: false
default: "false"
testType:
description: 'Specify the type of test (Plumbing or Scientific)'
required: true
truthBranch:
description: 'Specify the branch for truth files (default: master)'
required: false
default: "master"
env:
PROJECT_NAME: WARP
# Github repo name
REPOSITORY_NAME: ${{ github.event.repository.name }}
SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
jobs:
run_pipeline:
runs-on: ubuntu-latest
# Add "id-token" with the intended permissions.
permissions:
contents: 'read'
id-token: 'write'
steps:
# Add a step to wait to account for github -> dockstore -> terra delays
- name: Wait Before Starting
run: |
echo "Waiting for 5 minutes before starting..."
sleep 1 # time in seconds, update this when we really want a delay
# actions/checkout MUST come before auth
- uses: actions/checkout@v3
with:
ref: ${{ github.ref }}
# id: 'auth'
# name: 'Authenticate to Google Cloud'
# uses: 'google-github-actions/auth@v2'
# with:
# token_format: 'access_token'
# # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
# # This is provided by the DevOps team - do not change!
# workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
# # This is our tester service account
# service_account: '[email protected]'
# access_token_lifetime: '3600' # seconds, default is 3600
# access_token_scopes: 'profile, email, openid'
- name: Set up python
id: setup-python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: |
pwd
cd scripts/firecloud_api/
pip install -r requirements.txt
- name: Fetch Dockstore Workflow ID
run: |
python scripts/dockstore_api/fetch_dockstore_commit.py $DOCKSTORE_TOKEN github.com%2Fbroadinstitute%2Fwarp%2FIlluminaGenotypingArray BIOWORKFLOW np_jw_test_illumina_genotyping_arrays
env:
##TODO NEED TO ADD DOCKSTORE_TOKEN FOR SERVICE ACCOUNT##
DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
- name: Set Commit Hash
id: set_commit_hash
run: echo "COMMIT_HASH=${{ github.sha }}" >> $GITHUB_ENV
# Set the branch name.
# github.head_ref contains the name of the branch in the context of a pull request
# if github.head_ref is empty, it implies the workflow was triggered manually
# ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
# The ##*/ is a parameter expansion that removes the refs/heads/ prefix, leaving just the branch name.
- name: Set Branch Name
id: set_branch
run: |
if [ -z "${{ github.head_ref }}" ]; then
echo "Branch name is missing, using ${GITHUB_REF##*/}"
echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
else
echo "Branch name from PR: ${{ github.head_ref }}"
echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
fi
- name: Set Test Type for PRs
if: ${{ github.event_name == 'pull_request' }}
id: set_test_type
run: |
# Default to "Scientific" if targeting master
if [ "${{ github.base_ref }}" == "master" ]; then
echo "testType=Scientific" >> $GITHUB_ENV
else
echo "testType=Plumbing" >> $GITHUB_ENV
fi
- name: Use Provided Test Type
if: ${{ github.event_name == 'workflow_dispatch' }}
id: use_provided_test_type
run: |
# Use the testType provided by the user
echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
- name: Update test inputs and Upload to Terra
run: |
UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
MAX_RETRIES=2
RETRY_DELAY=300 # 300 seconds = 5 minutes
# Initialize variables to aggregate statuses and outputs
ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
ALL_OUTPUTS=""
# Initialize arrays to track submission and workflow statuses
declare -a SUBMISSION_IDS
declare -A WORKFLOW_STATUSES
# Convert UPDATE_TRUTH and USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
if [ "$UPDATE_TRUTH" = "true" ]; then
UPDATE_TRUTH_BOOL=true
else
UPDATE_TRUTH_BOOL=false
fi
if [ "$USE_CALL_CACHE" == "true" ]; then
USE_CALL_CACHE_BOOL=true
else
USE_CALL_CACHE_BOOL=false
fi
PIPELINE_NAME="TestIlluminaGenotypingArray"
PIPELINE_DIR="pipelines/broad/genotyping/illumina"
TEST_TYPE="${{ env.testType }}"
INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
echo "Running tests with test type: $TEST_TYPE"
TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
echo "Truth path: $TRUTH_PATH"
RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
# Create the submission_data.json file which will be the same for all inputs
SUBMISSION_DATA_FILE="submission_data.json"
# Use a heredoc to generate the JSON file content dynamically
cat <<EOF > "$SUBMISSION_DATA_FILE"
{
"methodConfigurationNamespace": "warp-pipelines",
"methodConfigurationName": "$PIPELINE_NAME",
"useCallCache": $USE_CALL_CACHE_BOOL,
"deleteIntermediateOutputFiles": false,
"useReferenceDisks": true,
"memoryRetryMultiplier": 1.2,
"workflowFailureMode": "NoNewCalls",
"userComment": "Automated submission",
"ignoreEmptyOutputs": false
}
EOF
echo "Created submission data file: $SUBMISSION_DATA_FILE"
# 1. Submit all jobs first and store their submission IDs
for input_file in "$INPUTS_DIR"/*.json; do
test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
--results_path "$RESULTS_PATH" \
--inputs_json "$input_file" \
--update_truth "$UPDATE_TRUTH_BOOL" \
--commit_hash "$COMMIT_HASH" )
echo "Uploading the test input file: $test_input_file"
python3 scripts/firecloud_api/firecloud_api2.py \
upload_test_inputs \
--workspace-namespace warp-pipelines \
--workspace-name "WARP Tests" \
--pipeline_name "$PIPELINE_NAME" \
--test_input_file "$test_input_file" \
--branch_name "$branch_name" \
--sa-json-b64 "$SA_JSON_B64" \
--user "[email protected]"
attempt=1
while [ $attempt -le $MAX_RETRIES ]; do
SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api2.py submit_job \
--workspace-namespace "warp-pipelines" \
--workspace-name "WARP Tests" \
--sa-json-b64 "$SA_JSON_B64" \
--user "[email protected]" \
--submission_data_file "$SUBMISSION_DATA_FILE")
echo "Submission ID: $SUBMISSION_ID"
if [[ "$SUBMISSION_ID" == *"404"* || -z "$SUBMISSION_ID" ]]; then
echo "Error in submission, retrying in $RETRY_DELAY seconds..."
((attempt++))
if [ $attempt -gt $MAX_RETRIES ]; then
echo "Max retries reached. Exiting..."
exit 1
fi
sleep $RETRY_DELAY
continue
fi
echo "Submission successful. Submission ID: $SUBMISSION_ID"
SUBMISSION_IDS+=("$SUBMISSION_ID")
break
done
done
echo "All jobs have been submitted. Starting to poll for statuses..."
# 2. After all submissions are done, start polling for statuses of all jobs
for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
attempt=1
while [ $attempt -le $MAX_RETRIES ]; do
echo "Polling for Submission ID: $SUBMISSION_ID"
RESPONSE=$(python3 scripts/firecloud_api/firecloud_api2.py poll_job_status \
--submission_id "$SUBMISSION_ID" \
--sa-json-b64 "$SA_JSON_B64" \
--user "[email protected]" \
--workspace-namespace "warp-pipelines" \
--workspace-name "WARP Tests")
if [ -z "$RESPONSE" ]; then
echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
((attempt++))
if [ $attempt -gt $MAX_RETRIES ]; then
echo "Max retries reached. Exiting..."
exit 1
fi
sleep $RETRY_DELAY
continue
fi
WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
WORKFLOW_STATUSES["$SUBMISSION_ID"]="$WORKFLOW_STATUSES_FOR_SUBMISSION"
# retrieve workflow outputs
echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
WORKFLOW_OUTPUT=$(python3 scripts/firecloud_api/firecloud_api2.py get_workflow_outputs \
--user "[email protected]" \
--sa-json-b64 "$SA_JSON_B64" \
--submission_id "$SUBMISSION_ID" \
--workspace-namespace warp-pipelines \
--workspace-name "WARP Tests" \
--workflow_id "$WORKFLOW_ID" \
--pipeline_name "$PIPELINE_NAME")
ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
done
break
done
# Generate final summary tables with hyperlinks for Submission IDs
echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
# Generate the Terra URL for the submission
SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
# Add the Submission ID as a hyperlink
echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
# Add the workflows and statuses for this submission
echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
# Add a blank line for separation
echo "" >> $GITHUB_STEP_SUMMARY
done
done
env:
PIPELINE_NAME: TestIlluminaGenotypingArray
NAMESPACE: warp-pipelines
WORKSPACE: WARP Tests
#- name: Download Commit Hash from GCP
# run: |
# gsutil cp gs://fc-cddd72b5-323c-495c-9557-5057fff0275a/commit_hash.txt ./commit_hash.txt
#- name: Check Commit Hash
# id: check_commit_hash
# run: |
# # Read the commit hash from the downloaded file
# COMMIT_HASH_FROM_WDL=$(cat commit_hash.txt)
#
# # Compare the two commit hashes
# if [ "$COMMIT_HASH_FROM_WDL" != "${{ env.COMMIT_HASH }}" ]; then
# echo "Error: The commit hash from the WDL output does not match the expected commit hash."
# exit 1
# else
# echo "Commit hash match successful: $COMMIT_HASH_FROM_WDL"
# fi
- name: Print Summary on Success
if: success()
run: |
echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
- name: Print Summary on Failure
if: failure()
run: |
echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY