Trying to parse the monitorSubmission properly. #46
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Illumina Genotyping Array | |
| # Controls when the workflow will run | |
| on: | |
| #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING | |
| push: | |
| branches: | |
| - kp_GHA_Terra_auth_PD-2682 | |
| #pull_request: | |
| # branches: [ "develop" ] | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| # inputs: | |
| # image_tag: | |
| # description: 'Docker Image Tag (default: branch_name)' | |
| env: | |
| PROJECT_NAME: WARP | |
| # Github repo name | |
| REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| # Set python buffer to avoid buffering issues | |
| PYTHONUNBUFFERED: 1 | |
| jobs: | |
| run_pipeline: | |
| runs-on: ubuntu-latest | |
| # Add "id-token" with the intended permissions. | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| # actions/checkout MUST come before auth | |
| - uses: 'actions/checkout@v3' | |
| - id: 'auth' | |
| name: 'Authenticate to Google Cloud' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| token_format: 'access_token' | |
| # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting | |
| workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider' | |
| # This is our tester service account | |
| service_account: '[email protected]' | |
| access_token_lifetime: '1000s' # optional, default: '3600s' (1 hour) | |
| access_token_scopes: 'profile, email, openid' | |
| # ... further steps are automatically authenticated | |
| - name: Check working directory | |
| run: | | |
| echo "Current directory:" | |
| pwd | |
| ls -lht | |
| - name: Submit job, poll status, and get outputs | |
| run: | | |
| # Set these environment variables or replace with your actual values | |
| TOKEN="${{ steps.auth.outputs.access_token }}" | |
| NAMESPACE="warp-pipelines" | |
| WORKSPACE="WARP Tests" | |
| # Function to call the Firecloud API using the firecloud_api.py script | |
| firecloud_action() { | |
| python3 -u scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}" | |
| } | |
| # Create the submission_data.json file | |
| SUBMISSION_DATA_FILE="submission_data.json" | |
| # Use a heredoc to generate the JSON file content dynamically | |
| cat <<EOF > "$SUBMISSION_DATA_FILE" | |
| { | |
| "methodConfigurationNamespace": "warp-pipelines", | |
| "methodConfigurationName": "IlluminaGenotypingArray", | |
| "useCallCache": true, | |
| "deleteIntermediateOutputFiles": true, | |
| "useReferenceDisks": true, | |
| "memoryRetryMultiplier": 1.2, | |
| "workflowFailureMode": "NoNewCalls", | |
| "userComment": "Automated submission", | |
| "ignoreEmptyOutputs": false | |
| } | |
| EOF | |
| echo "Created submission data file: $SUBMISSION_DATA_FILE" | |
| # 1. Submit a new workflow using the generated submission_data.json | |
| SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE") | |
| # Check if submission was successful | |
| if [ -z "$SUBMISSION_ID" ]; then | |
| echo "Submission failed." | |
| exit 1 | |
| fi | |
| echo "Submission ID: $SUBMISSION_ID" | |
| # 2. Poll submission status and capture all Workflow IDs from the response | |
| RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") | |
| #print the response | |
| echo "Response: $RESPONSE" | |
| #Parse the response to get the workflow IDs | |
| WORKFLOW_IDS=$(echo "$RESPONSE" | grep -oP 'workflow_id:\s*\K\S+') | |
| #WORKFLOW_IDS=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID" | grep -oP 'workflow_id:\s*\K\S+') | |
| # Check if Workflow IDs were retrieved | |
| if [ -z "$WORKFLOW_IDS" ]; then | |
| echo "Failed to retrieve Workflow IDs." | |
| exit 1 | |
| fi | |
| echo "Workflow IDs: $WORKFLOW_IDS" | |
| # 3. Iterate over all Workflow IDs to get outputs | |
| PIPELINE_NAME="IlluminaGenotypingArray" # Replace with your actual Pipeline name | |
| for WORKFLOW_ID in $WORKFLOW_IDS; do | |
| firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME" | |
| done | |
| echo "Workflow outputs retrieved successfully." | |
| # - name: Check the status of a Terra submission | |
| # run: | | |
| # submissionId="44d890f7-f6dd-4a2c-8841-a10faeab3a07" | |
| # namespace="warp-pipelines" | |
| # name="Illumina-Genotyping-Array_np_copy" | |
| # # Assign the access token to a variable | |
| # ACCESS_TOKEN="${{ steps.auth.outputs.access_token }}" | |
| # printf "\nFetching status for submission ID '%s':" "${submissionId}" | |
| # submissionDetails=$(curl \ | |
| # -X GET \ | |
| # --header 'Accept: application/json' \ | |
| # --header "Authorization: Bearer ${ACCESS_TOKEN}" \ | |
| # "https://api.firecloud.org/api/workspaces/$namespace/$name/submissions/$submissionId") | |
| # printf "\nFull JSON Response '%s':" "${submissionDetails}" | |
| # submissionStatus=$(jq -r '.status' <<< "${submissionDetails}") | |
| # workflowsStatus=$(jq -r '.workflows[] | .status' <<< "${submissionDetails}") | |
| # printf "\nSubmissionStatus '%s':" "${submissionStatus}" | |
| # printf "\nWorkflowsStatus '%s':" "${workflowsStatus}" | |
| # submissionList=$(curl -X 'GET' \ | |
| # "https://api.firecloud.org/api/workspaces/$namespace/$name/submissions" \ | |
| # -H 'accept: */*' \ | |
| # -H "Authorization: Bearer ${ACCESS_TOKEN}") | |
| # printf "\nSubmissionList '%s':" "${submissionList}" | |