test prototype

.github/workflows/traceability_tests_v2.yml

@@ -0,0 +1,88 @@
+name: traceability tests v2
+
+permissions:
+  contents: read
+  id-token: write
+
+on:
+  workflow_dispatch:
+    inputs:
+      include-expression:
+        description: 'Specify keyword expression for test selection. e.g. (req and test_directory and test_file or test_class and not test_func). Leave empty to run all.'
+        required: false
+        type: string
+  push:
+    branches:
+      - jmares-pytest
+#  schedule:
+#    - cron: '0 5 * * WED'
+#    - cron: '0 7 * * SUN'
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 150
+    concurrency:
+      group: parallel_tests
+      cancel-in-progress: false
+    env:
+      AWS_DEFAULT_REGION: us-east-1
+      ENVIRONMENT: staging
+      TEST_PATH: qe_tests
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: configure AWS ECR credentials
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502  # v4.0.2
+        with:
+          aws-access-key-id: ${{ secrets.VDEV_ECR_DOCKER_PUBLISH_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.VDEV_ECR_DOCKER_PUBLISH_AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{env.AWS_DEFAULT_REGION}}
+
+      - name: run tests
+        working-directory: ${{ env.TEST_PATH }}
+        run: bash run_traceability_tests.sh '${{ inputs.include-expression }}'
+
+      - name: create tests summary
+        if: always()
+        working-directory: ${{ env.TEST_PATH }}
+        run: |
+          docker-compose run medusa python /root/medusa/pytest_code/summary_writer.py
+          cat summary.md >> $GITHUB_STEP_SUMMARY
+
+      - name: archive traceability
+        if: always()
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808  # v4.3.3
+        with:
+          name: traceability_${{ inputs.artifact-archive-suffix }}
+          path: ${{ env.TEST_PATH }}/traceability.csv
+          retention-days: 15
+
+      - name: upload test report
+        if: always()
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808  # v4.3.3
+        with:
+          name: report
+          path: ${{ env.TEST_PATH }}/report.html
+          retention-days: 10
+
+      - name: archive test debug log files
+        if: always()
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808  # v4.3.3
+        with:
+          name: tests debug logs
+          path: ${{ env.TEST_PATH }}/*.debug.log
+          retention-days: 10
+
+#      - name: send notification
+#        if: ${{ always() && github.event_name == 'schedule' }}
+#        uses: dnanexus/qe-tools/.github/actions/slack_notification@3603ef012b1864eabe0fc0ab009fba12f9bd27c5
+#        with:
+#          slack-channel: ${{ secrets.SLACK_ALERT_BIG_STAGING_WEBHOOK_URL }}
+#          # for prod channel use SLACK_ALERT_BIG_PRODUCTION_WEBHOOK_URL
+#          status: ${{ job.status == 'success' && 'PASSED' || 'FAILED' }}
+#          image-url: https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png

qe_tests/.gitignore

@@ -0,0 +1,5 @@
+report.html
+summary_failed_tests.csv
+summary_test_results.csv
+traceability.csv
+unit-report.xml

qe_tests/conftest.py

@@ -0,0 +1,77 @@
+import json
+import logging
+import dxpy
+import os
+import pytest
+import subprocess
+import urllib.request
+
+from filelock import FileLock
+from medusa import biome
+# noinspection PyUnresolvedReferences
+from medusa.pytest_code.traceability import pytest_runtest_makereport
+# noinspection PyUnresolvedReferences
+from medusa.pytest_code.summary_writer import (
+    write_failed_test_summary_to_csv, write_test_result_summary_to_csv, report_test_jobs_price)
+
+
+log = logging.getLogger("regression_tests")
+logging.getLogger("filelock").setLevel(logging.INFO)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_dxpy():
+    default_user = os.environ.get("DEFAULT_USER")
+    token = get_secret(f"{default_user}_token")
+    dxpy.set_api_server_info(host=biome.API_URL, protocol=biome.PROTOCOL)
+    dxpy.set_security_context({"auth_token_type": "Bearer", "auth_token": token})
+    log.info(f"Set dxpy token: ***{dxpy.SECURITY_CONTEXT['auth_token'][-4:]}")
+    subprocess.run(f"dx login --noprojects --staging --token {token}", shell=True)
+
+def get_secret(key: str) -> str:
+    """
+    :param key: eg "test_user_token"
+    :return: secret value as string
+    """
+    secrets = load_secrets()
+    secret = secrets.get(key)
+    if not secret:
+        message = f"Couldn't find secret with key '{key}' in secrets."
+        log.error(message)
+        raise KeyError(message)
+    return secret
+
+def load_secrets(environment_variable_name: str = None) -> dict[str, str|int]:
+    """
+    Load secrets from environment variable with either given name or default "SECRETS"
+    Secrets are expected to be in json format, eg. '{"test_user_token": "xxxx"}'
+    :param environment_variable_name: variable name to load secrets from
+    :return: dictionary with secrets
+    """
+    secrets_name = environment_variable_name or "SECRETS"
+    secrets = json.loads(os.environ.get(secrets_name))
+    if not secrets:
+        message = f"Couldn't load secrets with variable name: {secrets_name}."
+        log.error(message)
+        raise KeyError(message)
+    return secrets
+
+@pytest.fixture(scope="session", autouse=True)
+def download_dxCompiler():
+    lock = FileLock(f"download_dxCompiler.lock")
+    dxCompiler_file_name = "dxCompiler.jar"
+    with lock:
+        if not os.path.exists(dxCompiler_file_name):
+            logging.debug("Downloading dxCompiler")
+            urllib.request.urlretrieve("https://github.com/dnanexus/dxCompiler/releases/download/2.11.9/dxCompiler-2.11.9.jar",
+                                       dxCompiler_file_name)
+            logging.debug("dxCompiler downloaded")
+
+@pytest.fixture(scope="session", autouse=True)
+def install_java():
+    lock = FileLock("install_java.lock")
+    with lock:
+        logging.debug("Installing java")
+        subprocess.check_output(["apt-get", "update"])
+        subprocess.check_output(["apt-get", "install", "-y", "--no-install-recommends", "default-jre-headless"])
+        logging.debug("java installed")

qe_tests/docker-compose.yml

@@ -0,0 +1,13 @@
+services:
+  medusa:
+    image: 263799606133.dkr.ecr.us-east-1.amazonaws.com/medusa:11.131
+    network_mode: host
+    environment:
+      - ENVIRONMENT=stg
+      - DEFAULT_USER=dnanexus_apps_test_robot
+      - SECRETS={"dnanexus_apps_test_robot_token":"${MY_STAGING_TOKEN}"}
+
+
+    volumes:
+      - ".:/qe_tests"
+    working_dir: /qe_tests

qe_tests/pytest.ini

@@ -0,0 +1,19 @@
+[pytest]
+addopts = --junit-xml=unit-report.xml --color=yes -p no:warnings --html=report.html --self-contained-html --tb=short
+junit_logging = system-out
+
+log_cli = true
+log_cli_format = %(asctime)s %(levelname)-8s %(message)s (%(filename)s:L%(lineno)s)
+
+log_level = DEBUG
+log_date_format = %Y%m%d-%H:%M:%S
+log_format = %(asctime)s %(levelname)-8s %(message)s (%(filename)s:L%(lineno)s)
+
+log_file_level = DEBUG
+log_file_date_format = %Y%m%d-%H:%M:%S
+log_file_format = %(asctime)s %(levelname)-8s %(message)s (%(filename)s:L%(lineno)s)
+
+
+markers:
+    traceability: tests covering traceability tags
+    smoke_test: tests running on PR merge to internal-release branch, ie branch for staging code

qe_tests/run_traceability_tests.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker compose run medusa pytest -vvv -m traceability -k "${1}" -n 5

qe_tests/scenarios/advanced.wdl

@@ -0,0 +1,84 @@
+import "library_sys_call.wdl" as lib
+import "library_string.wdl" as lib_string
+
+# Get version of BWA
+task BroadGenomicsDocker {
+  command {
+    /usr/gitc/bwa 2>&1 | \
+    grep -e '^Version' | \
+    sed 's/Version: //'
+  }
+  runtime {
+      docker: "broadinstitute/genomes-in-the-cloud:2.2.5-1485277291"
+      memory: "3 GB"
+      cpu: "1"
+      disks: "local-disk 10 HDD"
+  }
+  output {
+    String ver = read_string(stdout())
+  }
+}
+
+
+task Animals {
+    String s
+    Int num_cores
+    Int disk_size
+    Int? num
+    String? foo
+    String family_i = "Family ${s}"
+
+    command {
+        echo "${s} --K -S --flags --contamination ${default=0 num} --s ${default="foobar" foo}"
+    }
+    runtime {
+       disks: "local-disk ${disk_size} HDD"
+       cpu: num_cores
+       memory: "2 GB"
+    }
+    output {
+        String result = read_string(stdout())
+        String family = family_i
+    }
+}
+
+workflow advanced {
+    String pattern
+    String species
+    File file
+    String p_nowhere="wdl ${pattern}"
+    String p_test = "test"
+    Array[String] patterns = [pattern, p_nowhere, p_test]
+    Int? i
+    File? empty
+    String unmapped_bam_suffix = "bam"
+    Array[String] names = ["Jack.XX", "Gil.XX", "Jane.UU"]
+
+    call BroadGenomicsDocker
+    call Animals as str_animals {
+        input: s= species + "_family", num_cores=3, disk_size=40
+    }
+    scatter (name in names) {
+        call lib_string.Concat as concat {
+          input:
+             x = sub(name, ".XX", "") + ".XY",
+             y = sub(sub(name, ".XX", ""), ".UU", "") + ".unmerged"
+        }
+    }
+    scatter (pt in patterns) {
+        String s = "Salamander"
+        String sub_strip_unmapped = unmapped_bam_suffix + "$"
+        Int k = 5
+
+        call lib.cgrep as cgrep {
+           input: in_file = file, pattern = pt
+        }
+    }
+
+    output {
+        Array[Int] cgrep_count = cgrep.count
+        String str_animals_result = str_animals.result
+        String str_animals_family = str_animals.family
+        String BroadGenomicsDocker_version = BroadGenomicsDocker.ver
+    }
+}

qe_tests/scenarios/advanced_input.json

@@ -0,0 +1,5 @@
+{
+  "advanced.pattern" : "jar",
+  "advanced.file" : "dx://dxCompiler_playground:/test_data/fileB",
+  "advanced.species" : "Arctic fox"
+}

qe_tests/scenarios/advanced_results.json

@@ -0,0 +1,6 @@
+{
+  "cgrep_count": [8, 0, 3],
+  "str_animals_result" : "Arctic fox_family --K -S --flags --contamination 0 --s foobar",
+  "str_animals_family" : "Family Arctic fox_family",
+  "BroadGenomicsDocker_version" : "0.7.15-r1140"
+}

qe_tests/scenarios/call_with_defaults1.wdl

@@ -0,0 +1,23 @@
+workflow call_with_defaults1 {
+    Int x
+    Boolean flag
+
+    if (flag) {
+        call SumWithDefaults { input: x = x }
+    }
+    output {
+        Int? result = SumWithDefaults.result
+    }
+}
+
+task SumWithDefaults {
+    Int x = 2
+    Int? y = 3
+
+  Int z = select_first([y])
+    command {
+    }
+    output {
+        Int result = x + z
+    }
+}

qe_tests/scenarios/call_with_defaults1_input.json

@@ -0,0 +1,4 @@
+{
+  "call_with_defaults1.x" : 1,
+  "call_with_defaults1.flag" : true
+}

qe_tests/scenarios/call_with_defaults1_results.json

@@ -0,0 +1,3 @@
+{
+  "result" : 4
+}

qe_tests/scenarios/check_route.wdl

@@ -0,0 +1,29 @@
+workflow check_route {
+    String prefix
+    Array[String] ranges
+
+    scatter (range in ranges) {
+        call Concat as concat1 {
+            input:
+              x=prefix,
+              y=range
+        }
+    }
+
+    output {
+        Array[String] result = concat1.result
+    }
+}
+
+# Concatenate two string
+task Concat {
+    String x
+    String y
+
+    command {
+        echo ${x}_${y}
+    }
+    output {
+        String result = read_string(stdout())
+    }
+}

qe_tests/scenarios/cwl/cat.cwl.json

@@ -0,0 +1,26 @@
+{
+    "class": "CommandLineTool",
+    "doc": "Write a file to stdout using cat",
+    "baseCommand": "cat",
+    "stdout": "output.txt",
+    "inputs": [
+        {
+            "type": "File",
+            "inputBinding": {
+                "position": 1
+            },
+            "id": "#main/file"
+        }
+    ],
+    "outputs": [
+        {
+            "type": "File",
+            "id": "#main/contents",
+            "outputBinding": {
+                "glob": "output.txt"
+            }
+        }
+    ],
+    "id": "#main",
+    "cwlVersion": "v1.2"
+}