From 2510aae8e7ef29d14951990a9cd824b114e615d2 Mon Sep 17 00:00:00 2001 From: npetrill Date: Thu, 1 Aug 2024 13:14:52 -0400 Subject: [PATCH 1/3] pin all latest docker version --- .../arrays/multi_sample/MultiSampleArrays.wdl | 2 +- .../cram_to_unmapped_bams/CramToUnmappedBams.wdl | 2 +- pipelines/skylab/atac/atac.wdl | 4 ++-- projects/tasks/AdapterTasks.wdl | 2 +- tasks/broad/IlluminaGenotypingArrayTasks.wdl | 2 +- tasks/broad/InternalArraysTasks.wdl | 6 +++--- tasks/broad/InternalTasks.wdl | 2 +- tasks/skylab/CheckInputs.wdl | 2 +- .../src/main/resources/dummy/dummyWorkflow.wdl | 2 +- verification/VerifyExternalReprocessing.wdl | 2 +- verification/VerifyGermlineSingleSample.wdl | 2 +- verification/VerifyIlluminaGenotypingArray.wdl | 4 ++-- verification/VerifyJointGenotyping.wdl | 2 +- verification/VerifyMetrics.wdl | 2 +- verification/VerifyTasks.wdl | 14 +++++++------- .../VerifyUltimaGenomicsJointGenotyping.wdl | 2 +- .../VerifyUltimaGenomicsWholeGenomeGermline.wdl | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/pipelines/broad/arrays/multi_sample/MultiSampleArrays.wdl b/pipelines/broad/arrays/multi_sample/MultiSampleArrays.wdl index 1633c8cd31..6e5757c101 100644 --- a/pipelines/broad/arrays/multi_sample/MultiSampleArrays.wdl +++ b/pipelines/broad/arrays/multi_sample/MultiSampleArrays.wdl @@ -83,7 +83,7 @@ task SplitFoFnToListFoFn { >>> runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" memory: "1 GiB" } diff --git a/pipelines/broad/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl b/pipelines/broad/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl index d54d9a4cf7..bef7a7893b 100644 --- a/pipelines/broad/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl +++ b/pipelines/broad/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl @@ -222,7 +222,7 @@ task SplitUpOutputMapFile { >>> runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk " + disk_size + " HDD" memory: "~{memory_in_MiB} MiB" } diff --git a/pipelines/skylab/atac/atac.wdl b/pipelines/skylab/atac/atac.wdl index 8526d0cbc1..8b2fd36af7 100644 --- a/pipelines/skylab/atac/atac.wdl +++ b/pipelines/skylab/atac/atac.wdl @@ -173,12 +173,12 @@ task GetNumSplits { Int nthreads Int mem_size String cpu_platform - String docker_image = "ubuntu:latest" + String docker_image = "ubuntu@sha256:2e863c44b718727c860746568e1d54afd13b2fa71b160f5cd9058fc436217b30" String vm_size } parameter_meta { - docker_image: "the ubuntu docker image (default: ubuntu:latest)" + docker_image: "the ubuntu docker image (default: ubuntu@sha256:2e863c44b718727c860746568e1d54afd13b2fa71b160f5cd9058fc436217b30)" nthreads: "Number of threads per node (default: 128)" mem_size: "the size of memory used during alignment" vm_size: "the virtual machine used for the task" diff --git a/projects/tasks/AdapterTasks.wdl b/projects/tasks/AdapterTasks.wdl index 8f57dd2e80..d8916e38bd 100644 --- a/projects/tasks/AdapterTasks.wdl +++ b/projects/tasks/AdapterTasks.wdl @@ -452,7 +452,7 @@ task GetCloudFileCreationDate { input { String file_path - String docker = "gcr.io/google.com/cloudsdktool/cloud-sdk:latest" + String docker = "gcr.io/google.com/cloudsdktool/cloud-sdk:486.0.0" Int cpu = 1 Int memory_mb = 1000 Int disk_size_gb = 1 diff --git a/tasks/broad/IlluminaGenotypingArrayTasks.wdl b/tasks/broad/IlluminaGenotypingArrayTasks.wdl index 2598bed60b..4ca4074fce 100644 --- a/tasks/broad/IlluminaGenotypingArrayTasks.wdl +++ b/tasks/broad/IlluminaGenotypingArrayTasks.wdl @@ -14,7 +14,7 @@ task Md5Sum { >>> runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "3.5 GiB" preemptible: 3 diff --git a/tasks/broad/InternalArraysTasks.wdl b/tasks/broad/InternalArraysTasks.wdl index 76d56fb23d..e9ec3afe15 100644 --- a/tasks/broad/InternalArraysTasks.wdl +++ b/tasks/broad/InternalArraysTasks.wdl @@ -335,7 +335,7 @@ task CreateChipWellBarcodeParamsFile { >>> runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "2 GiB" preemptible: preemptible_tries @@ -445,7 +445,7 @@ task ResolveExtendedIlluminaManifestFile { >>> runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "2 GiB" preemptible: preemptible_tries @@ -480,7 +480,7 @@ task ResolveMinorAlleleFrequencyFile { >>> runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "2 GiB" preemptible: preemptible_tries diff --git a/tasks/broad/InternalTasks.wdl b/tasks/broad/InternalTasks.wdl index 75dc79adb5..b3bef735a6 100644 --- a/tasks/broad/InternalTasks.wdl +++ b/tasks/broad/InternalTasks.wdl @@ -17,7 +17,7 @@ task MakeSafeFilename { >>> runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "1 GiB" preemptible: 3 diff --git a/tasks/skylab/CheckInputs.wdl b/tasks/skylab/CheckInputs.wdl index 57fbcaad1a..526586fedd 100644 --- a/tasks/skylab/CheckInputs.wdl +++ b/tasks/skylab/CheckInputs.wdl @@ -43,7 +43,7 @@ task checkInputArrays { } runtime { - docker: "bashell/alpine-bash:latest" + docker: "bashell/alpine-bash@sha256:965a718a07c700a5204c77e391961edee37477634ce2f9cf652a8e4c2db858ff" cpu: 1 memory: "1 GiB" disks: "local-disk ~{disk} HDD" diff --git a/tests/broad/scala_test/src/main/resources/dummy/dummyWorkflow.wdl b/tests/broad/scala_test/src/main/resources/dummy/dummyWorkflow.wdl index 38a9735689..aed9ae2a9e 100644 --- a/tests/broad/scala_test/src/main/resources/dummy/dummyWorkflow.wdl +++ b/tests/broad/scala_test/src/main/resources/dummy/dummyWorkflow.wdl @@ -33,6 +33,6 @@ task PrintMessageToStdout { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" } } \ No newline at end of file diff --git a/verification/VerifyExternalReprocessing.wdl b/verification/VerifyExternalReprocessing.wdl index 7e6a697d5d..a6b6005b5b 100644 --- a/verification/VerifyExternalReprocessing.wdl +++ b/verification/VerifyExternalReprocessing.wdl @@ -32,7 +32,7 @@ task AssertTrue { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "3.5 GiB" preemptible: 3 diff --git a/verification/VerifyGermlineSingleSample.wdl b/verification/VerifyGermlineSingleSample.wdl index 314f3f51a3..9bea981c37 100755 --- a/verification/VerifyGermlineSingleSample.wdl +++ b/verification/VerifyGermlineSingleSample.wdl @@ -90,7 +90,7 @@ task CompareGvcfs { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 300 HDD" memory: "${memory_mb} MiB" preemptible: 3 diff --git a/verification/VerifyIlluminaGenotypingArray.wdl b/verification/VerifyIlluminaGenotypingArray.wdl index a5422ee5cf..6b3bc77ef7 100644 --- a/verification/VerifyIlluminaGenotypingArray.wdl +++ b/verification/VerifyIlluminaGenotypingArray.wdl @@ -155,7 +155,7 @@ task CompareFiles { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "2 GiB" preemptible: 3 @@ -175,7 +175,7 @@ task CompareMetricFilesAsText { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "2 GiB" preemptible: 3 diff --git a/verification/VerifyJointGenotyping.wdl b/verification/VerifyJointGenotyping.wdl index 8f5c54458b..64311f55ea 100644 --- a/verification/VerifyJointGenotyping.wdl +++ b/verification/VerifyJointGenotyping.wdl @@ -93,7 +93,7 @@ task CompareFingerprints { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "2 GiB" preemptible: 3 diff --git a/verification/VerifyMetrics.wdl b/verification/VerifyMetrics.wdl index 53a638259a..3baa400998 100644 --- a/verification/VerifyMetrics.wdl +++ b/verification/VerifyMetrics.wdl @@ -58,7 +58,7 @@ task CompareTwoNumbers { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "3.5 GiB" preemptible: 3 diff --git a/verification/VerifyTasks.wdl b/verification/VerifyTasks.wdl index 547fac62ee..e6be841384 100644 --- a/verification/VerifyTasks.wdl +++ b/verification/VerifyTasks.wdl @@ -19,7 +19,7 @@ task CompareVcfs { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 70 HDD" memory: "32 GiB" preemptible: 3 @@ -49,7 +49,7 @@ task CompareVcfsAllowingQualityDifferences { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 50 HDD" memory: "3 GiB" preemptible: 3 @@ -182,7 +182,7 @@ task CompareTextFiles { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 100 HDD" memory: "50 GiB" preemptible: 3 @@ -210,7 +210,7 @@ task CompareCrams { cmp -i "$test_offset:$truth_offset" ~{test_cram} ~{truth_cram} } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk " + disk_size_gb + " HDD" memory: "2 GiB" preemptible: 3 @@ -230,7 +230,7 @@ task CompareCrais { cmp <(zcat ~{test_crai} | cut -f1,2,3,5,6) <(zcat ~{truth_crai} | cut -f1,2,3,5,6) } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "2 GiB" preemptible: 3 @@ -320,7 +320,7 @@ task CompareCompressedTextFiles { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk " + disk_size + " HDD" memory: "20 GiB" preemptible: 3 @@ -589,7 +589,7 @@ task CompareSnapTextFiles { >>> runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 50 HDD" memory: "25 GiB" preemptible: 3 diff --git a/verification/VerifyUltimaGenomicsJointGenotyping.wdl b/verification/VerifyUltimaGenomicsJointGenotyping.wdl index 4d2c19a5cc..aa0c6ded02 100644 --- a/verification/VerifyUltimaGenomicsJointGenotyping.wdl +++ b/verification/VerifyUltimaGenomicsJointGenotyping.wdl @@ -79,7 +79,7 @@ task CompareFingerprints { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 10 HDD" memory: "2 GiB" preemptible: 3 diff --git a/verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl b/verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl index c206f04926..bf7cfdfb85 100755 --- a/verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl +++ b/verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl @@ -135,7 +135,7 @@ task CompareGvcfs { } runtime { - docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" + docker: "gcr.io/gcp-runtimes/ubuntu_16_0_4@sha256:025124e2f1cf4d29149958f17270596bffe13fc6acca6252977c572dd5ba01bf" disks: "local-disk 70 HDD" memory: "2 GiB" preemptible: 3 From ef640353858adbd4abae7053299c5b812661b120 Mon Sep 17 00:00:00 2001 From: npetrill Date: Tue, 15 Oct 2024 14:02:05 -0400 Subject: [PATCH 2/3] add test for atac lib metrics --- verification/VerifyMultiome.wdl | 8 ++++++++ verification/test-wdls/TestMultiome.wdl | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/verification/VerifyMultiome.wdl b/verification/VerifyMultiome.wdl index b98de16dcb..4c001dfd53 100644 --- a/verification/VerifyMultiome.wdl +++ b/verification/VerifyMultiome.wdl @@ -29,6 +29,9 @@ workflow VerifyMultiome { File test_library_metrics File truth_library_metrics + File test_atac_library_metrics + File truth_atac_library_metrics + Boolean? done } @@ -77,4 +80,9 @@ workflow VerifyMultiome { test_text_files = select_all([test_library_metrics]), truth_text_files = select_all([truth_library_metrics]) } + call VerifyTasks.CompareTextFiles as CompareAtacLibraryMetrics { + input: + test_text_files = select_all([test_atac_library_metrics]), + truth_text_files = select_all([truth_atac_library_metrics]) + } } \ No newline at end of file diff --git a/verification/test-wdls/TestMultiome.wdl b/verification/test-wdls/TestMultiome.wdl index 3b214d9ea8..ff23b2d3ca 100644 --- a/verification/test-wdls/TestMultiome.wdl +++ b/verification/test-wdls/TestMultiome.wdl @@ -194,6 +194,13 @@ workflow TestMultiome { } } + call Utilities.GetValidationInputs as GetAtacLibraryMetrics { + input: + input_file = Multiome.atac_library_metrics, + results_path = results_path, + truth_path = truth_path + } + call VerifyMultiome.VerifyMultiome as Verify { input: truth_optimus_h5ad = GetOptimusH5ad.truth_file, @@ -212,6 +219,8 @@ workflow TestMultiome { test_atac_h5ad = GetSnapMetrics.results_file, test_library_metrics = select_first([GetLibraryMetrics.results_file, ""]), truth_library_metrics = select_first([GetLibraryMetrics.truth_file, ""]), + test_atac_library_metrics = GetAtacLibraryMetrics.results_file, + truth_atac_library_metrics = GetAtacLibraryMetrics.truth_file, done = CopyToTestResults.done } } From 38f6000c79c44d3295a7653164ce389c19915b8e Mon Sep 17 00:00:00 2001 From: npetrill Date: Tue, 15 Oct 2024 15:58:58 -0400 Subject: [PATCH 3/3] add metrics to truth --- verification/test-wdls/TestMultiome.wdl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/verification/test-wdls/TestMultiome.wdl b/verification/test-wdls/TestMultiome.wdl index ff23b2d3ca..d2fcd0eb9a 100644 --- a/verification/test-wdls/TestMultiome.wdl +++ b/verification/test-wdls/TestMultiome.wdl @@ -114,7 +114,8 @@ workflow TestMultiome { Array[String] pipeline_metrics = flatten([ [ # File outputs Multiome.gene_metrics_gex, - Multiome.cell_metrics_gex + Multiome.cell_metrics_gex, + Multiome.atac_library_metrics ], # File? outputs select_all([Multiome.library_metrics]),