Merge pull request #1422 from broadinstitute/staging

staging --> master
broadinstitute · Nov 15, 2024 · ef0a523 · ef0a523
2 parents 599f119 + 7788894
commit ef0a523
Show file tree

Hide file tree

Showing 24 changed files with 271 additions and 57 deletions.
diff --git a/deprecated/pipelines/cemba/cemba_methylcseq/README.md b/deprecated/pipelines/cemba/cemba_methylcseq/README.md
@@ -1,6 +1,8 @@
 ## Announcement: CEMBA is Deprecated 9/12/2024
 
-The CEMBA workflow is deprecated and is no longer supported. However, the CEMBA documentation is still available. See [CEMBA Pipeline Overview](https://broadinstitute.github.io/warp/docs/Pipelines/CEMBA_MethylC_Seq_Pipeline/README) on the [WARP documentation site](https://broadinstitute.github.io/warp/)!
+The CEMBA workflow is deprecated and no longer supported. File paths in the JSON file are also no longer supported.
+
+However, the CEMBA documentation is still available. See [CEMBA Pipeline Overview](https://broadinstitute.github.io/warp/docs/Pipelines/CEMBA_MethylC_Seq_Pipeline/README) on the [WARP documentation site](https://broadinstitute.github.io/warp/)! The CEMBA data is also available on the NEMO data portal. The whitelist includes the following barcodes: CTCACG, CAGATC, CGATGT, ACTTGA, TTAGGC, GATCAG, TGACCA, TAGCTT, ACAGTG, GGCTAC, GCCAAT, CTTGTA.
 
 ### CEMBA summary
 

diff --git a/pipeline_versions.txt b/pipeline_versions.txt
@@ -30,11 +30,11 @@ ExomeReprocessing	 3.3.3	2024-11-04
 BuildIndices	 3.0.0	2023-12-06 
 scATAC	 1.3.2	2023-08-03 
 snm3C	 4.0.4	2024-08-06 
-Multiome	 5.9.0	2024-10-21 
-PairedTag	 1.8.1	2024-11-04 
+Multiome	 5.9.1	2024-11-12 
+PairedTag	 1.8.2	2024-11-12 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
-MultiSampleSmartSeq2SingleNucleus	 2.0.3	2024-11-04 
-Optimus	 7.8.1	2024-11-04 
-atac	 2.5.0	2024-10-23 
+MultiSampleSmartSeq2SingleNucleus	 2.0.4	2024-11-12 
+Optimus	 7.8.2	2024-11-12 
+atac	 2.5.2	2024-11-12 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
-SlideSeq	 3.4.4	2024-11-04 
+SlideSeq	 3.4.5	2024-11-12 
diff --git a/pipelines/skylab/atac/atac.changelog.md b/pipelines/skylab/atac/atac.changelog.md
@@ -1,7 +1,18 @@
+# 2.5.2
+2024-11-12 (Date of Last Commit)
+
+* Added memory and disk updates to Multiome JoinBarcodes; this does not impact the ATAC workflow
+
+# 2.5.1
+2024-11-12 (Date of Last Commit)
+
+* Renamed the ATAC workflow library metric percent_target to atac_percent_target for compatibility with downstream tools
+
 # 2.5.0
 2024-10-23 (Date of Last Commit)
 
 * Updated the tabix flag in CreateFragmentFile task to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp
+* Renamed the ATAC workflow library metric percent_target to atac_percent_target for compatibility with downstream tools
 
 # 2.4.0
 2024-10-23 (Date of Last Commit)

diff --git a/pipelines/skylab/atac/atac.wdl b/pipelines/skylab/atac/atac.wdl
@@ -49,7 +49,7 @@ workflow ATAC {
     String adapter_seq_read3 = "TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG"
   }
 
-  String pipeline_version = "2.5.0"
+  String pipeline_version = "2.5.2"
 
   # Determine docker prefix based on cloud provider
   String gcr_docker_prefix = "us.gcr.io/broad-gotc-prod/"
@@ -575,7 +575,7 @@ task CreateFragmentFile {
     print("Print number of cells", number_of_cells)
     atac_percent_target = number_of_cells / expected_cells*100
     print("Setting percent target in nested dictionary")
-    data['Cells']['percent_target'] = atac_percent_target
+    data['Cells']['atac_percent_target'] = atac_percent_target
     
     
     # Flatten the dictionary

diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md
@@ -1,7 +1,16 @@
+# 5.9.1
+2024-11-12 (Date of Last Commit)
+
+* Renamed the ATAC workflow library metric percent_target to atac_percent_target for compatibility with downstream tools
+* Added more disk and memory to the JoinBarcodes task
+
 # 5.9.0
 2024-10-21 (Date of Last Commit)
+
 * Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task changes the format for the ATAC fragment file index 
 * Renamed the fragment file index from atac_fragment_tsv_tbi to atac_fragment_tsv_index
+
+
 # 5.8.0
 2024-10-23 (Date of Last Commit)
 

diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl
@@ -9,7 +9,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Multiome {
 
-    String pipeline_version = "5.9.0"
+    String pipeline_version = "5.9.1"
 
     input {
         String cloud_provider

diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md
@@ -1,8 +1,14 @@
+# 7.8.2
+2024-11-12 (Date of Last Commit)
+
+* Added memory and disk updates to Multiome JoinBarcodes; this does not impact the Optimus workflow
+
 # 7.8.1
 2024-11-04 (Date of Last Commit)
 
 * Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task should not affect the Optimus pipeline
 
+
 # 7.8.0
 2024-10-23 (Date of Last Commit)
 

diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl
@@ -71,7 +71,7 @@ workflow Optimus {
   # version of this pipeline
 
 
-  String pipeline_version = "7.8.1"
+  String pipeline_version = "7.8.2"
 
 
   # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays

diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md
@@ -1,3 +1,9 @@
+# 1.8.2
+2024-11-12 (Date of Last Commit)
+
+* Renamed the ATAC workflow library metric percent_target to atac_percent_target for compatibility with downstream tools
+* Added more disk and memory to the ParseBarcodes task
+
 # 1.8.1
 2024-11-04 (Date of Last Commit)
 

diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl
@@ -8,7 +8,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow PairedTag {
 
-    String pipeline_version = "1.8.1"
+    String pipeline_version = "1.8.2"
 
 
     input {

diff --git a/pipelines/skylab/slideseq/SlideSeq.changelog.md b/pipelines/skylab/slideseq/SlideSeq.changelog.md
@@ -1,8 +1,14 @@
+# 3.4.5
+2024-11-12 (Date of Last Commit)
+
+* Added memory and disk updates to Multiome JoinBarcodes; this does not impact the SlideSeq workflow
+
 # 3.4.4
 2024-11-04 (Date of Last Commit)
 
 * Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task should not affect the Slide-seq pipeline
 
+
 # 3.4.3
 2024-10-24 (Date of Last Commit)
 

diff --git a/pipelines/skylab/slideseq/SlideSeq.wdl b/pipelines/skylab/slideseq/SlideSeq.wdl
@@ -25,7 +25,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow SlideSeq {
 
-    String pipeline_version = "3.4.4"
+    String pipeline_version = "3.4.5"
 
     input {
         Array[File] r1_fastq

diff --git a/...tseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md b/...tseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md
@@ -1,8 +1,14 @@
+# 2.0.4
+2024-11-12 (Date of Last Commit)
+
+* Added memory and disk updates to Multiome JoinBarcodes; this does not impact the snSS2 workflow
+
 # 2.0.3
 2024-11-04 (Date of Last Commit)
 
 * Updated the tabix flag in JoinMultiomeBarcodes task in H5adUtils.wdl to use CSI instead of TBI indexing, which supports chromosomes larger than 512 Mbp; this task should not affect the snSS2 pipeline
 
+
 # 2.0.2
 2024-10-23 (Date of Last Commit)
 

diff --git a/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl b/pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl
@@ -57,7 +57,7 @@ workflow MultiSampleSmartSeq2SingleNucleus {
   }
 
   # Version of this pipeline
-  String pipeline_version = "2.0.3"
+  String pipeline_version = "2.0.4"
 
   if (false) {
      String? none = "None"

diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl
@@ -235,8 +235,8 @@ task JoinMultiomeBarcodes {
 
     Int nthreads = 1
     String cpuPlatform = "Intel Cascade Lake"
-    Int machine_mem_mb = ceil((size(atac_h5ad, "MiB") + size(gex_h5ad, "MiB") + size(atac_fragment, "MiB")) * 6) + 10000
-    Int disk =  ceil((size(atac_h5ad, "GiB") + size(gex_h5ad, "GiB") + size(atac_fragment, "GiB")) * 8) + 10
+    Int machine_mem_mb = ceil((size(atac_h5ad, "MiB") + size(gex_h5ad, "MiB") + size(atac_fragment, "MiB")) * 8) + 10000
+    Int disk =  ceil((size(atac_h5ad, "GiB") + size(gex_h5ad, "GiB") + size(atac_fragment, "GiB")) * 8) + 100
     String docker_path
   }
   String gex_base_name = basename(gex_h5ad, ".h5ad")

diff --git a/tasks/skylab/PairedTagUtils.wdl b/tasks/skylab/PairedTagUtils.wdl
@@ -205,8 +205,8 @@ task ParseBarcodes {
         Int nthreads = 1
         String cpuPlatform = "Intel Cascade Lake"
         String docker_path
-        Int disk =  ceil((size(atac_h5ad, "GiB") + size(atac_fragment, "GiB")) * 8) + 10
-        Int machine_mem_mb = ceil((size(atac_h5ad, "MiB") + size(atac_fragment, "MiB")) * 6) + 10000
+        Int disk =  ceil((size(atac_h5ad, "GiB") + size(atac_fragment, "GiB")) * 8) + 100
+        Int machine_mem_mb = ceil((size(atac_h5ad, "MiB") + size(atac_fragment, "MiB")) * 8) + 10000
     }
 
     String atac_base_name = basename(atac_h5ad, ".h5ad")

diff --git a/verification/VerifyMultiome.wdl b/verification/VerifyMultiome.wdl
@@ -80,7 +80,7 @@ workflow VerifyMultiome {
             test_text_file = test_library_metrics,
             truth_text_file = truth_library_metrics
     }
-    call VerifyTasks.CompareTextFiles as CompareAtacLibraryMetrics {
+    call VerifyTasks.CompareAtacLibraryMetrics as CompareAtacLibraryMetrics {
         input:
             test_text_files = select_all([test_atac_library_metrics]),
             truth_text_files = select_all([truth_atac_library_metrics])

diff --git a/verification/VerifyPairedTag.wdl b/verification/VerifyPairedTag.wdl
@@ -29,6 +29,9 @@ workflow VerifyPairedTag {
         File test_library_metrics
         File truth_library_metrics
 
+        File test_atac_library_metrics
+        File truth_atac_library_metrics
+
         Boolean? done
     }
 
@@ -77,4 +80,10 @@ workflow VerifyPairedTag {
             test_text_file = test_library_metrics,
             truth_text_file = truth_library_metrics
     }
+
+    call VerifyTasks.CompareAtacLibraryMetrics as CompareAtacLibraryMetrics {
+        input:
+        test_text_files = select_all([test_atac_library_metrics]),
+        truth_text_files = select_all([truth_atac_library_metrics])
+    }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -8,7 +8,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils @@
     workflow PairedTag {
-        String pipeline_version = "1.8.1"
+        String pipeline_version = "1.8.2"
         input {
@@ Expand Down @@