add cellbender to optimus (#1446)

* add cellbender to optimus * changelogs * Updated pipeline_versions.txt with all pipeline version information * changelogs * try cellbender is true * try cellbender is true * extra slash --------- Co-authored-by: GitHub Action <[email protected]>
broadinstitute · Dec 11, 2024 · 2fffa87 · 2fffa87
1 parent 145b6bf
commit 2fffa87
Show file tree

Hide file tree

Showing 12 changed files with 103 additions and 59 deletions.
diff --git a/pipeline_versions.txt b/pipeline_versions.txt
@@ -30,11 +30,11 @@ ExomeReprocessing	 3.3.3	2024-11-04
 BuildIndices	 3.1.0	2024-11-26 
 scATAC	 1.3.2	2023-08-03 
 snm3C	 4.0.4	2024-08-06 
-Multiome	 5.9.3	2024-12-3 
-PairedTag	 1.8.4	2024-12-3 
+Multiome	 5.9.4	2024-12-05 
+PairedTag	 1.9.0	2024-12-05 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
 MultiSampleSmartSeq2SingleNucleus	 2.0.6	2024-11-15 
-Optimus	 7.8.4	2024-12-3 
+Optimus	 7.9.0	2024-12-05 
 atac	 2.5.3	2024-11-22 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
 SlideSeq	 3.4.7	2024-12-3 
diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md
@@ -1,3 +1,8 @@
+# 5.9.4
+2024-12-05 (Date of Last Commit)
+
+* Moved the optional CellBender task to the Optimus.wdl
+
 # 5.9.3
 2024-12-3 (Date of Last Commit)
 

diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl
@@ -3,13 +3,11 @@ version 1.0
 import "../../../pipelines/skylab/atac/atac.wdl" as atac
 import "../../../pipelines/skylab/optimus/Optimus.wdl" as optimus
 import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils
-import "https://raw.githubusercontent.com/aawdeh/CellBender/aa-cbwithoutcuda/wdl/cellbender_remove_background_azure.wdl" as CellBender_no_cuda
-import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender
 import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Multiome {
 
-    String pipeline_version = "5.9.3"
+    String pipeline_version = "5.9.4"
 
     input {
         String cloud_provider
@@ -103,7 +101,8 @@ workflow Multiome {
             count_exons = count_exons,
             soloMultiMappers = soloMultiMappers,
             cloud_provider = cloud_provider,
-            gex_expected_cells = expected_cells
+            gex_expected_cells = expected_cells,
+            run_cellbender = run_cellbender
     }
 
     # Call the ATAC workflow
@@ -134,39 +133,6 @@ workflow Multiome {
             atac_fragment = Atac.fragment_file
     }
 
-    # Call CellBender
-    if (run_cellbender) {
-        if (cloud_provider == "gcp") {
-            call CellBender.run_cellbender_remove_background_gpu as CellBender {
-                input:
-                    sample_name = input_id,
-                    input_file_unfiltered = Optimus.h5ad_output_file,
-                    hardware_boot_disk_size_GB = 20,
-                    hardware_cpu_count = 4,
-                    hardware_disk_size_GB = 50,
-                    hardware_gpu_type = "nvidia-tesla-t4",
-                    hardware_memory_GB = 32,
-                    hardware_preemptible_tries = 2,
-                    hardware_zones = "us-central1-a us-central1-c",
-                    nvidia_driver_version = "470.82.01"
-            }
-        } 
-        if (cloud_provider == "azure") {
-            call CellBender_no_cuda.run_cellbender_remove_background_gpu as CellBender_no_cuda {
-                input:
-                    sample_name = input_id,
-                    input_file_unfiltered = Optimus.h5ad_output_file,
-                    hardware_boot_disk_size_GB = 20,
-                    hardware_cpu_count = 4,
-                    hardware_disk_size_GB = 50,
-                    hardware_gpu_type = "nvidia-tesla-t4",
-                    hardware_memory_GB = 32,
-                    hardware_preemptible_tries = 2,
-                    hardware_zones = "us-central1-a us-central1-c",
-                    nvidia_driver_version = "470.82.01"
-            }
-        }           
-    }
 
     meta {
         allowNestedInputs: true
@@ -201,15 +167,14 @@ workflow Multiome {
         File? gex_aligner_metrics = Optimus.aligner_metrics
         File? library_metrics = Optimus.library_metrics
         File? mtx_files = Optimus.mtx_files
+        File? cell_barcodes_csv = Optimus.cell_barcodes_csv
+        File? checkpoint_file = Optimus.checkpoint_file
+        Array[File]? h5_array = Optimus.h5_array
+        Array[File]? html_report_array = Optimus.html_report_array
+        File? log = Optimus.log
+        Array[File]? metrics_csv_array = Optimus.metrics_csv_array
+        String? output_directory = Optimus.output_directory
+        File? summary_pdf = Optimus.summary_pdf
 
-        # cellbender outputs
-        File? cell_barcodes_csv = CellBender.cell_csv
-        File? checkpoint_file = CellBender.ckpt_file
-        Array[File]? h5_array = CellBender.h5_array
-        Array[File]? html_report_array = CellBender.report_array
-        File? log = CellBender.log
-        Array[File]? metrics_csv_array = CellBender.metrics_array
-        String? output_directory = CellBender.output_dir
-        File? summary_pdf = CellBender.pdf
     }
 }
diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md
@@ -1,3 +1,8 @@
+# 7.9.0
+2024-12-05 (Date of Last Commit)
+
+* Added an optional task to the Optimus.wdl that will run CellBender on the Optimus output h5ad file
+
 # 7.8.4
 2024-12-3 (Date of Last Commit)
 

diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl
@@ -8,6 +8,8 @@ import "../../../tasks/skylab/CheckInputs.wdl" as OptimusInputChecks
 import "../../../tasks/skylab/MergeSortBam.wdl" as Merge
 import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils
 import "../../../tasks/broad/Utilities.wdl" as utils
+import "https://raw.githubusercontent.com/aawdeh/CellBender/aa-cbwithoutcuda/wdl/cellbender_remove_background_azure.wdl" as CellBender_no_cuda
+import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender
 
 workflow Optimus {
   meta {
@@ -38,6 +40,9 @@ workflow Optimus {
     String? soloMultiMappers = "Uniform"
     Int gex_expected_cells = 3000
 
+    # CellBender
+    Boolean run_cellbender = false
+
     # Chemistry options include: 2 or 3
     Int tenx_chemistry_version
     # Whitelist is selected based on the input tenx_chemistry_version
@@ -69,9 +74,7 @@ workflow Optimus {
   }
 
   # version of this pipeline
-
-
-  String pipeline_version = "7.8.4"
+  String pipeline_version = "7.9.0"
 
 
   # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays
@@ -301,6 +304,40 @@ workflow Optimus {
     }
   }
 
+  # Call CellBender
+  if (run_cellbender) {
+    if (cloud_provider == "gcp") {
+      call CellBender.run_cellbender_remove_background_gpu as CellBender {
+        input:
+          sample_name = input_id,
+          input_file_unfiltered = final_h5ad_output,
+          hardware_boot_disk_size_GB = 20,
+          hardware_cpu_count = 4,
+          hardware_disk_size_GB = 50,
+          hardware_gpu_type = "nvidia-tesla-t4",
+          hardware_memory_GB = 32,
+          hardware_preemptible_tries = 2,
+          hardware_zones = "us-central1-a us-central1-c",
+          nvidia_driver_version = "470.82.01"
+      }
+    }
+    if (cloud_provider == "azure") {
+      call CellBender_no_cuda.run_cellbender_remove_background_gpu as CellBender_no_cuda {
+        input:
+          sample_name = input_id,
+          input_file_unfiltered = final_h5ad_output,
+          hardware_boot_disk_size_GB = 20,
+          hardware_cpu_count = 4,
+          hardware_disk_size_GB = 50,
+          hardware_gpu_type = "nvidia-tesla-t4",
+          hardware_memory_GB = 32,
+          hardware_preemptible_tries = 2,
+          hardware_zones = "us-central1-a us-central1-c",
+          nvidia_driver_version = "470.82.01"
+      }
+    }
+  }
+
   File final_h5ad_output = select_first([OptimusH5adGenerationWithExons.h5ad_output, OptimusH5adGeneration.h5ad_output])
   File final_library_metrics = select_first([OptimusH5adGenerationWithExons.library_metrics, OptimusH5adGeneration.library_metrics])
 
@@ -327,5 +364,15 @@ workflow Optimus {
 
     # h5ad
     File h5ad_output_file = final_h5ad_output
+
+    # cellbender outputs
+    File? cell_barcodes_csv = CellBender.cell_csv
+    File? checkpoint_file = CellBender.ckpt_file
+    Array[File]? h5_array = CellBender.h5_array
+    Array[File]? html_report_array = CellBender.report_array
+    File? log = CellBender.log
+    Array[File]? metrics_csv_array = CellBender.metrics_array
+    String? output_directory = CellBender.output_dir
+    File? summary_pdf = CellBender.pdf
   }
 }
diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md
@@ -1,3 +1,8 @@
+# 1.9.0
+2024-12-05 (Date of Last Commit)
+
+* Added an optional task to the Optimus.wdl that will run CellBender on the Optimus output h5ad file
+
 # 1.8.4
 2024-12-3 (Date of Last Commit)
 

diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl
@@ -8,7 +8,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow PairedTag {
 
-    String pipeline_version = "1.8.4"
+    String pipeline_version = "1.9.0"
 
 
     input {
@@ -56,6 +56,9 @@ workflow PairedTag {
 
         # Expected to be either 'gcp' or 'azure'
         String cloud_provider
+
+        # If true, run cellbender
+        Boolean run_cellbender = false
     }
 
     # All docker images that are needed for tasks in this workflow
@@ -98,7 +101,8 @@ workflow PairedTag {
             count_exons = count_exons,
             cloud_provider = cloud_provider,
             soloMultiMappers = soloMultiMappers,
-            gex_nhash_id = gex_nhash_id
+            gex_nhash_id = gex_nhash_id,
+            run_cellbender = run_cellbender
     }
 
     # Call the ATAC workflow
@@ -176,5 +180,13 @@ workflow PairedTag {
         Array[File?] multimappers_Uniform_matrix = Optimus.multimappers_Uniform_matrix
         Array[File?] multimappers_Rescue_matrix = Optimus.multimappers_Rescue_matrix
         Array[File?] multimappers_PropUnique_matrix = Optimus.multimappers_PropUnique_matrix
+        File? cell_barcodes_csv = Optimus.cell_barcodes_csv
+        File? checkpoint_file = Optimus.checkpoint_file
+        Array[File]? h5_array = Optimus.h5_array
+        Array[File]? html_report_array = Optimus.html_report_array
+        File? log = Optimus.log
+        Array[File]? metrics_csv_array = Optimus.metrics_csv_array
+        String? output_directory = Optimus.output_directory
+        File? summary_pdf = Optimus.summary_pdf
     }
 }
diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json
@@ -26,5 +26,6 @@
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",
-  "PairedTag.atac_nhash_id":"example_1234"
+  "PairedTag.atac_nhash_id":"example_1234",
+  "PairedTag.run_cellbender":"false"
 }
diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json
@@ -26,5 +26,6 @@
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",
-  "PairedTag.atac_nhash_id":"example_1234"
+  "PairedTag.atac_nhash_id":"example_1234",
+  "PairedTag.run_cellbender":"false"
 }
diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json
@@ -26,5 +26,6 @@
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.cloud_provider": "gcp",
   "PairedTag.gex_nhash_id":"example_1234",
-  "PairedTag.atac_nhash_id":"example_1234"
+  "PairedTag.atac_nhash_id":"example_1234",
+  "PairedTag.run_cellbender":"false"
 }
diff --git a/pipelines/skylab/paired_tag/test_inputs/Scientific/10k_pbmc.json b/pipelines/skylab/paired_tag/test_inputs/Scientific/10k_pbmc.json
@@ -35,5 +35,6 @@
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.gex_nhash_id":"example_1234",
   "PairedTag.atac_nhash_id":"example_1234",
-  "PairedTag.cloud_provider": "gcp"
+  "PairedTag.cloud_provider": "gcp",
+  "PairedTag.run_cellbender":"false"
 }
diff --git a/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json b/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json
@@ -32,5 +32,6 @@
   "PairedTag.soloMultiMappers":"Uniform",
   "PairedTag.gex_nhash_id":"example_1234",
   "PairedTag.atac_nhash_id":"example_1234",
-  "PairedTag.cloud_provider": "gcp"
+  "PairedTag.cloud_provider": "gcp",
+  "PairedTag.run_cellbender":"false"
 }