diff --git a/pipeline_versions.txt b/pipeline_versions.txt index 5ecffd5530..19b6185405 100644 --- a/pipeline_versions.txt +++ b/pipeline_versions.txt @@ -30,11 +30,11 @@ ExomeReprocessing 3.3.3 2024-11-04 BuildIndices 3.1.0 2024-11-26 scATAC 1.3.2 2023-08-03 snm3C 4.0.4 2024-08-06 -Multiome 5.9.3 2024-12-3 -PairedTag 1.8.4 2024-12-3 +Multiome 5.9.4 2024-12-05 +PairedTag 1.9.0 2024-12-05 MultiSampleSmartSeq2 2.2.22 2024-09-11 MultiSampleSmartSeq2SingleNucleus 2.0.6 2024-11-15 -Optimus 7.8.4 2024-12-3 +Optimus 7.9.0 2024-12-05 atac 2.5.3 2024-11-22 SmartSeq2SingleSample 5.1.21 2024-09-11 SlideSeq 3.4.7 2024-12-3 diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md index e444ccd6fd..7edb86afb6 100644 --- a/pipelines/skylab/multiome/Multiome.changelog.md +++ b/pipelines/skylab/multiome/Multiome.changelog.md @@ -1,3 +1,8 @@ +# 5.9.4 +2024-12-05 (Date of Last Commit) + +* Moved the optional CellBender task to the Optimus.wdl + # 5.9.3 2024-12-3 (Date of Last Commit) diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl index 93b88997a2..ac615983cc 100644 --- a/pipelines/skylab/multiome/Multiome.wdl +++ b/pipelines/skylab/multiome/Multiome.wdl @@ -3,13 +3,11 @@ version 1.0 import "../../../pipelines/skylab/atac/atac.wdl" as atac import "../../../pipelines/skylab/optimus/Optimus.wdl" as optimus import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils -import "https://raw.githubusercontent.com/aawdeh/CellBender/aa-cbwithoutcuda/wdl/cellbender_remove_background_azure.wdl" as CellBender_no_cuda -import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender import "../../../tasks/broad/Utilities.wdl" as utils workflow Multiome { - String pipeline_version = "5.9.3" + String pipeline_version = "5.9.4" input { String cloud_provider @@ -103,7 +101,8 @@ workflow Multiome { count_exons = count_exons, soloMultiMappers = soloMultiMappers, cloud_provider = cloud_provider, - gex_expected_cells = expected_cells + gex_expected_cells = expected_cells, + run_cellbender = run_cellbender } # Call the ATAC workflow @@ -134,39 +133,6 @@ workflow Multiome { atac_fragment = Atac.fragment_file } - # Call CellBender - if (run_cellbender) { - if (cloud_provider == "gcp") { - call CellBender.run_cellbender_remove_background_gpu as CellBender { - input: - sample_name = input_id, - input_file_unfiltered = Optimus.h5ad_output_file, - hardware_boot_disk_size_GB = 20, - hardware_cpu_count = 4, - hardware_disk_size_GB = 50, - hardware_gpu_type = "nvidia-tesla-t4", - hardware_memory_GB = 32, - hardware_preemptible_tries = 2, - hardware_zones = "us-central1-a us-central1-c", - nvidia_driver_version = "470.82.01" - } - } - if (cloud_provider == "azure") { - call CellBender_no_cuda.run_cellbender_remove_background_gpu as CellBender_no_cuda { - input: - sample_name = input_id, - input_file_unfiltered = Optimus.h5ad_output_file, - hardware_boot_disk_size_GB = 20, - hardware_cpu_count = 4, - hardware_disk_size_GB = 50, - hardware_gpu_type = "nvidia-tesla-t4", - hardware_memory_GB = 32, - hardware_preemptible_tries = 2, - hardware_zones = "us-central1-a us-central1-c", - nvidia_driver_version = "470.82.01" - } - } - } meta { allowNestedInputs: true @@ -201,15 +167,14 @@ workflow Multiome { File? gex_aligner_metrics = Optimus.aligner_metrics File? library_metrics = Optimus.library_metrics File? mtx_files = Optimus.mtx_files + File? cell_barcodes_csv = Optimus.cell_barcodes_csv + File? checkpoint_file = Optimus.checkpoint_file + Array[File]? h5_array = Optimus.h5_array + Array[File]? html_report_array = Optimus.html_report_array + File? log = Optimus.log + Array[File]? metrics_csv_array = Optimus.metrics_csv_array + String? output_directory = Optimus.output_directory + File? summary_pdf = Optimus.summary_pdf - # cellbender outputs - File? cell_barcodes_csv = CellBender.cell_csv - File? checkpoint_file = CellBender.ckpt_file - Array[File]? h5_array = CellBender.h5_array - Array[File]? html_report_array = CellBender.report_array - File? log = CellBender.log - Array[File]? metrics_csv_array = CellBender.metrics_array - String? output_directory = CellBender.output_dir - File? summary_pdf = CellBender.pdf } } diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md index 5fa565ef4a..adc98a1ee7 100644 --- a/pipelines/skylab/optimus/Optimus.changelog.md +++ b/pipelines/skylab/optimus/Optimus.changelog.md @@ -1,3 +1,8 @@ +# 7.9.0 +2024-12-05 (Date of Last Commit) + +* Added an optional task to the Optimus.wdl that will run CellBender on the Optimus output h5ad file + # 7.8.4 2024-12-3 (Date of Last Commit) diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl index 55be72f690..37d28381e2 100644 --- a/pipelines/skylab/optimus/Optimus.wdl +++ b/pipelines/skylab/optimus/Optimus.wdl @@ -8,6 +8,8 @@ import "../../../tasks/skylab/CheckInputs.wdl" as OptimusInputChecks import "../../../tasks/skylab/MergeSortBam.wdl" as Merge import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils import "../../../tasks/broad/Utilities.wdl" as utils +import "https://raw.githubusercontent.com/aawdeh/CellBender/aa-cbwithoutcuda/wdl/cellbender_remove_background_azure.wdl" as CellBender_no_cuda +import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender workflow Optimus { meta { @@ -38,6 +40,9 @@ workflow Optimus { String? soloMultiMappers = "Uniform" Int gex_expected_cells = 3000 + # CellBender + Boolean run_cellbender = false + # Chemistry options include: 2 or 3 Int tenx_chemistry_version # Whitelist is selected based on the input tenx_chemistry_version @@ -69,9 +74,7 @@ workflow Optimus { } # version of this pipeline - - - String pipeline_version = "7.8.4" + String pipeline_version = "7.9.0" # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays @@ -301,6 +304,40 @@ workflow Optimus { } } + # Call CellBender + if (run_cellbender) { + if (cloud_provider == "gcp") { + call CellBender.run_cellbender_remove_background_gpu as CellBender { + input: + sample_name = input_id, + input_file_unfiltered = final_h5ad_output, + hardware_boot_disk_size_GB = 20, + hardware_cpu_count = 4, + hardware_disk_size_GB = 50, + hardware_gpu_type = "nvidia-tesla-t4", + hardware_memory_GB = 32, + hardware_preemptible_tries = 2, + hardware_zones = "us-central1-a us-central1-c", + nvidia_driver_version = "470.82.01" + } + } + if (cloud_provider == "azure") { + call CellBender_no_cuda.run_cellbender_remove_background_gpu as CellBender_no_cuda { + input: + sample_name = input_id, + input_file_unfiltered = final_h5ad_output, + hardware_boot_disk_size_GB = 20, + hardware_cpu_count = 4, + hardware_disk_size_GB = 50, + hardware_gpu_type = "nvidia-tesla-t4", + hardware_memory_GB = 32, + hardware_preemptible_tries = 2, + hardware_zones = "us-central1-a us-central1-c", + nvidia_driver_version = "470.82.01" + } + } + } + File final_h5ad_output = select_first([OptimusH5adGenerationWithExons.h5ad_output, OptimusH5adGeneration.h5ad_output]) File final_library_metrics = select_first([OptimusH5adGenerationWithExons.library_metrics, OptimusH5adGeneration.library_metrics]) @@ -327,5 +364,15 @@ workflow Optimus { # h5ad File h5ad_output_file = final_h5ad_output + + # cellbender outputs + File? cell_barcodes_csv = CellBender.cell_csv + File? checkpoint_file = CellBender.ckpt_file + Array[File]? h5_array = CellBender.h5_array + Array[File]? html_report_array = CellBender.report_array + File? log = CellBender.log + Array[File]? metrics_csv_array = CellBender.metrics_array + String? output_directory = CellBender.output_dir + File? summary_pdf = CellBender.pdf } } diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md index 9d37ccc547..dcc91d6fe6 100644 --- a/pipelines/skylab/paired_tag/PairedTag.changelog.md +++ b/pipelines/skylab/paired_tag/PairedTag.changelog.md @@ -1,3 +1,8 @@ +# 1.9.0 +2024-12-05 (Date of Last Commit) + +* Added an optional task to the Optimus.wdl that will run CellBender on the Optimus output h5ad file + # 1.8.4 2024-12-3 (Date of Last Commit) diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl index a15b67a21f..29759736e9 100644 --- a/pipelines/skylab/paired_tag/PairedTag.wdl +++ b/pipelines/skylab/paired_tag/PairedTag.wdl @@ -8,7 +8,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils workflow PairedTag { - String pipeline_version = "1.8.4" + String pipeline_version = "1.9.0" input { @@ -56,6 +56,9 @@ workflow PairedTag { # Expected to be either 'gcp' or 'azure' String cloud_provider + + # If true, run cellbender + Boolean run_cellbender = false } # All docker images that are needed for tasks in this workflow @@ -98,7 +101,8 @@ workflow PairedTag { count_exons = count_exons, cloud_provider = cloud_provider, soloMultiMappers = soloMultiMappers, - gex_nhash_id = gex_nhash_id + gex_nhash_id = gex_nhash_id, + run_cellbender = run_cellbender } # Call the ATAC workflow @@ -176,5 +180,13 @@ workflow PairedTag { Array[File?] multimappers_Uniform_matrix = Optimus.multimappers_Uniform_matrix Array[File?] multimappers_Rescue_matrix = Optimus.multimappers_Rescue_matrix Array[File?] multimappers_PropUnique_matrix = Optimus.multimappers_PropUnique_matrix + File? cell_barcodes_csv = Optimus.cell_barcodes_csv + File? checkpoint_file = Optimus.checkpoint_file + Array[File]? h5_array = Optimus.h5_array + Array[File]? html_report_array = Optimus.html_report_array + File? log = Optimus.log + Array[File]? metrics_csv_array = Optimus.metrics_csv_array + String? output_directory = Optimus.output_directory + File? summary_pdf = Optimus.summary_pdf } } diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json index a1df3f587c..a9a58c7955 100644 --- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json +++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json @@ -26,5 +26,6 @@ "PairedTag.soloMultiMappers":"Uniform", "PairedTag.cloud_provider": "gcp", "PairedTag.gex_nhash_id":"example_1234", - "PairedTag.atac_nhash_id":"example_1234" + "PairedTag.atac_nhash_id":"example_1234", + "PairedTag.run_cellbender":"false" } diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json index fd2ffd1510..418063d6eb 100644 --- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json +++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BC011_BC015_downsampled.json @@ -26,5 +26,6 @@ "PairedTag.soloMultiMappers":"Uniform", "PairedTag.cloud_provider": "gcp", "PairedTag.gex_nhash_id":"example_1234", - "PairedTag.atac_nhash_id":"example_1234" + "PairedTag.atac_nhash_id":"example_1234", + "PairedTag.run_cellbender":"false" } diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json index 1b185c8d47..f682f59a1c 100644 --- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json +++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/BI015_downsampled.json @@ -26,5 +26,6 @@ "PairedTag.soloMultiMappers":"Uniform", "PairedTag.cloud_provider": "gcp", "PairedTag.gex_nhash_id":"example_1234", - "PairedTag.atac_nhash_id":"example_1234" + "PairedTag.atac_nhash_id":"example_1234", + "PairedTag.run_cellbender":"false" } diff --git a/pipelines/skylab/paired_tag/test_inputs/Scientific/10k_pbmc.json b/pipelines/skylab/paired_tag/test_inputs/Scientific/10k_pbmc.json index 47c8ab54bc..d4692bf352 100644 --- a/pipelines/skylab/paired_tag/test_inputs/Scientific/10k_pbmc.json +++ b/pipelines/skylab/paired_tag/test_inputs/Scientific/10k_pbmc.json @@ -35,5 +35,6 @@ "PairedTag.soloMultiMappers":"Uniform", "PairedTag.gex_nhash_id":"example_1234", "PairedTag.atac_nhash_id":"example_1234", - "PairedTag.cloud_provider": "gcp" + "PairedTag.cloud_provider": "gcp", + "PairedTag.run_cellbender":"false" } \ No newline at end of file diff --git a/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json b/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json index b4ffd4d14c..80aa4ee457 100644 --- a/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json +++ b/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json @@ -32,5 +32,6 @@ "PairedTag.soloMultiMappers":"Uniform", "PairedTag.gex_nhash_id":"example_1234", "PairedTag.atac_nhash_id":"example_1234", - "PairedTag.cloud_provider": "gcp" + "PairedTag.cloud_provider": "gcp", + "PairedTag.run_cellbender":"false" } \ No newline at end of file