Lk dynamically select barcode orientation (#1086)

* Added dynamic barcode selection to the ATAC FastqProcess task
broadinstitute · Sep 23, 2023 · b7461f1 · b7461f1
1 parent e47ae8e
commit b7461f1
Show file tree

Hide file tree

Showing 12 changed files with 37 additions and 11 deletions.
diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md
@@ -1,3 +1,7 @@
+# 2.1.0
+2023-09-21 (Date of Last Commit)
+* Added dynamic barcode orientation selection to the ATAC workflow FastqProcess task
+
 # 2.0.0
 2023-09-05 (Date of Last Commit)
 

diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl
@@ -4,7 +4,7 @@ import "../../../pipelines/skylab/multiome/atac.wdl" as atac
 import "../../../pipelines/skylab/optimus/Optimus.wdl" as optimus
 
 workflow Multiome {
-    String pipeline_version = "2.0.0"
+    String pipeline_version = "2.1.0"
 
     input {
         String input_id

diff --git a/pipelines/skylab/multiome/atac.changelog.md b/pipelines/skylab/multiome/atac.changelog.md
@@ -1,3 +1,7 @@
+# 1.1.0
+2023-09-21 (Date of Last Commit)
+* Added dynamic barcode selection to the ATAC FastqProcess task
+
 # 1.0.1
 2023-09-05
 

diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl
@@ -34,7 +34,7 @@ workflow ATAC {
     String adapter_seq_read3 = "TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG"
   }
 
-  String pipeline_version = "1.0.1"
+  String pipeline_version = "1.1.0"
 
   parameter_meta {
     read1_fastq_gzipped: "read 1 FASTQ file as input for the pipeline, contains read 1 of paired reads"

diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md
@@ -1,4 +1,7 @@
 
+# 6.1.0
+2023-09-21 (Date of Last Commit)
+* Added dynamic barcode orientation selection to ATAC workflow FastqProcess task; this has no impact on Optimus
 # 6.0.0
 2023-08-22 (Date of Last Commit)
 

diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl
@@ -64,7 +64,7 @@ workflow Optimus {
 
   # version of this pipeline
 
-  String pipeline_version = "6.0.0"
+  String pipeline_version = "6.1.0"
 
   # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays
   Array[Int] indices = range(length(r1_fastq))

diff --git a/pipelines/skylab/slideseq/SlideSeq.changelog.md b/pipelines/skylab/slideseq/SlideSeq.changelog.md
@@ -1,3 +1,7 @@
+# 2.0.1
+2023-09-21 (Date of Last Commit)
+* Added dynamic barcode orientation selection to the ATAC workflow FastqProcess task; this does not impact Slideseq
+
 # 2.0.0
 2023-08-22 (Date of Last Commit)
 

diff --git a/pipelines/skylab/slideseq/SlideSeq.wdl b/pipelines/skylab/slideseq/SlideSeq.wdl
@@ -23,7 +23,7 @@ import "../../../tasks/skylab/MergeSortBam.wdl" as Merge
 
 workflow SlideSeq {
 
-    String pipeline_version = "2.0.0"
+    String pipeline_version = "2.0.1"
 
     input {
         Array[File] r1_fastq

diff --git a/tasks/skylab/FastqProcessing.wdl b/tasks/skylab/FastqProcessing.wdl
@@ -242,10 +242,11 @@ task FastqProcessATAC {
         String barcode_orientation = "FIRST_BP_RC"
         String output_base_name
         File whitelist
+        String barcode_index1 = basename(barcodes_fastq[0])
 
         # [?] copied from corresponding optimus wdl for fastqprocessing
         # using the latest build of warp-tools in GCR
-        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:aa_updatefastqprocess_cbtag"
+        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.7-1695393479"
 
         # Runtime attributes [?]
         Int mem_size = 5
@@ -288,12 +289,17 @@ task FastqProcessATAC {
         read3_fastq_files=`printf '%s ' "${FASTQ3_ARRAY[@]}"; echo`
 
         echo $read1_fastq_files
-        
+        # Make downsample fq for barcode orientation check of R2 barcodes
         mkdir /cromwell_root/input_fastq
         gcloud storage cp $read1_fastq_files /cromwell_root/input_fastq
         gcloud storage cp $read2_fastq_files /cromwell_root/input_fastq
         gcloud storage cp $read3_fastq_files /cromwell_root/input_fastq
 
+        path="/cromwell_root/input_fastq/"
+        barcode_index="~{barcode_index1}"
+        file="${path}${barcode_index}"
+        zcat "$file" | sed -n '2~4p' | shuf -n 1000 > downsample.fq
+        head -n 1 downsample.fq
         # barcodes R2
         R1_FILES_CONCAT=""
         for fastq in "${FASTQ2_ARRAY[@]}"
@@ -324,6 +330,11 @@ task FastqProcessATAC {
         done
         echo $R3_FILES_CONCAT
 
+        python3 /warptools/scripts/dynamic-barcode-orientation.py downsample.fq ~{whitelist} best_match.txt
+        
+        cat best_match.txt
+        barcode_choice=$(<best_match.txt)
+        echo $barcode_choice
         # Call fastq process
         # outputs fastq files where the corrected barcode is in the read name
         mkdir /cromwell_root/output_fastq
@@ -337,7 +348,7 @@ task FastqProcessATAC {
         $R3_FILES_CONCAT \
         --white-list "~{whitelist}" \
         --output-format "FASTQ" \
-        --barcode-orientation "~{barcode_orientation}" \
+        --barcode-orientation $barcode_choice \
         --read-structure "~{read_structure}"
 
     >>>

diff --git a/website/docs/Pipelines/ATAC/README.md b/website/docs/Pipelines/ATAC/README.md
@@ -8,7 +8,7 @@ slug: /Pipelines/ATAC/README
 
 | Pipeline Version | Date Updated | Documentation Author | Questions or Feedback |
 | :----: | :---: | :----: | :--------------: |
-| [1.0.0](https://github.com/broadinstitute/warp/releases) | May, 2023 | Kaylee Mathews | Please file GitHub issues in warp or contact [the WARP team](mailto:[email protected]) |
+| [1.1.0](https://github.com/broadinstitute/warp/releases) | September, 2023 | Kaylee Mathews | Please file GitHub issues in warp or contact [the WARP team](mailto:[email protected]) |
 
 ## Introduction to the ATAC workflow
 ATAC is an open-source, cloud-optimized pipeline developed collaboration with members of the [BRAIN Initiative](https://braininitiative.nih.gov/) (BICCN and [BICAN](https://brainblog.nih.gov/brain-blog/brain-issues-suite-funding-opportunities-advance-brain-cell-atlases-through-centers) Sequencing Working Group) and [SCORCH](https://nida.nih.gov/about-nida/organization/divisions/division-neuroscience-behavior-dnb/basic-research-hiv-substance-use-disorder/scorch-program) (see [Acknowledgements](#acknowledgements) below). It supports the processing of 10x single-nucleus data generated with 10x Multiome [ATAC-seq (Assay for Transposase-Accessible Chromatin)](https://www.10xgenomics.com/products/single-cell-multiome-atac-plus-gene-expression), a technique used in molecular biology to assess genome-wide chromatin accessibility. 
@@ -71,7 +71,7 @@ To see specific tool parameters, select the task WDL link in the table; then vie
 
 | Task name and WDL link | Tool | Software | Description | 
 | --- | --- | --- | ------------------------------------ | 
-| [FastqProcessing as SplitFastq](https://github.com/broadinstitute/warp/blob/develop/tasks/skylab/FastqProcessing.wdl) | fastqprocess | custom | Corrects cell barcodes and splits fastqs into 30 GB FASTQs that are grouped by cell barcode with each read having the corrected (CB) and raw barcode (CR) in the read name. |
+| [FastqProcessing as SplitFastq](https://github.com/broadinstitute/warp/blob/develop/tasks/skylab/FastqProcessing.wdl) | fastqprocess | custom | Dynamically selects the correct barcode orientation, corrects cell barcodes and splits FASTQs into 30 GB FASTQs that are grouped by cell barcode with each read having the corrected (CB) and raw barcode (CR) in the read name. |
 | [TrimAdapters](https://github.com/broadinstitute/warp/blob/develop/pipelines/skylab/multiome/atac.wdl) | Cutadapt v4.4 | cutadapt | Trims adaptor sequences. |
 | [BWAPairedEndAlignment](https://github.com/broadinstitute/warp/blob/develop/pipelines/skylab/multiome/atac.wdl) | bwa-mem2 | mem | Aligns reads from each set of partitioned FASTQ files to the genome and outputs a BAM with ATAC barcodes in the CB:Z tag. |
 | [Merge.MergeSortBamFiles as MergeBam](https://github.com/broadinstitute/warp/blob/develop/tasks/skylab/MergeSortBam.wdl) | MergeSamFiles | Picard | Merges each BAM into a final aligned BAM with corrected cell barcodes in the CB tag. |

diff --git a/website/docs/Pipelines/Multiome_Pipeline/README.md b/website/docs/Pipelines/Multiome_Pipeline/README.md
@@ -7,7 +7,7 @@ slug: /Pipelines/Multiome_Pipeline/README
 
 | Pipeline Version | Date Updated | Documentation Author | Questions or Feedback |
 | :----: | :---: | :----: | :--------------: |
-| [Multiome v2.0.0](https://github.com/broadinstitute/warp/releases) | September, 2023 | Kaylee Mathews | Please file GitHub issues in warp or contact the [WARP Pipeline Development team](mailto:[email protected]) |
+| [Multiome v2.1.0](https://github.com/broadinstitute/warp/releases) | September, 2023 | Kaylee Mathews | Please file GitHub issues in warp or contact the [WARP Pipeline Development team](mailto:[email protected]) |
 
 ![Multiome_diagram](./multiome_diagram.png)
 

diff --git a/website/docs/Pipelines/Optimus_Pipeline/README.md b/website/docs/Pipelines/Optimus_Pipeline/README.md
@@ -7,7 +7,7 @@ slug: /Pipelines/Optimus_Pipeline/README
 
 | Pipeline Version | Date Updated | Documentation Author | Questions or Feedback |
 | :----: | :---: | :----: | :--------------: |
-| [optimus_v5.8.4](https://github.com/broadinstitute/warp/releases?q=optimus&expanded=true) | July, 2023 | Elizabeth Kiernan | Please file GitHub issues in warp or contact [the WARP team](mailto:[email protected]) |
+| [optimus_v6.1.0](https://github.com/broadinstitute/warp/releases?q=optimus&expanded=true) | July, 2023 | Elizabeth Kiernan | Please file GitHub issues in warp or contact [the WARP team](mailto:[email protected]) |
 
 ![Optimus_diagram](Optimus_diagram.png)