PD2413

pipelines/skylab/multiome/Multiome.wdl

@@ -10,6 +10,9 @@ workflow Multiome {
 
     input {
         String input_id
+
+        # Monitoring script
+        File monitoring_script = "gs://fc-51792410-8543-49ba-ad3f-9e274900879f/cromwell_monitoring_script2.sh"
 
         # Optimus Inputs
         String counting_mode = "sn_rna"
@@ -81,7 +84,8 @@ workflow Multiome {
             chrom_sizes = chrom_sizes,
             whitelist = atac_whitelist,
             adapter_seq_read1 = adapter_seq_read1,
-            adapter_seq_read3 = adapter_seq_read3
+            adapter_seq_read3 = adapter_seq_read3,
+            monitoring_script = monitoring_script
     }
     call H5adUtils.JoinMultiomeBarcodes as JoinBarcodes {
         input:

pipelines/skylab/multiome/atac.wdl

@@ -32,6 +32,9 @@ workflow ATAC {
     # TrimAdapters input
     String adapter_seq_read1 = "GTCTCGTGGGCTCGGAGATGTGTATAAGAGACAG"
     String adapter_seq_read3 = "TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG"
+
+    # Monitoring script
+    File monitoring_script
   }
 
   String pipeline_version = "1.1.1"
@@ -50,49 +53,50 @@ workflow ATAC {
       read3_fastq = read3_fastq_gzipped,
       barcodes_fastq = read2_fastq_gzipped,
       output_base_name = input_id,
-      whitelist = whitelist
+      whitelist = whitelist,
+      monitoring_script = monitoring_script
   }
 
-  scatter(idx in range(length(SplitFastq.fastq_R1_output_array))) {
-
-    call TrimAdapters {
-      input:
-        read1_fastq = SplitFastq.fastq_R1_output_array[idx],
-        read3_fastq = SplitFastq.fastq_R3_output_array[idx],
-        output_base_name = input_id + "_" + idx,
-        adapter_seq_read1 = adapter_seq_read1,
-        adapter_seq_read3 = adapter_seq_read3
-    }
-
-    call BWAPairedEndAlignment {
-      input:
-        read1_fastq = TrimAdapters.fastq_trimmed_adapter_output_read1,
-        read3_fastq = TrimAdapters.fastq_trimmed_adapter_output_read3,
-        tar_bwa_reference = tar_bwa_reference,
-        output_base_name = input_id + "_" + idx
-    }
-  }
-
-  call Merge.MergeSortBamFiles as MergeBam {
-    input:
-      output_bam_filename = input_id + ".bam",
-      bam_inputs = BWAPairedEndAlignment.bam_aligned_output,
-      sort_order = "coordinate"
-  }
-
-
-  call CreateFragmentFile {
-    input:
-      bam = MergeBam.output_bam,
-      chrom_sizes = chrom_sizes,
-      annotations_gtf = annotations_gtf
-  }
-
-  output {
-    File bam_aligned_output = MergeBam.output_bam
-    File fragment_file = CreateFragmentFile.fragment_file
-    File snap_metrics = CreateFragmentFile.Snap_metrics
-  }
+  # scatter(idx in range(length(SplitFastq.fastq_R1_output_array))) {
+
+  #   call TrimAdapters {
+  #     input:
+  #       read1_fastq = SplitFastq.fastq_R1_output_array[idx],
+  #       read3_fastq = SplitFastq.fastq_R3_output_array[idx],
+  #       output_base_name = input_id + "_" + idx,
+  #       adapter_seq_read1 = adapter_seq_read1,
+  #       adapter_seq_read3 = adapter_seq_read3
+  #   }
+
+  #   call BWAPairedEndAlignment {
+  #     input:
+  #       read1_fastq = TrimAdapters.fastq_trimmed_adapter_output_read1,
+  #       read3_fastq = TrimAdapters.fastq_trimmed_adapter_output_read3,
+  #       tar_bwa_reference = tar_bwa_reference,
+  #       output_base_name = input_id + "_" + idx
+  #   }
+  # }
+
+  # call Merge.MergeSortBamFiles as MergeBam {
+  #   input:
+  #     output_bam_filename = input_id + ".bam",
+  #     bam_inputs = BWAPairedEndAlignment.bam_aligned_output,
+  #     sort_order = "coordinate"
+  # }
+
+
+  # call CreateFragmentFile {
+  #   input:
+  #     bam = MergeBam.output_bam,
+  #     chrom_sizes = chrom_sizes,
+  #     annotations_gtf = annotations_gtf
+  # }
+
+  # output {
+  #   File bam_aligned_output = MergeBam.output_bam
+  #   File fragment_file = CreateFragmentFile.fragment_file
+  #   File snap_metrics = CreateFragmentFile.Snap_metrics
+  # }
 }
 
 # trim read 1 and read 2 adapter sequeunce with cutadapt

tasks/skylab/FastqProcessing.wdl

@@ -249,12 +249,14 @@ task FastqProcessATAC {
         String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.7-1695393479"
 
         # Runtime attributes [?]
-        Int mem_size = 5
+        Int mem_size = 20
         Int cpu = 16
         # TODO decided cpu
         # estimate that bam is approximately equal in size to fastq, add 20% buffer
         Int disk_size = ceil(2 * ( size(read1_fastq, "GiB") + size(read3_fastq, "GiB") + size(barcodes_fastq, "GiB") )) + 400
         Int preemptible = 3
+
+        File monitoring_script
     }
 
     meta {
@@ -277,8 +279,14 @@ task FastqProcessATAC {
     }
 
     command <<<
+        set -euo pipefail
 
-        set -e
+        if [ ! -z "~{monitoring_script}" ]; then
+        chmod a+x ~{monitoring_script}
+        ~{monitoring_script} > monitoring.log &
+        else
+        echo "No monitoring script given as input" > monitoring.log &
+        fi
 
         declare -a FASTQ1_ARRAY=(~{sep=' ' read1_fastq})
         declare -a FASTQ2_ARRAY=(~{sep=' ' barcodes_fastq})
@@ -355,15 +363,16 @@ task FastqProcessATAC {
 
     runtime {
         docker: docker
-        cpu: cpu
-        memory: "${mem_size} MiB"
-        disks: "local-disk ${disk_size} HDD"
+        cpuPlatform: "Intel Cascade Lake"
+        memory: "${mem_size} GiB"
+        disks: "local-disk ${disk_size} SSD"
         preemptible: preemptible
     }
 
     output {
         Array[File] fastq_R1_output_array = glob("/cromwell_root/output_fastq/fastq_R1_*")
         Array[File] fastq_R3_output_array = glob("/cromwell_root/output_fastq/fastq_R3_*")
+        File monitoring_log = "monitoring.log"
     }
 }