Added liftoff, integrated with pfr/nxf-modules, and some minor improvements

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_size = 4
+indent_style = space
+
+[*.{md,yml,yaml,cff}]
+indent_size = 2
+
+[*.nf.test]
+insert_final_newline = false

.gitignore

@@ -1,26 +1,16 @@
-.DS_Store
-
-*.pyc
-__pycahce__
-
-nextflow
 .nextflow*
 work/
-*.dot
-
-Results/
 results/
-report/
-Report/
-
-*.log
-.nfs*
-
-*.sif
+.DS_Store
+*.code-workspace
+.screenrc
+.*.sw?
+__pycache__
+*.pyo
+*.pyc
 
-pan_gene_slurm.sh
 *.stdout
 *.stderr
 
 .literature
-.test
+pangene-test/

.nf-core.yml

@@ -0,0 +1 @@
+repository_type: pipeline

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v3.1.0"
+    hooks:
+      - id: prettier

.prettierignore

@@ -0,0 +1,19 @@
+includes/Maven_Pro/
+
+# gitignore
+.nextflow*
+work/
+results/
+.DS_Store
+*.code-workspace
+.screenrc
+.*.sw?
+__pycache__
+*.pyo
+*.pyc
+
+*.stdout
+*.stderr
+
+.literature
+pangene-test/

.prettierrc.yml

@@ -0,0 +1 @@
+printWidth: 120

README.md

@@ -1,105 +1,95 @@
-# PAN-GENE
+# PANGENE
+
 A NextFlow pipeline for pan-genome annotation.
 
 ## Pipeline Flowchart
 
 ```mermaid
 flowchart TD
-    ribo_db((ribo_db))
-    SAMPLESHEET((samples))
-    TE_LIBRARIES(("[te_libs]"))
-    TARGET_ASSEMBLIES(("[assemblies]"))
-    EXTERNAL_PROTEIN_SEQS(("[ext_prots]"))
-
-    GUNZIP_PROT[GUNZIP]
-    GUNZIP_TE[GUNZIP]
-    SKIP_EDTA{Skip EDTA}
-    pend((dev))
-
-    TE_LIBRARIES --> GUNZIP_TE
-    GUNZIP_TE --> SKIP_EDTA
-
-    TARGET_ASSEMBLIES --> GUNZIP
-    GUNZIP --> FASTA_VALIDATE
-    FASTA_VALIDATE --> FASTA_PERFORM_EDTA
-    FASTA_VALIDATE --> SKIP_EDTA
-
-    SKIP_EDTA --> REPEATMASKER
-    FASTA_PERFORM_EDTA --> REPEATMASKER
-    REPEATMASKER --> STAR_GENOMEGENERATE
-
-    SAMPLESHEET --> SAMPLESHEET_CHECK
-    SAMPLESHEET_CHECK --> |Technical replicates|CAT_FASTQ
-    CAT_FASTQ --> FASTQC
-    SAMPLESHEET_CHECK --> FASTQC
-    FASTQC --> FASTP
-
-    ribo_db --> SORTMERNA
-    FASTP --> SORTMERNA
-    SORTMERNA --> STAR_ALIGN
-    STAR_GENOMEGENERATE --> STAR_ALIGN
-    STAR_ALIGN --> GROUP_BY_ASSEMBLY([Group by assembly])
-    GROUP_BY_ASSEMBLY --> SAMTOOLS_CAT
-    SAMTOOLS_CAT --> |RNASeq bam|BRAKER3
-
-    REPEATMASKER --> BRAKER3
-
-    EXTERNAL_PROTEIN_SEQS --> GUNZIP_PROT
-    GUNZIP_PROT --> CAT
-    CAT --> BRAKER3
-
-    BRAKER3 --> pend
-
-    subgraph Params
+    subgraph PrepareAssembly [ ]
     TARGET_ASSEMBLIES
     TE_LIBRARIES
-    SAMPLESHEET
-    ribo_db
-    EXTERNAL_PROTEIN_SEQS
-    end
-
-    subgraph GenomePrep
-    GUNZIP
     FASTA_VALIDATE
-    GUNZIP_TE
-    FASTA_PERFORM_EDTA
-    SKIP_EDTA
+    fasta_file_from_fasta_validate
+    EDTA
+    REPEATMODELER
+    te_lib_absent_node
     REPEATMASKER
-    STAR_GENOMEGENERATE
-    end
-
-    subgraph Braker
-    CAT
-    GUNZIP_PROT
-    BRAKER3
     end
 
-    subgraph SamplePrep
-    SAMPLESHEET_CHECK
+    TARGET_ASSEMBLIES(["[target_assemblies]"])
+    TE_LIBRARIES(["[te_libs]"])
+    TARGET_ASSEMBLIES --> FASTA_VALIDATE
+    FASTA_VALIDATE --- |Fasta|fasta_file_from_fasta_validate(( ))
+    fasta_file_from_fasta_validate --> |or|EDTA
+    fasta_file_from_fasta_validate --> |default|REPEATMODELER
+    REPEATMODELER --- te_lib_absent_node(( ))
+    EDTA --- te_lib_absent_node
+    TE_LIBRARIES --> REPEATMASKER
+    te_lib_absent_node --> REPEATMASKER
+
+    subgraph Samplesheet [ ]
+    SAMPLESHEET
     CAT_FASTQ
     FASTQC
     FASTP
+    FASTP_FASTQC
     SORTMERNA
-    STAR_ALIGN
-    GROUP_BY_ASSEMBLY
+    fasta_file_for_star
+    STAR
     SAMTOOLS_CAT
     end
 
-    style Params fill:#00FFFF21,stroke:#00FFFF21
-    style GenomePrep fill:#00FFFF21,stroke:#00FFFF21
-    style SamplePrep fill:#00FFFF21,stroke:#00FFFF21
-    style Braker fill:#00FFFF21,stroke:#00FFFF21
+    SAMPLESHEET([samplesheet])
+    SAMPLESHEET --> |Tech. reps|CAT_FASTQ
+    CAT_FASTQ --> FASTQC
+    SAMPLESHEET --> FASTQC
+    FASTQC --> FASTP
+    FASTP --> FASTP_FASTQC[FASTQC]
+    FASTP_FASTQC --> SORTMERNA
+    fasta_file_for_star(( ))
+    fasta_file_for_star --> |Fasta|STAR
+    SORTMERNA --> STAR
+    STAR --> SAMTOOLS_CAT
+
+    subgraph Annotation [ ]
+    anno_fasta(( ))
+    anno_masked_fasta(( ))
+    anno_bam(( ))
+    EXTERNAL_PROTEIN_SEQS(["[ext_prots]"])
+    XREF_ANNOTATIONS(["[xref_annotations]"])
+    CAT
+    BRAKER3
+    GFFREAD
+    LIFTOFF
+    end
+
+    PrepareAssembly --> |Fasta, Masked fasta|Annotation
+    Samplesheet --> |RNASeq bam|Annotation
+
+    XREF_ANNOTATIONS --> |xref_gff|GFFREAD
+    XREF_ANNOTATIONS --> |xref_fasta|LIFTOFF
+    GFFREAD --> LIFTOFF
+    anno_fasta --> |Fasta|LIFTOFF
+
+    EXTERNAL_PROTEIN_SEQS --> CAT
+    anno_masked_fasta --> |Masked fasta|BRAKER3
+    anno_bam --> |RNASeq bam|BRAKER3
+    CAT --> BRAKER3
+
+    style Samplesheet fill:#00FFFF21,stroke:#00FFFF21
+    style PrepareAssembly fill:#00FFFF21,stroke:#00FFFF21
+    style Annotation fill:#00FFFF21,stroke:#00FFFF21
 ```
 
 ## Plant&Food Users
 
 Configure the pipeline by modifying `nextflow.config` and submit to SLURM for execution.
 
 ```bash
-sbatch ./pan_gene_pfr.sh 
+sbatch ./pangene_pfr
 ```
 
-
 ## Third-party Sources
 
 Some software components of this pipeline have been adopted from following third-party sources:
@@ -112,5 +102,6 @@ Some software components of this pipeline have been adopted from following third
 >
 > _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x).
 
-2. rewarewaannotation [MIT](https://github.com/kherronism/rewarewaannotation/blob/master/LICENSE): https://github.com/kherronism/rewarewaannotation
-3. assembly_qc [GPL-3.0](https://github.com/Plant-Food-Research-Open/assembly_qc/blob/main/LICENSE): https://github.com/Plant-Food-Research-Open/assembly_qc
+2. nf-core/rnaseq [MIT](https://github.com/nf-core/rnaseq/blob/master/LICENSE): https://github.com/nf-core/rnaseq
+3. rewarewaannotation [MIT](https://github.com/kherronism/rewarewaannotation/blob/master/LICENSE): https://github.com/kherronism/rewarewaannotation
+4. assembly_qc [GPL-3.0](https://github.com/Plant-Food-Research-Open/assembly_qc/blob/main/LICENSE): https://github.com/Plant-Food-Research-Open/assembly_qc

TODO.md

@@ -1,4 +1,21 @@
-- [ ] Rename perform_edta_annotation to FASTA_PERFORM_EDTA
-- [ ] Extract subworkflows
-- [ ] STAR ignores softmasking and, thus, should be fed the unmasked genome so that masking and mapping can run in parallel.
-- [ ] Add --eval=reference.gtf
+- [ ] Add --eval=reference.gtf
+- [ ] From Ross regarding post-processing:
+
+> [9:49 am] Ross Crowhurst
+> Here is an easy one: BLATSp vs swissprot & Arabidpsis and check query is with set thresholds of reference - if so accept; If not move to BLASTp vs Uniref90 or Refeq (or some other predetermined model species) - same deal accept if within threshold limits. Else BLASTn of cds vs NCBI nt (really scrapping the bottom of the barrel here). If not a hit to anything then chances are its garbage and should be removed. Some ppl might try to claim its a unique protein to the genotype but in 20 years I have never seen one of those be supported - mostly this category is garbage. The screen agains NCBI nt also assists to classify "bits" as well retroposonss etc. Idea being you want to remove garbage predictions - as this does take time you can see why some papers just filter out by size.
+
+- [ ] From Cecilia:
+
+> https://github.com/zhaotao1987/SynNet-Pipeline
+
+- [ ] From Ross:
+
+> https://www.biorxiv.org/content/10.1101/096529v2.full.pdf
+
+- [ ] Sort out EDTA testing
+
+- Mib finder, eggnog, blastp against TAIR and uniprot (Wait)
+- entap to merge (Wait)
+- trinity and PASA + StringTie2 -> Evigene (Do)
+- othrofinder paper
+- gffcompre on braker and liftoff

assets/rrna-db-defaults.txt

@@ -5,4 +5,4 @@ https://raw.githubusercontent.com/biocore/sortmerna/v4.3.4/data/rRNA_databases/s
 https://raw.githubusercontent.com/biocore/sortmerna/v4.3.4/data/rRNA_databases/silva-bac-16s-id90.fasta
 https://raw.githubusercontent.com/biocore/sortmerna/v4.3.4/data/rRNA_databases/silva-bac-23s-id98.fasta
 https://raw.githubusercontent.com/biocore/sortmerna/v4.3.4/data/rRNA_databases/silva-euk-18s-id95.fasta
-https://raw.githubusercontent.com/biocore/sortmerna/v4.3.4/data/rRNA_databases/silva-euk-28s-id98.fasta
+https://raw.githubusercontent.com/biocore/sortmerna/v4.3.4/data/rRNA_databases/silva-euk-28s-id98.fasta

assets/rrna-db-test.txt

@@ -0,0 +1 @@
+https://raw.githubusercontent.com/biocore/sortmerna/v4.3.4/data/rRNA_databases/silva-euk-28s-id98.fasta

bin/make-samplesheet.py

@@ -282,4 +282,4 @@ def main():
         make_samplesheet_from_command(input_path_or_command, exp_name)
 
 if __name__ == "__main__":
-    main()
+    main()

cleanNXF.sh

@@ -8,4 +8,4 @@ for i in $(ls work | grep -v "conda");
 do
     rm -rf "work/$i"
 done
-echo "Cleaned work..."
+echo "Cleaned work..."

conf/base.config

@@ -1,19 +1,33 @@
 profiles {
-    slurm {
+    pfr {
         process {
             executor        = 'slurm'
         }
+
+        apptainer {
+            envWhitelist    = 'APPTAINER_BINDPATH,APPTAINER_BIND'
+        }
     }
-    
+
     local {
         process {
             executor        = 'local'
         }
     }
+
+    apptainer {
+        apptainer.enabled   = true
+        apptainer.autoMounts= true
+        apptainer.registry  = 'quay.io'
+    }
+
+    docker {
+        docker.enabled      = true
+        docker.runOptions   = '-u $(id -u):$(id -g) --platform=linux/amd64'
+        docker.registry     = 'quay.io'
+    }
 }
 
-// Source: https://github.com/nf-core/rnaseq
-// License: https://github.com/nf-core/rnaseq/blob/master/LICENSE
 process {
 
     cpus                    = { check_max( 1    * task.attempt, 'cpus'   ) }
@@ -24,12 +38,6 @@ process {
     maxRetries              = 1
     maxErrors               = '-1'
 
-    // Process-specific resource requirements
-    // NOTE - Please try and re-use the labels below as much as possible.
-    //        These labels are used and recognised by default in DSL2 files hosted on nf-core/modules.
-    //        If possible, it would be nice to keep the same label naming convention when
-    //        adding in your local modules too.
-    // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors
     withLabel:process_single {
         cpus                = { check_max( 1                  , 'cpus'     ) }
         memory              = { check_max( 6.GB * task.attempt, 'memory'   ) }
@@ -53,17 +61,13 @@ process {
     withLabel:process_long {
         time                = { check_max( 20.h  * task.attempt, 'time'    ) }
     }
-    withLabel:process_week_long {
-        time                = { check_max( 7.days  * task.attempt, 'time'  ) }
-    }
     withLabel:process_high_memory {
         memory              = { check_max( 200.GB * task.attempt, 'memory' ) }
     }
-}
 
-singularity {
-    enabled                 = true
-    autoMounts              = true
+    withName:CUSTOM_DUMPSOFTWAREVERSIONS {
+        cache = false
+    }
 }
 
 nextflow {
@@ -72,8 +76,6 @@ nextflow {
     }
 }
 
-// Source: https://github.com/nf-core/rnaseq
-// License: https://github.com/nf-core/rnaseq/blob/master/LICENSE
 def check_max(obj, type) {
     if (type == 'memory') {
         try {