first version

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
         id: release
         with:
           release-type: python # just keep a changelog, no version anywhere outside of git tags
-          package-name: juno_template
+          package-name: juno_assembly_verfication
   lint:
     name: Lint Code Base
     runs-on: ubuntu-latest

README.md

@@ -1,7 +1,28 @@
-# Juno-Template
-A template pipeline where the other juno pipelines are based on.
+# Juno-assembly verification
+
+This pipeline aims to make the verification of Juno-assembly versions easier.
+
+Validations and verification are organised around pipeline functionalities, which are managed by different groups. Each group has their own criteria for validation and verification. This workflow is organised around this concept: each group has their own verification workflow which is are separate from each other.
+
+Configuration files and reference data are stored in /mnt/db/juno and are under version control of a local (not uploaded) git.
+
+## General components for verification workflows
+
+General components used by workflows
+
+- `config/config_GROUP.tsv`: Config file indicating which columns should be compare with which operator. E.g. "# contigs" from results file (`result_col`) should be less than or equal to (`operator`) 150 contigs for sample X (`criterium_col`). Thresholds/values per sample are defined in `config/verification_criteria_GROUP.tsv`. 
+- `config/verification_criteria_GROUP.tsv`: defines per sample which thresholds/values the results should be compared to. `criterium_col` from `config/config_GROUP.tsv` should be present in this file.
+- `rules/GROUP/verify_qc.smk`: Snakemake rule which calls `workflow/scripts/verify_qc.py`. The Python script uses `config/config_GROUP.tsv`, `config/verification_criteria_GROUP.tsv` and result files and compares these to assign TRUE/FALSE indicating whether a threshold/value was met. TO DO: separate parsing/summarising of results and actual comparison, this now happens in a single script.
+
+Specific components used
+
+- The AMR workflow also compares typing results. Plasmidfinder and Resfinder containers (including fixed database versions) are used to type assemblies. Relevant data (gene name, nucleotide identity, reference coverage and accession number) are written to a sorted table of which hashes are compared with reference hashes. A hash of the raw output file is not used, as the table order can differ as well positions of hits within an assembly.
+- The AMR workflow also outputs a folder `de_novo_assembly_filtered` with assemblies renamed to reflect the Juno-assembly version in the file name. This folder can be used by an automated SeqSphere script to type assemblies according to MLST and wgMLST.
+
 
 ## Contribution guidelines
+This pipeline is based on Juno-template.
+
 Juno pipelines use a [feature branch workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow). To work on features, create a branch from the `main` branch to make changes to. This branch can be merged to the main branch via a pull request. Hotfixes for bugs can be committed to the `main` branch.
 
 Please adhere to the [conventional commits](https://www.conventionalcommits.org/) specification for commit messages. These commit messages can be picked up by [release please](https://github.com/googleapis/release-please) to create meaningful release messages.

Snakefile

@@ -1,25 +1,115 @@
 import yaml
+import pandas as pd
 
+def select_samples_per_group(verification_table, sample_sheet_dict):
+    """
+    Selects subset of samples based on verification table.
+
+    Args:
+        verification_table: path to verification table to parse sample names from.
+        sample_sheet_dict: dict with sample names as key, generated by juno-library.
+    
+    Returns:
+        list of sample names present in both sample_sheet_dict and verification table.
+    """
+    df = pd.read_csv(verification_table, sep='\t')
+    selected_samples = []
+    for sample in df['sample']:
+        if sample in sample_sheet_dict.keys():
+            selected_samples.append(sample)
+    return selected_samples
 
 sample_sheet=config["sample_sheet"]
 with open(sample_sheet) as f:
     SAMPLES = yaml.safe_load(f)
 
-for param in ["threads", "mem_gb"]:
-    for k in config[param]:
-        config[param][k] = int(config[param][k])
-
-# print(SAMPLES)
+INPUT = config["input_dir"]
+OUT = config["out"]
 
-OUT = config["output_dir"]
 
 localrules:
     all,
+    verify_accuracy_BVI,
+    verify_accuracy_AMR,
+    collect_and_rename,
+    select_cols_resfinder,
+    select_cols_plasmidfinder,
+    get_hashes_resfinder,
+    get_hashes_plasmidfinder,
+    typing_report,
+    display_typing_discrepancies,
+
+
+####################################################
+###             BVI verification                 ###
+####################################################
+
+BVI_SAMPLES = select_samples_per_group("config/verification_criteria_BVI.tsv", SAMPLES) # has to be implemented in main wrapper
+
+# Create empty list and append.
+# If no samples from the input folder are also in the verification table,
+# Snakemake will not attempt to make output for this group of samples.
+BVI_OUTPUT = []
+
+if len(BVI_SAMPLES) > 0:
+    BVI_OUTPUT.append(OUT + "/verification_subreports/verify_accuracy_BVI.tsv")
+    BVI_OUTPUT.append(OUT + "/verification_subreports/BVI_assembly_hashes.txt")
+    BVI_OUTPUT.append(OUT + "/verification_subreports/BVI_clean_reads_hashes.txt")
+    BVI_OUTPUT.append(OUT + "/verification_subreports/BVI_id_species_hashes.txt")
+    BVI_OUTPUT.append(OUT + "/verification_subreports/BVI_qc_assembly_hashes.txt")
+
+# Include BVI rules
+include: "workflow/rules/BVI/verify_accuracy.smk"
 
+####################################################
+###             AMR verification                 ###
+####################################################
 
-include: "workflow/rules/rule.smk"
+AMR_SAMPLES = select_samples_per_group("config/verification_criteria_AMR.tsv", SAMPLES) # has to be implemented in main wrapper
 
+# Create empty list and append.
+# If no samples from the input folder are also in the verification table,
+# Snakemake will not attempt to make output for this group of samples.
+AMR_OUTPUT = []
+
+if len(AMR_SAMPLES) > 0:
+    AMR_OUTPUT.append(expand(OUT + "/verification_subreports/verify_accuracy_AMR.tsv"))
+    AMR_OUTPUT.append(OUT + "/de_novo_assembly_filtered")
+    AMR_OUTPUT.append(OUT + "/discrepancies")
+
+# Include AMR rules
+include: "workflow/rules/AMR/typing.smk"
+include: "workflow/rules/AMR/copy_assemblies.smk"
+include: "workflow/rules/AMR/verify_accuracy.smk"
+
+####################################################
+###             RVP verification                 ###
+####################################################
+
+# RVP_SAMPLES = select_samples_per_group("config/verification_criteria_RVP.tsv", SAMPLES) # has to be implemented in main wrapper
+
+# RVP_OUTPUT = []
+
+# if len(RVP_SAMPLES) > 0:
+#     RVP_OUTPUT.append(OUT + "/verification_subreports/verify_qc_RVP.tsv")
+
+# # Include rules
+# include: "workflow/rules/RVP/verify_qc.smk"
 
 rule all:
     input:
-        expand(OUT + "/{sample}_combined.fastq", sample=SAMPLES),
+        BVI_OUTPUT,
+        AMR_OUTPUT,
+        OUT + "/git_version.txt"
+
+rule get_git_version:
+    output:
+        OUT + "/git_version.txt"
+    params:
+        reference_dir = config["reference_dir"],
+        config_dir = config["config_dir"],
+    shell:
+        """
+date | tr '\n' '\t' > {output}
+git --git-dir {params.reference_dir}/../.git -n 1 --pretty=format:"%H" >> {output}
+        """

config/pipeline_parameters.yaml

@@ -1,5 +1,5 @@
 threads:
-    template_rule: 1
+    typing: 2
 
 mem_gb:
-    template_rule: 1
+    typing: 4

envs/juno_assembly_verification.yaml

@@ -0,0 +1,16 @@
+name: juno_assembly_verification
+channels:
+  - bioconda
+  - conda-forge
+  - anaconda
+  - defaults
+dependencies:
+  - git
+  - mamba
+  - pandas
+  - snakemake
+  - pyyaml
+  - pip
+  - openpyxl
+  - pip:
+    - "--editable=git+https://github.com/RIVM-bioinformatics/[email protected]#egg=base_juno"