-
Notifications
You must be signed in to change notification settings - Fork 277
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into UCD_SONIC
- Loading branch information
Showing
29 changed files
with
679 additions
and
84 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
|
|
||
| params { | ||
| config_profile_name = 'apollo' | ||
| config_profile_description = 'COH Apollo cluster profile provided by nf-core/configs.' | ||
| config_profile_contact = 'Denis O\'Meally (@drejom) or HPRCC Helpdesk ([email protected])' | ||
| config_profile_url = 'https://www.cityofhope.org/ Intranet http://hprcc.coh.org/' | ||
| max_memory = 1500.GB | ||
| max_cpus = 72 | ||
| max_time = 500.h | ||
| igenomes_base = '/ref_genome/igenomes' | ||
| } | ||
|
|
||
| process { | ||
| executor = 'slurm' | ||
| queue = 'all' | ||
| beforeScript = 'module load singularity' | ||
| } | ||
|
|
||
| singularity { | ||
| enabled = true | ||
| libraryDir = '/opt/singularity-images/nextflow' | ||
| } | ||
|
|
||
| executor { | ||
| name = 'slurm' | ||
| queueSize = 1000 | ||
| } | ||
|
|
||
| cleanup = true | ||
|
|
||
| profiles { | ||
| debug { | ||
| cleanup = false | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
|
|
||
| params { | ||
| config_profile_description = 'Biowulf nf-core config' | ||
| config_profile_contact = '[email protected]' | ||
| config_profile_url = 'https://hpc.nih.gov/apps/nextflow.html' | ||
| max_memory = '224 GB' | ||
| max_cpus = 32 | ||
| max_time = '72 h' | ||
|
|
||
| igenomes_base = '/fdb/igenomes/' | ||
| } | ||
|
|
||
| executor { | ||
|
|
||
| $slurm { | ||
| queue = 'norm' | ||
| queueSize = 200 | ||
| pollInterval = '2 min' | ||
| queueStatInterval = '5 min' | ||
| submitRateLimit = '6/1min' | ||
| retry.maxAttempts = 1 | ||
| } | ||
| } | ||
|
|
||
| singularity { | ||
| enabled = true | ||
| autoMounts = true | ||
| cacheDir = "/data/$USER/singularity" | ||
| envWhitelist='https_proxy,http_proxy,ftp_proxy,DISPLAY,SLURM_JOBID,SINGULARITY_BINDPATH' | ||
| } | ||
|
|
||
| env { | ||
| SINGULARITY_CACHEDIR="/data/$USER/singularity" | ||
| PYTHONNOUSERSITE = 1 | ||
| } | ||
|
|
||
|
|
||
| process { | ||
| executor = 'slurm' | ||
| maxRetries = 1 | ||
|
|
||
| clusterOptions = ' --gres=lscratch:200 ' | ||
|
|
||
| scratch = '/lscratch/$SLURM_JOBID' | ||
|
|
||
| stageInMode = 'symlink' | ||
| stageOutMode = 'rsync' | ||
|
|
||
| // for running pipeline on group sharing data directory, this can avoid inconsistent files timestamps | ||
| cache = 'lenient' | ||
| } | ||
|
|
||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| /* | ||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| Nextflow config file for EMBL-EBI Codon Cluster | ||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| Author: Saul Pierotti | ||
| Mail: [email protected] | ||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| */ | ||
|
|
||
| params { | ||
| config_profile_contact = "Saul Pierotti (@saulpierotti-ebi)" | ||
| config_profile_description = "The European Bioinformatics Institute HPC cluster (codon) profile" | ||
| config_profile_url = "https://www.ebi.ac.uk/" | ||
| } | ||
|
|
||
| singularity { | ||
| enabled = true | ||
| // the default is 20 minutes and fails with large images | ||
| pullTimeout = "3 hours" | ||
| } | ||
|
|
||
| process{ | ||
| queue = { | ||
| if ( task.time > 7.day && task.memory > 300.GB ) { | ||
| "long_bigmem" | ||
| } else if ( task.memory > 300.GB ) { | ||
| "bigmem" | ||
| } else if ( task.time > 7.day ) { | ||
| "long" | ||
| } else if ( task.time > 1.day ) { | ||
| "research" | ||
| } else { | ||
| "short" | ||
| } | ||
| } | ||
| // this is to avoid errors for missing files due to shared filesystem latency | ||
| maxRetries = 3 | ||
| errorStrategy = { ( task.exitStatus == 0 ) ? "retry" : "terminate" } | ||
| cache = "lenient" | ||
| afterScript = "sleep 60" // to avoid fail when using storeDir for missing output | ||
| } | ||
|
|
||
| executor { | ||
| name = "lsf" | ||
| queueSize = 2000 | ||
| submitRateLimit = "10/1sec" | ||
| exitReadTimeout = "30 min" | ||
| jobName = { | ||
| task.name // [] and " " not allowed in lsf job names | ||
| .replace("[", "(") | ||
| .replace("]", ")") | ||
| .replace(" ", "_") | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| /* | ||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| Nextflow config file for EMBL-EBI Codon Cluster for the SLURM login nodes | ||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| Author: Saul Pierotti | ||
| Mail: [email protected] | ||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| */ | ||
|
|
||
| params { | ||
| config_profile_contact = "Saul Pierotti (@saulpierotti-ebi)" | ||
| config_profile_description = "The European Bioinformatics Institute HPC cluster (codon) profile for the SLURM login nodes" | ||
| config_profile_url = "https://www.ebi.ac.uk/" | ||
| } | ||
|
|
||
| singularity { | ||
| enabled = true | ||
| // the default is 20 minutes and fails with large images | ||
| pullTimeout = "3 hours" | ||
| } | ||
|
|
||
| process{ | ||
| // this is to avoid errors for missing files due to shared filesystem latency | ||
| maxRetries = 3 | ||
| errorStrategy = { ( task.exitStatus == 0 ) ? "retry" : "terminate" } | ||
| cache = "lenient" | ||
| afterScript = "sleep 60" // to avoid fail when using storeDir for missing output | ||
| } | ||
|
|
||
| executor { | ||
| name = "slurm" | ||
| queueSize = 2000 | ||
| submitRateLimit = "10/1sec" | ||
| exitReadTimeout = "30 min" | ||
| jobName = { | ||
| task.name // [] and " " not allowed in lsf job names | ||
| .replace("[", "(") | ||
| .replace("]", ")") | ||
| .replace(" ", "_") | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| // Config profile metadata | ||
| params { | ||
| config_profile_name = 'FUB Curta' | ||
| config_profile_description = 'Freie Universität Berlin HPC cluster (Curta) profile' | ||
| config_profile_contact = '[email protected]' | ||
| config_profile_url = 'https://www.fu-berlin.de/en/sites/high-performance-computing/index.html' | ||
| max_memory = 93.GB | ||
| max_time = 14.d | ||
| max_cpus = 32 | ||
| } | ||
|
|
||
| // Slurm configuration parameters | ||
| process { | ||
| scratch = true | ||
| executor = 'slurm' | ||
| clusterOptions = '--qos=standard' | ||
| maxRetries = 2 | ||
| } | ||
|
|
||
| // Singularity parameters | ||
| singularity { | ||
| enabled = true | ||
| autoMounts = true | ||
| } | ||
|
|
||
| cleanup = true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| // NCI Gadi nf-core configuration profile | ||
| params { | ||
| config_profile_description = 'NCI Gadi HPC profile provided by nf-core/configs' | ||
| config_profile_contact = 'Georgie Samaha (@georgiesamaha), Matthew Downton (@mattdton)' | ||
| config_profile_url = 'https://opus.nci.org.au/display/Help/Gadi+User+Guide' | ||
| project = System.getenv("PROJECT") | ||
| } | ||
|
|
||
| // Enable use of Singularity to run containers | ||
| singularity { | ||
| enabled = true | ||
| autoMounts = true | ||
| } | ||
|
|
||
| // Submit up to 300 concurrent jobs (Gadi exec max) | ||
| // pollInterval and queueStatInterval of every 5 minutes | ||
| // submitRateLimit of 20 per minute | ||
| executor { | ||
| queueSize = 300 | ||
| pollInterval = '5 min' | ||
| queueStatInterval = '5 min' | ||
| submitRateLimit = '20 min' | ||
| } | ||
|
|
||
| // Define process resource limits | ||
| process { | ||
| executor = 'pbspro' | ||
| project = System.getenv("PROJECT") | ||
| storage = 'scratch/params.project' | ||
| module = 'singularity' | ||
| cache = 'lenient' | ||
| stageInMode = 'symlink' | ||
| queue = { task.memory < 128.GB ? 'normalbw' : (task.memory >= 128.GB && task.memory <= 190.GB ? 'normal' : (task.memory > 190.GB && task.memory <= 1020.GB ? 'hugemembw' : '')) } | ||
| beforeScript = 'module load singularity' | ||
| } | ||
|
|
||
| // Write custom trace file with outputs required for SU calculation | ||
| def trace_timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') | ||
| trace { | ||
| enabled = true | ||
| overwrite = false | ||
| file = "./gadi-nf-core-trace-${trace_timestamp}.txt" | ||
| fields = 'name,status,exit,duration,realtime,cpus,%cpu,memory,%mem,rss' | ||
| } |
Oops, something went wrong.