From a8f24667d19d17b2282872879f8ab8f29d0aae0c Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 14:36:55 +0000 Subject: [PATCH 01/10] change multiqc resource label --- modules/QualityAssessment/multiqc.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/QualityAssessment/multiqc.nf b/modules/QualityAssessment/multiqc.nf index 0b0aba33..649c3b11 100644 --- a/modules/QualityAssessment/multiqc.nf +++ b/modules/QualityAssessment/multiqc.nf @@ -1,6 +1,6 @@ process multiqc { publishDir "${params.output}/${prefix}/00_quality_assessment", mode: 'copy' - label 'process_ultralow' + label 'process_low' input: tuple val(id), val(entrypoint), val(prefix), file(quast_dirs) From 4bd6242d5877bbbae2f60c95658728dcf873731d Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 14:37:19 +0000 Subject: [PATCH 02/10] add new start_asm resources parameter and update defaults --- conf/defaults.config | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/conf/defaults.config b/conf/defaults.config index 56517220..2a926041 100644 --- a/conf/defaults.config +++ b/conf/defaults.config @@ -134,10 +134,32 @@ params { skip_shasta = false // Nanopore longreads only assemblies shasta_additional_parameters = null // Must be given as shown in shasta manual. E.g. " --Reads.minReadLength 5000 " -// Max resource options -// Defaults only, expecting to be overwritten - max_memory = '20.GB' - max_cpus = 6 + + /* + * Resources controlling parameters + * + * Here some parameters that allow the user to better tune the resources used by the pipeline. + * + * The start_asm_{mem,cpus} parameter tells the pipeline how much memory should the assembly + * modules and quast request in the first try. This is essential for bigger genomes in order + * to avoid having to fail the first try due lack of memory and then running again (automatically) + * using all the max values allowed with the max_{mem,cpus} parameters. + * + * The max_memory and max_cpus parameters, tell the pipeline how much is the maximum number of + * these items that is allowoed per job. The pipeline start by requesting less mem&cpus than + * what is defined by these params, and, in case the first try fails, it then maxes out the job + * to use the maximum number you allowed. + * + * The max_time parameter defines how long a single job is allowed to run. + */ + + // starting values for the assembly jobs (and quast) to ask for in the very first try + start_asm_mem = 20.GB + start_asm_cpus = 6 + + // maximum values to be used on automatic second try in case of lack of memory (all jobs) + max_memory = 40.GB + max_cpus = 10 max_time = '40.h' } \ No newline at end of file From ad32a19e37b4557bdafa1c65f46a6f2275da4d86 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 14:37:58 +0000 Subject: [PATCH 03/10] update default resources for modules without labels, modify assembly modules to use newly added parameters and add function to parse resources for quast based on them. --- conf/base.config | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/conf/base.config b/conf/base.config index a1fdaee7..ebf7d6c4 100644 --- a/conf/base.config +++ b/conf/base.config @@ -1,9 +1,9 @@ process { // The defaults for all processes (without labels) - cpus = { params.max_cpus } - memory = { params.max_memory } - time = { params.max_time } + cpus = 2 + memory = 4.GB + time = { params.max_time } errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } maxRetries = 1 @@ -18,7 +18,7 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( 1.h * task.attempt, 'time' ) } + time = { check_max( 2.h * task.attempt, 'time' ) } errorStrategy = { task.exitStatus in [21,143,137,104,134,139,247] ? 'retry' : 'finish' } maxRetries = 1 @@ -35,9 +35,8 @@ process { // Assemblies will first try to adjust themselves to a parallel execution // If it is not possible, then it waits to use all the resources allowed withLabel:process_assembly { - cpus = { if (task.attempt == 1) { check_max( 6 * task.attempt, 'cpus' ) } else { params.max_cpus } } - memory = { if (task.attempt == 1) { check_max( 20.GB * task.attempt, 'memory' ) } else { params.max_memory } } - time = { if (task.attempt == 1) { check_max( 24.h * task.attempt, 'time' ) } else { params.max_time } } + cpus = { if (task.attempt == 1) { params.start_asm_cpus } else { params.max_cpus } } + memory = { if (task.attempt == 1) { params.start_asm_mem } else { params.max_memory } } // retry at least once to try it with full resources errorStrategy = { task.exitStatus in [1,21,143,137,104,134,139,247] ? 'retry' : 'finish' } @@ -46,10 +45,12 @@ process { } // Quast sometimes can take too long + def quast_mem = { ((params.start_asm_mem % 2) > 6.GB) ? (params.start_asm_mem % 2) : 6.GB } + def quast_cpus = { ((params.start_asm_cpus % 2) > 4 ) ? (params.start_asm_mem % 2) : 4 } withName:quast { - cpus = { if (task.attempt == 1) { check_max( 4 * task.attempt, 'cpus' ) } else { params.max_cpus } } - memory = { if (task.attempt == 1) { check_max( 10.GB * task.attempt, 'memory' ) } else { params.max_memory } } - time = { if (task.attempt == 1) { check_max( 12.h * task.attempt, 'time' ) } else { params.max_time } } + cpus = { if (task.attempt == 1) { quast_cpus } else { params.max_cpus } } + memory = { if (task.attempt == 1) { quast_mem } else { params.max_memory } } + time = { if (task.attempt == 1) { check_max( 12.h * task.attempt, 'time' ) } else { params.max_time } } // retry at least once to try it with full resources errorStrategy = { task.exitStatus in [21,143,137,104,134,139,247] ? 'retry' : 'finish' } From 14ae77fd6d4ace75bbc3ac29bedbba38536f5cb3 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 15:00:11 +0000 Subject: [PATCH 04/10] fix order of inclusion --- nextflow.config | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nextflow.config b/nextflow.config index 921b5f11..657b5789 100644 --- a/nextflow.config +++ b/nextflow.config @@ -4,11 +4,10 @@ Maintained by Felipe Marques de Almeida Contact: almeidafmarques@outlook.com */ -// Load base.config (contains some label resources configuration) -includeConfig 'conf/base.config' - // loading required / default pipeline parameters includeConfig 'conf/defaults.config' +// Load base.config (contains some label resources configuration) +includeConfig 'conf/base.config' // fix type of variable expected params.hybrid_strategy = params.hybrid_strategy.toString() From eb0a71cbf52f118fc2641c33bb7c96752856e9f1 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 15:00:24 +0000 Subject: [PATCH 05/10] fix quast resources function --- conf/base.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/base.config b/conf/base.config index ebf7d6c4..af045fc9 100644 --- a/conf/base.config +++ b/conf/base.config @@ -45,8 +45,8 @@ process { } // Quast sometimes can take too long - def quast_mem = { ((params.start_asm_mem % 2) > 6.GB) ? (params.start_asm_mem % 2) : 6.GB } - def quast_cpus = { ((params.start_asm_cpus % 2) > 4 ) ? (params.start_asm_mem % 2) : 4 } + def quast_mem = ((params.start_asm_mem / 2) > 6.GB) ? (params.start_asm_mem / 2) : 6.GB + def quast_cpus = ((params.start_asm_cpus / 2) > 4 ) ? (params.start_asm_mem / 2) : 4 withName:quast { cpus = { if (task.attempt == 1) { quast_cpus } else { params.max_cpus } } memory = { if (task.attempt == 1) { quast_mem } else { params.max_memory } } From 150d0f01a1e8b1e7c8a33577c1193a7d712ac8e4 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 15:05:53 +0000 Subject: [PATCH 06/10] add explanation of parameter in documentation --- docs/manual.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/manual.md b/docs/manual.md index 12278164..b7217853 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -84,13 +84,15 @@ Please note that, through the command line, the parameters that are boolean (tru | `--input` | :material-check: | NA | Path to input [samplesheet](samplesheet.md#) in YAML format | | `--output` | :material-check: | NA | Directory to store output files | -## Max job request +## Start/Max resources on job request |
Parameter
| Required | Default | Description | | :--------------------------------------- | :------- | :------ | :---------- | -| `--max_cpus` | :material-close: | 4 | Max number of threads a job can use across attempts | -| `--max_memory` | :material-close: | 6.GB | Max amount of memory a job can use across attempts | -| `--max_time` | :material-close: | 40.h | Max amount of time a job can take to run | +| `--start_asm_cpus` | :material-close: | 6 | How many cpus should an assembly job request in the very first attempt?. This is essential for bigger genomes in order to avoid having to fail the first try due lack of memory and then running again (automatically) using all the max values allowed with the max_cpus parameter. | +| `--start_asm_mem` | :material-close: | 20.GB | How much memory should an assembly job request in the very first attempt?. This is essential for bigger genomes in order to avoid having to fail the first try due lack of memory and then running again (automatically) using all the max values allowed with the max_mem parameter. | +| `--max_cpus` | :material-close: | 10 | Max number of threads a job can use across attempts. After one failed attempt this is maxed out. | +| `--max_memory` | :material-close: | 40.GB | Max amount of memory a job can use across attempts. After one failed attempt this is maxed out. | +| `--max_time` | :material-close: | 40.h | Max amount of time a job can take to run | ## Assemblies configuration From 0b9d484a0dd865c55fdcd69f013d03026de6a8e3 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 15:08:22 +0000 Subject: [PATCH 07/10] add parameter in nextflow schema and update defaults --- nextflow_schema.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index bd6f93b0..d5c8010f 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -41,14 +41,24 @@ "description": "Set the top limit of resources for pipeline", "help_text": "If you are running on a smaller system, a pipeline step requesting more resources than are available may cause the Nextflow to stop the run with an error. These options allow you to cap the maximum resources requested by any single job so that the pipeline will run on your system.\n\nNote that you can not _increase_ the resources requested by any job using these options. For that you will need your own configuration file. See [the nf-core website](https://nf-co.re/usage/configuration) for details.", "properties": { - "max_cpus": { + "start_asm_cpus": { "type": "integer", "default": 6, + "description": "Starting (1st try) amount of cpus that assembly jobs should use. Essential for avoiding 1st-try errors due lack of resources for big genomes." + }, + "start_asm_mem": { + "type": "string", + "default": "20.GB", + "description": "Starting (1st try) amount of memory that assembly jobs should use. Essential for avoiding 1st-try errors due lack of resources for big genomes." + }, + "max_cpus": { + "type": "integer", + "default": 10, "description": "Max amount of threads to use" }, "max_memory": { "type": "string", - "default": "20.GB", + "default": "40.GB", "description": "Max amount of memory to use" }, "max_time": { From db878fb727f3b73cfa4c01b665fcd3f2f9ac5c3f Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 15:09:49 +0000 Subject: [PATCH 08/10] update changelog --- markdown/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/markdown/CHANGELOG.md b/markdown/CHANGELOG.md index e70a591b..35d5528c 100644 --- a/markdown/CHANGELOG.md +++ b/markdown/CHANGELOG.md @@ -10,6 +10,7 @@ The tracking for changes started in v2. * Increase default `--max_memory` value to 20.GB. * Add a directory called `final_assemblies` in the main output directory holding all the assemblies generated in the pipeline execution. * Updated documentation as discussed in [[#58](https://github.com/fmalmeida/MpGAP/issues/58)] and [[#57](https://github.com/fmalmeida/MpGAP/issues/57)]. +* [[#61](https://github.com/fmalmeida/MpGAP/issues/61)] - Add a simple parameter to adjust how many cpus and how much memory should the assembly jobs request in the first attempt to avoid lack of resources errors. ## v3.1.4 -- [2022-Sep-03] From a9c43586ebb338f42c5d1cff36e0b28bcc3ec346 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 15:12:47 +0000 Subject: [PATCH 09/10] also wrap-up 1st attempt in check_max function to avoid requesting more than what is allowed --- conf/base.config | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/base.config b/conf/base.config index af045fc9..b0d9e26e 100644 --- a/conf/base.config +++ b/conf/base.config @@ -35,8 +35,8 @@ process { // Assemblies will first try to adjust themselves to a parallel execution // If it is not possible, then it waits to use all the resources allowed withLabel:process_assembly { - cpus = { if (task.attempt == 1) { params.start_asm_cpus } else { params.max_cpus } } - memory = { if (task.attempt == 1) { params.start_asm_mem } else { params.max_memory } } + cpus = { if (task.attempt == 1) { check_max( params.start_asm_cpus, 'cpus' ) } else { params.max_cpus } } + memory = { if (task.attempt == 1) { check_max( params.start_asm_mem , 'memory' ) } else { params.max_memory } } // retry at least once to try it with full resources errorStrategy = { task.exitStatus in [1,21,143,137,104,134,139,247] ? 'retry' : 'finish' } @@ -48,9 +48,9 @@ process { def quast_mem = ((params.start_asm_mem / 2) > 6.GB) ? (params.start_asm_mem / 2) : 6.GB def quast_cpus = ((params.start_asm_cpus / 2) > 4 ) ? (params.start_asm_mem / 2) : 4 withName:quast { - cpus = { if (task.attempt == 1) { quast_cpus } else { params.max_cpus } } - memory = { if (task.attempt == 1) { quast_mem } else { params.max_memory } } - time = { if (task.attempt == 1) { check_max( 12.h * task.attempt, 'time' ) } else { params.max_time } } + cpus = { if (task.attempt == 1) { check_max( params.quast_cpus, 'cpus' ) } else { params.max_cpus } } + memory = { if (task.attempt == 1) { check_max( params.quast_mem , 'memory' ) } else { params.max_memory } } + time = { if (task.attempt == 1) { check_max( 12.h * task.attempt, 'time' ) } else { params.max_time } } // retry at least once to try it with full resources errorStrategy = { task.exitStatus in [21,143,137,104,134,139,247] ? 'retry' : 'finish' } From 2ec82801e802fbe9176b1ca4b9af43c7285ea84b Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 16 Feb 2024 15:45:17 +0000 Subject: [PATCH 10/10] update docs/assets --- docs/assets/defaults.config | 40 +++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/docs/assets/defaults.config b/docs/assets/defaults.config index a7541241..2a926041 100644 --- a/docs/assets/defaults.config +++ b/docs/assets/defaults.config @@ -56,14 +56,16 @@ params { // Select the appropriate shasta config to use for assembly // Since shasta v0.8 (Oct/2021) this parameter is now mandatory. +// You can check availability at: https://paoloshasta.github.io/shasta/Configurations.html shasta_config = "Nanopore-Oct2021" // Tells the pipeline to interpret the long reads as "corrected" long reads. -// This will activate (if available) the options for corrected reads in the -// assemblers: -corrected (in canu), --pacbio-corr|--nano-corr (in flye), etc. -// Be cautious when using this parameter. If your reads are not corrected, and +// This will activate (if available) the options for corrected or even high +// quality (hq) reads in the assemblers. +// Be cautious when using this parameter. If your reads are not corrected|hq, and // you use this parameter, you will probably do not generate any contig. - corrected_long_reads = false + corrected_longreads = false + high_quality_longreads = false // This parameter below (hybrid_strategy) is to select the hybrid strategies adopted by the pipeline. // Read the documentation https://mpgap.readthedocs.io/en/latest/manual.html to know more about the hybrid strategies. @@ -132,10 +134,32 @@ params { skip_shasta = false // Nanopore longreads only assemblies shasta_additional_parameters = null // Must be given as shown in shasta manual. E.g. " --Reads.minReadLength 5000 " -// Max resource options -// Defaults only, expecting to be overwritten - max_memory = '14.GB' - max_cpus = 6 + + /* + * Resources controlling parameters + * + * Here some parameters that allow the user to better tune the resources used by the pipeline. + * + * The start_asm_{mem,cpus} parameter tells the pipeline how much memory should the assembly + * modules and quast request in the first try. This is essential for bigger genomes in order + * to avoid having to fail the first try due lack of memory and then running again (automatically) + * using all the max values allowed with the max_{mem,cpus} parameters. + * + * The max_memory and max_cpus parameters, tell the pipeline how much is the maximum number of + * these items that is allowoed per job. The pipeline start by requesting less mem&cpus than + * what is defined by these params, and, in case the first try fails, it then maxes out the job + * to use the maximum number you allowed. + * + * The max_time parameter defines how long a single job is allowed to run. + */ + + // starting values for the assembly jobs (and quast) to ask for in the very first try + start_asm_mem = 20.GB + start_asm_cpus = 6 + + // maximum values to be used on automatic second try in case of lack of memory (all jobs) + max_memory = 40.GB + max_cpus = 10 max_time = '40.h' } \ No newline at end of file