diff --git a/docs/developer/plugins.md b/docs/developer/plugins.md index 98044a4ee1..52d9b6dc0a 100644 --- a/docs/developer/plugins.md +++ b/docs/developer/plugins.md @@ -98,7 +98,7 @@ class MyObserver implements TraceObserver { @Override void onFlowCreate(Session session) { - final message = session.config.navigate('myplugin.create.message') + final message = session.config.navigate('myplugin.createMessage') println message } } @@ -108,16 +108,43 @@ You can then set this option in your config file: ```groovy // dot syntax -myplugin.create.message = "I'm alive!" +myplugin.createMessage = "I'm alive!" -// closure syntax +// block syntax myplugin { - create { - message = "I'm alive!" + createMessage = "I'm alive!" +} +``` + +:::{versionadded} 25.02.0-edge +::: + +Plugins can declare their config options by implementing the `ConfigScope` interface and declaring each config option as a field with the `@ConfigOption` annotation: + +```groovy +import nextflow.config.dsl.ConfigOption +import nextflow.config.dsl.ConfigScope +import nextflow.config.dsl.ScopeName +import nextflow.script.dsl.Description + +@ScopeName('myplugin') +@Description(''' + The `myplugin` scope allows you to configure the `nf-myplugin` plugin. +''') +class MyPluginConfig implements ConfigScope { + + MyPluginConfig(Map opts) { + this.createMessage = opts.createMessage } + + @ConfigOption + @Description('Message to print to standard output when a run is initialized.') + String createMessage } ``` +While this approach is not required to support plugin config options, it allows Nextflow to recognize plugin definitions when validating the config. + ### Executors Plugins can define custom executors that can then be used with the `executor` process directive. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy index e120653127..e9a38a375d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy @@ -25,6 +25,7 @@ import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.config.ConfigBuilder +import nextflow.config.ConfigValidator import nextflow.exception.AbortOperationException import nextflow.plugin.Plugins import nextflow.scm.AssetManager @@ -81,6 +82,7 @@ class CmdConfig extends CmdBase { if( args ) base = getBaseDir(args[0]) if( !base ) base = Paths.get('.') + // -- validate command line options if( profile && showAllProfiles ) { throw new AbortOperationException("Option `-profile` conflicts with option `-show-profiles`") } @@ -103,6 +105,7 @@ class CmdConfig extends CmdBase { if( printProperties ) outputFormat = 'properties' + // -- build the config final builder = new ConfigBuilder() .setShowClosures(true) .setStripSecrets(true) @@ -113,6 +116,10 @@ class CmdConfig extends CmdBase { final config = builder.buildConfigObject() + // -- validate config options + new ConfigValidator().validate(config) + + // -- print config options if( printValue ) { printValue0(config, printValue, stdout) } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index fca4596a9c..86dbac6282 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -37,6 +37,7 @@ import nextflow.NextflowMeta import nextflow.SysEnv import nextflow.config.ConfigBuilder import nextflow.config.ConfigMap +import nextflow.config.ConfigValidator import nextflow.exception.AbortOperationException import nextflow.file.FileHelper import nextflow.plugin.Plugins @@ -334,13 +335,13 @@ class CmdRun extends CmdBase implements HubOptions { // check DSL syntax in the config launchInfo(config, scriptFile) - // check if NXF_ variables are set in nextflow.config - checkConfigEnv(config) - // -- load plugins final cfg = plugins ? [plugins: plugins.tokenize(',')] : config Plugins.load(cfg) + // -- validate config options + new ConfigValidator().validate(config) + // -- create a new runner instance final runner = new ScriptRunner(config) runner.setScript(scriptFile) @@ -407,17 +408,6 @@ class CmdRun extends CmdBase implements HubOptions { } } - protected checkConfigEnv(ConfigMap config) { - // Warn about setting NXF_ environment variables within env config scope - final env = config.env as Map - for( String name : env.keySet() ) { - if( name.startsWith('NXF_') && name!='NXF_DEBUG' ) { - final msg = "Nextflow variables must be defined in the launching environment - The following variable set in the config file is going to be ignored: '$name'" - log.warn(msg) - } - } - } - protected void launchInfo(ConfigMap config, ScriptFile scriptFile) { // -- determine strict mode detectStrictFeature(config, sysEnv) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigValidator.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigValidator.groovy new file mode 100644 index 0000000000..bbc0ed494d --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigValidator.groovy @@ -0,0 +1,131 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.config.dsl.ConfigSchema +import nextflow.config.dsl.ConfigScope +import nextflow.config.dsl.SchemaNode +import nextflow.config.dsl.ScopeName +import nextflow.config.dsl.ScopeNode +import nextflow.plugin.Plugins +import nextflow.script.dsl.Description +/** + * Validate the Nextflow configuration + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class ConfigValidator { + + /** + * Hidden options added by ConfigBuilder + */ + private static final List hiddenOptions = List.of( + 'bucketDir', + 'cacheable', + 'dumpChannels', + 'libDir', + 'poolSize', + 'plugins', + 'preview', + 'runName', + 'stubRun', + ); + + /** + * Additional config scopes added by third-party plugins + */ + private ScopeNode pluginScopes + + ConfigValidator() { + loadPluginScopes() + } + + private void loadPluginScopes() { + final scopes = new HashMap() + for( final scope : Plugins.getExtensions(ConfigScope) ) { + final clazz = scope.getClass() + final name = clazz.getAnnotation(ScopeName)?.value() + final description = clazz.getAnnotation(Description)?.value() + if( !name ) + continue + if( name in scopes ) { + log.warn "Plugin config scope `${clazz.name}` conflicts with existing scope: `${name}`" + continue + } + scopes.put(name, ScopeNode.of(clazz, description)) + } + pluginScopes = new ScopeNode('', [:], scopes) + } + + void validate(ConfigMap config) { + validate(config.toConfigObject()) + } + + void validate(ConfigObject config) { + final flatConfig = config.flatten() + for( String key : flatConfig.keySet() ) { + final names = key.tokenize('.') + if( names.first() == 'profiles' ) { + if( !names.isEmpty() ) names.remove(0) + if( !names.isEmpty() ) names.remove(0) + } + final scope = names.first() + if( scope == 'env' ) { + checkEnv(names.last()) + continue + } + if( scope == 'params' ) + continue + final fqName = names.join('.') + if( fqName.startsWith('process.ext.') ) + return + if( !isValid(names) ) { + log.warn "Unrecognized config option '${fqName}'" + continue + } + } + } + + /** + * Determine whether a config option is defined in the schema. + * + * @param names + */ + boolean isValid(List names) { + if( names.size() == 1 && names.first() in hiddenOptions ) + return true + if( ConfigSchema.ROOT.getOption(names) ) + return true + if( pluginScopes.getOption(names) ) + return true + return false + } + + /** + * Warn about setting `NXF_*` environment variables in the config. + * + * @param name + */ + private void checkEnv(String name) { + if( name.startsWith('NXF_') && name!='NXF_DEBUG' ) + log.warn "Nextflow environment variables must be defined in the launch environment -- the following environment variable in the config will be ignored: `$name`" + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy index a2f97b7496..5860721ded 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy @@ -406,49 +406,6 @@ class CmdConfigTest extends Specification { .stripIndent() } - @IgnoreIf({System.getenv('NXF_SMOKE')}) - def 'should resolve profiles into profiles config' () { - given: - def folder = Files.createTempDirectory('test') - def CONFIG = folder.resolve('nextflow.config') - - CONFIG.text = ''' - params { - foo = 'baz' - } - - profiles { - test { - params { - foo = 'foo' - } - profiles { - debug { - cleanup = false - } - } - } - } - ''' - - def buffer = new ByteArrayOutputStream() - // command definition - def cmd = new CmdConfig(showAllProfiles: true) - cmd.launcher = new Launcher(options: new CliOptions(config: [CONFIG.toString()])) - cmd.stdout = buffer - cmd.args = ['.'] - - when: - cmd.run() - def result = new ConfigSlurper().parse(buffer.toString()) - - then: - result.params.foo == 'baz' - result.profiles.test.params.foo == 'foo' - result.profiles.debug.cleanup == false - } - - def 'should remove secrets for config' () { given: SecretsLoader.instance.reset() diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy index df6448c25f..57b944aa3e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy @@ -23,10 +23,8 @@ import nextflow.NextflowMeta import nextflow.SysEnv import nextflow.config.ConfigMap import nextflow.exception.AbortOperationException -import org.junit.Rule import spock.lang.Specification import spock.lang.Unroll -import test.OutputCapture /** * @@ -34,9 +32,6 @@ import test.OutputCapture */ class CmdRunTest extends Specification { - @Rule - OutputCapture capture = new OutputCapture() - @Unroll def 'should parse cmd param=#STR' () { @@ -390,40 +385,6 @@ class CmdRunTest extends Specification { CmdRun.detectDslMode(new ConfigMap(), DSL2_SCRIPT, [:]) == '2' } - def 'should warn for invalid config vars' () { - given: - def ENV = [NXF_ANSI_SUMMARY: 'true'] - - when: - new CmdRun().checkConfigEnv(new ConfigMap([env:ENV])) - - then: - def warning = capture - .toString() - .readLines() - .findResults { line -> line.contains('WARN') ? line : null } - .join('\n') - and: - warning.contains('Nextflow variables must be defined in the launching environment - The following variable set in the config file is going to be ignored: \'NXF_ANSI_SUMMARY\'') - } - - def 'should not warn for valid config vars' () { - given: - def ENV = [FOO: '/something', NXF_DEBUG: 'true'] - - when: - new CmdRun().checkConfigEnv(new ConfigMap([env:ENV])) - - then: - def warning = capture - .toString() - .readLines() - .findResults { line -> line.contains('WARN') ? line : null } - .join('\n') - and: - !warning - } - @Unroll def 'should detect moduleBinaries' () { given: diff --git a/tests/checks/profiles.nf/.expected_profile_advanced.txt b/tests/checks/profiles.nf/.expected_profile_advanced.txt index 8f37848937..7249777246 100644 --- a/tests/checks/profiles.nf/.expected_profile_advanced.txt +++ b/tests/checks/profiles.nf/.expected_profile_advanced.txt @@ -1,5 +1,4 @@ -echo = true -foo = 'bar' +outputDir = 'results' process { cpus = 8 diff --git a/tests/checks/profiles.nf/.expected_profile_all.txt b/tests/checks/profiles.nf/.expected_profile_all.txt index 100d21af18..911fd3b7ae 100644 --- a/tests/checks/profiles.nf/.expected_profile_all.txt +++ b/tests/checks/profiles.nf/.expected_profile_all.txt @@ -1,5 +1,4 @@ -echo = true -foo = 'bar' +outputDir = 'results' profiles { standard { diff --git a/tests/checks/profiles.nf/.expected_profile_standard.txt b/tests/checks/profiles.nf/.expected_profile_standard.txt index bb23e44f1c..42f335ced9 100644 --- a/tests/checks/profiles.nf/.expected_profile_standard.txt +++ b/tests/checks/profiles.nf/.expected_profile_standard.txt @@ -1,5 +1,4 @@ -echo = true -foo = 'bar' +outputDir = 'results' process { cpus = 2 diff --git a/tests/delta.config b/tests/delta.config index 79e60a11c9..ac94966274 100644 --- a/tests/delta.config +++ b/tests/delta.config @@ -1 +1 @@ -foo = 'bar' \ No newline at end of file +outputDir = 'results' \ No newline at end of file diff --git a/tests/profiles.config b/tests/profiles.config index 95f0ac4ee9..9eab9b8a83 100644 --- a/tests/profiles.config +++ b/tests/profiles.config @@ -14,8 +14,6 @@ * limitations under the License. */ -echo = true - includeConfig "${'delta'}.config" profiles {