From 17af1e878c238a201003024c9e9f546d72483567 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Tue, 25 Feb 2025 15:50:31 -0600 Subject: [PATCH] Validate config schema Signed-off-by: Ben Sherman --- docs/developer/plugins.md | 43 ++++++++- .../main/groovy/nextflow/cli/CmdConfig.groovy | 7 ++ .../main/groovy/nextflow/cli/CmdRun.groovy | 18 +--- .../nextflow/config/ConfigValidator.groovy | 92 +++++++++++++++++++ 4 files changed, 141 insertions(+), 19 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/config/ConfigValidator.groovy diff --git a/docs/developer/plugins.md b/docs/developer/plugins.md index 98044a4ee1..b3afc135a5 100644 --- a/docs/developer/plugins.md +++ b/docs/developer/plugins.md @@ -98,7 +98,7 @@ class MyObserver implements TraceObserver { @Override void onFlowCreate(Session session) { - final message = session.config.navigate('myplugin.create.message') + final message = session.config.navigate('myplugin.createMessage') println message } } @@ -108,16 +108,49 @@ You can then set this option in your config file: ```groovy // dot syntax -myplugin.create.message = "I'm alive!" +myplugin.createMessage = "I'm alive!" -// closure syntax +// block syntax myplugin { - create { - message = "I'm alive!" + createMessage = "I'm alive!" +} +``` + +:::{versionadded} 25.02.0-edge +::: + +Plugins can declare their config options by implementing the `ConfigScope` interface and declaring each config option as a field with the `@ConfigOption` annotation: + +```groovy +class MyPluginConfig implements ConfigScope { + + MyPluginConfig() {} + + MyPluginConfig(Map opts) { + this.createMessage = opts.createMessage } + + @Override + String name() { + return 'myplugin' + } + + @Override + String description() { + return ''' + The `myplugin` scope... + ''' + } + + @ConfigOption(''' + Message to print to standard output when a run is initialized. + ''') + String createMessage } ``` +While this approach is not required to support plugin config options, it is used by Nextflow to recognize plugin definitions when validating the config. + ### Executors Plugins can define custom executors that can then be used with the `executor` process directive. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy index e120653127..e9a38a375d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy @@ -25,6 +25,7 @@ import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.config.ConfigBuilder +import nextflow.config.ConfigValidator import nextflow.exception.AbortOperationException import nextflow.plugin.Plugins import nextflow.scm.AssetManager @@ -81,6 +82,7 @@ class CmdConfig extends CmdBase { if( args ) base = getBaseDir(args[0]) if( !base ) base = Paths.get('.') + // -- validate command line options if( profile && showAllProfiles ) { throw new AbortOperationException("Option `-profile` conflicts with option `-show-profiles`") } @@ -103,6 +105,7 @@ class CmdConfig extends CmdBase { if( printProperties ) outputFormat = 'properties' + // -- build the config final builder = new ConfigBuilder() .setShowClosures(true) .setStripSecrets(true) @@ -113,6 +116,10 @@ class CmdConfig extends CmdBase { final config = builder.buildConfigObject() + // -- validate config options + new ConfigValidator().validate(config) + + // -- print config options if( printValue ) { printValue0(config, printValue, stdout) } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index fca4596a9c..86dbac6282 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -37,6 +37,7 @@ import nextflow.NextflowMeta import nextflow.SysEnv import nextflow.config.ConfigBuilder import nextflow.config.ConfigMap +import nextflow.config.ConfigValidator import nextflow.exception.AbortOperationException import nextflow.file.FileHelper import nextflow.plugin.Plugins @@ -334,13 +335,13 @@ class CmdRun extends CmdBase implements HubOptions { // check DSL syntax in the config launchInfo(config, scriptFile) - // check if NXF_ variables are set in nextflow.config - checkConfigEnv(config) - // -- load plugins final cfg = plugins ? [plugins: plugins.tokenize(',')] : config Plugins.load(cfg) + // -- validate config options + new ConfigValidator().validate(config) + // -- create a new runner instance final runner = new ScriptRunner(config) runner.setScript(scriptFile) @@ -407,17 +408,6 @@ class CmdRun extends CmdBase implements HubOptions { } } - protected checkConfigEnv(ConfigMap config) { - // Warn about setting NXF_ environment variables within env config scope - final env = config.env as Map - for( String name : env.keySet() ) { - if( name.startsWith('NXF_') && name!='NXF_DEBUG' ) { - final msg = "Nextflow variables must be defined in the launching environment - The following variable set in the config file is going to be ignored: '$name'" - log.warn(msg) - } - } - } - protected void launchInfo(ConfigMap config, ScriptFile scriptFile) { // -- determine strict mode detectStrictFeature(config, sysEnv) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigValidator.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigValidator.groovy new file mode 100644 index 0000000000..492028d2f3 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigValidator.groovy @@ -0,0 +1,92 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.config.dsl.ConfigSchema +import nextflow.config.dsl.ConfigScope +import nextflow.plugin.Plugins +/** + * Validate the Nextflow configuration + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class ConfigValidator { + + void validate(ConfigMap config) { + validate(config.toConfigObject()) + } + + void validate(ConfigObject config) { + final schema = getSchema() + final flatConfig = config.flatten() + for( String key : flatConfig.keySet() ) { + final names = key.tokenize('.') + if( names.first() == 'profiles' ) { + if( !names.isEmpty() ) names.remove(0) + if( !names.isEmpty() ) names.remove(0) + } + final scope = names.first() + if( scope == 'env' ) { + checkEnv(names.last()) + continue + } + if( scope == 'params' ) + continue + final fqName = names.join('.') + if( fqName.startsWith('process.ext.') ) + return + if( fqName !in schema ) { + log.warn "Unrecognized config option '${fqName}'" + continue + } + } + } + + protected Set getSchema() { + final schema = new HashSet() + schema.addAll(ConfigSchema.OPTIONS.keySet()) + for( final scope : Plugins.getExtensions(ConfigScope) ) + schema.addAll(ConfigSchema.getConfigOptions(scope).keySet()) + // hidden options added by ConfigBuilder + schema.addAll(List.of( + 'bucketDir', + 'cacheable', + 'dumpChannels', + 'libDir', + 'poolSize', + 'plugins', + 'preview', + 'runName', + 'stubRun', + )) + return schema + } + + /** + * Warn about setting `NXF_*` environment variables in the config. + * + * @param name + */ + protected void checkEnv(String name) { + if( name.startsWith('NXF_') && name!='NXF_DEBUG' ) + log.warn "Nextflow environment variables must be defined in the launch environment -- the following environment variable in the config will be ignored: `$name`" + } +} \ No newline at end of file