Validate config schema

docs/developer/plugins.md

@@ -98,7 +98,7 @@ class MyObserver implements TraceObserver {
 
     @Override
     void onFlowCreate(Session session) {
-        final message = session.config.navigate('myplugin.create.message')
+        final message = session.config.navigate('myplugin.createMessage')
         println message
     }
 }
@@ -108,16 +108,43 @@ You can then set this option in your config file:
 
 ```groovy
 // dot syntax
-myplugin.create.message = "I'm alive!"
+myplugin.createMessage = "I'm alive!"
 
-// closure syntax
+// block syntax
 myplugin {
-    create {
-        message = "I'm alive!"
+    createMessage = "I'm alive!"
+}
+```
+
+:::{versionadded} 25.02.0-edge
+:::
+
+Plugins can declare their config options by implementing the `ConfigScope` interface and declaring each config option as a field with the `@ConfigOption` annotation:
+
+```groovy
+import nextflow.config.dsl.ConfigOption
+import nextflow.config.dsl.ConfigScope
+import nextflow.config.dsl.ScopeName
+import nextflow.script.dsl.Description
+
+@ScopeName('myplugin')
+@Description('''
+    The `myplugin` scope allows you to configure the `nf-myplugin` plugin.
+''')
+class MyPluginConfig implements ConfigScope {
+
+    MyPluginConfig(Map opts) {
+        this.createMessage = opts.createMessage
     }
+
+    @ConfigOption
+    @Description('Message to print to standard output when a run is initialized.')
+    String createMessage
 }
 ```
 
+While this approach is not required to support plugin config options, it allows Nextflow to recognize plugin definitions when validating the config.
+
 ### Executors
 
 Plugins can define custom executors that can then be used with the `executor` process directive.

modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy

@@ -25,6 +25,7 @@ import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.config.ConfigBuilder
+import nextflow.config.ConfigValidator
 import nextflow.exception.AbortOperationException
 import nextflow.plugin.Plugins
 import nextflow.scm.AssetManager
@@ -81,6 +82,7 @@ class CmdConfig extends CmdBase {
         if( args ) base = getBaseDir(args[0])
         if( !base ) base = Paths.get('.')
 
+        // -- validate command line options
         if( profile && showAllProfiles ) {
             throw new AbortOperationException("Option `-profile` conflicts with option `-show-profiles`")
         }
@@ -103,6 +105,7 @@ class CmdConfig extends CmdBase {
         if( printProperties )
             outputFormat = 'properties'
 
+        // -- build the config
         final builder = new ConfigBuilder()
                 .setShowClosures(true)
                 .setStripSecrets(true)
@@ -113,6 +116,10 @@ class CmdConfig extends CmdBase {
 
         final config = builder.buildConfigObject()
 
+        // -- validate config options
+        new ConfigValidator().validate(config)
+
+        // -- print config options
         if( printValue ) {
             printValue0(config, printValue, stdout)
         }

modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy

@@ -37,6 +37,7 @@ import nextflow.NextflowMeta
 import nextflow.SysEnv
 import nextflow.config.ConfigBuilder
 import nextflow.config.ConfigMap
+import nextflow.config.ConfigValidator
 import nextflow.exception.AbortOperationException
 import nextflow.file.FileHelper
 import nextflow.plugin.Plugins
@@ -334,13 +335,13 @@ class CmdRun extends CmdBase implements HubOptions {
         // check DSL syntax in the config
         launchInfo(config, scriptFile)
 
-        // check if NXF_ variables are set in nextflow.config
-        checkConfigEnv(config)
-
         // -- load plugins
         final cfg = plugins ? [plugins: plugins.tokenize(',')] : config
         Plugins.load(cfg)
 
+        // -- validate config options
+        new ConfigValidator().validate(config)
+
         // -- create a new runner instance
         final runner = new ScriptRunner(config)
         runner.setScript(scriptFile)
@@ -407,17 +408,6 @@ class CmdRun extends CmdBase implements HubOptions {
         }
     }
 
-    protected checkConfigEnv(ConfigMap config) {
-        // Warn about setting NXF_ environment variables within env config scope
-        final env = config.env as Map<String, String>
-        for( String name : env.keySet() ) {
-            if( name.startsWith('NXF_') && name!='NXF_DEBUG' ) {
-                final msg = "Nextflow variables must be defined in the launching environment - The following variable set in the config file is going to be ignored: '$name'"
-                log.warn(msg)
-            }
-        }
-    }
-
     protected void launchInfo(ConfigMap config, ScriptFile scriptFile) {
         // -- determine strict mode
         detectStrictFeature(config, sysEnv)

modules/nextflow/src/main/groovy/nextflow/config/ConfigValidator.groovy

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.config
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.config.dsl.ConfigSchema
+import nextflow.config.dsl.ConfigScope
+import nextflow.config.dsl.SchemaNode
+import nextflow.config.dsl.ScopeName
+import nextflow.config.dsl.ScopeNode
+import nextflow.plugin.Plugins
+import nextflow.script.dsl.Description
+/**
+ * Validate the Nextflow configuration
+ *
+ * @author Ben Sherman <[email protected]>
+ */
+@Slf4j
+@CompileStatic
+class ConfigValidator {
+
+    /**
+     * Hidden options added by ConfigBuilder
+     */
+    private static final List<String> hiddenOptions = List.of(
+        'bucketDir',
+        'cacheable',
+        'dumpChannels',
+        'libDir',
+        'poolSize',
+        'plugins',
+        'preview',
+        'runName',
+        'stubRun',
+    );
+
+    /**
+     * Additional config scopes added by third-party plugins
+     */
+    private ScopeNode pluginScopes
+
+    ConfigValidator() {
+        loadPluginScopes()
+    }
+
+    private void loadPluginScopes() {
+        final scopes = new HashMap<String, SchemaNode>()
+        for( final scope : Plugins.getExtensions(ConfigScope) ) {
+            final clazz = scope.getClass()
+            final name = clazz.getAnnotation(ScopeName)?.value()
+            final description = clazz.getAnnotation(Description)?.value()
+            if( !name )
+                continue
+            if( name in scopes ) {
+                log.warn "Plugin config scope `${clazz.name}` conflicts with existing scope: `${name}`"
+                continue
+            }
+            scopes.put(name, ScopeNode.of(clazz, description))
+        }
+        pluginScopes = new ScopeNode('', [:], scopes)
+    }
+
+    void validate(ConfigMap config) {
+        validate(config.toConfigObject())
+    }
+
+    void validate(ConfigObject config) {
+        final flatConfig = config.flatten()
+        for( String key : flatConfig.keySet() ) {
+            final names = key.tokenize('.')
+            if( names.first() == 'profiles' ) {
+                if( !names.isEmpty() ) names.remove(0)
+                if( !names.isEmpty() ) names.remove(0)
+            }
+            final scope = names.first()
+            if( scope == 'env' ) {
+                checkEnv(names.last())
+                continue
+            }
+            if( scope == 'params' )
+                continue
+            final fqName = names.join('.')
+            if( fqName.startsWith('process.ext.') )
+                return
+            if( !isValid(names) ) {
+                log.warn "Unrecognized config option '${fqName}'"
+                continue
+            }
+        }
+    }
+
+    /**
+     * Determine whether a config option is defined in the schema.
+     *
+     * @param names
+     */
+    boolean isValid(List<String> names) {
+        if( names.size() == 1 && names.first() in hiddenOptions )
+            return true
+        if( ConfigSchema.ROOT.getOption(names) )
+            return true
+        if( pluginScopes.getOption(names) )
+            return true
+        return false
+    }
+
+    /**
+     * Warn about setting `NXF_*` environment variables in the config.
+     *
+     * @param name
+     */
+    private void checkEnv(String name) {
+        if( name.startsWith('NXF_') && name!='NXF_DEBUG' )
+            log.warn "Nextflow environment variables must be defined in the launch environment -- the following environment variable in the config will be ignored: `$name`"
+    }
+}

modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy

@@ -406,49 +406,6 @@ class CmdConfigTest extends Specification {
             .stripIndent()
     }
 
-    @IgnoreIf({System.getenv('NXF_SMOKE')})
-    def 'should resolve profiles into profiles config' () {
-        given:
-        def folder = Files.createTempDirectory('test')
-        def CONFIG = folder.resolve('nextflow.config')
-
-        CONFIG.text = '''
-            params {
-               foo = 'baz'
-            }
-
-            profiles {
-               test {
-                  params {
-                    foo = 'foo'
-                  }
-                  profiles {                      
-                      debug {
-                        cleanup = false
-                      }                    
-                  }
-               }
-            }        
-        '''
-
-        def buffer = new ByteArrayOutputStream()
-        // command definition
-        def cmd = new CmdConfig(showAllProfiles: true)
-        cmd.launcher = new Launcher(options: new CliOptions(config: [CONFIG.toString()]))
-        cmd.stdout = buffer
-        cmd.args = ['.']
-
-        when:
-        cmd.run()
-        def result = new ConfigSlurper().parse(buffer.toString())
-
-        then:
-        result.params.foo == 'baz'
-        result.profiles.test.params.foo == 'foo'
-        result.profiles.debug.cleanup == false
-    }
-
-
     def 'should remove secrets for config' () {
         given:
         SecretsLoader.instance.reset()

modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy

@@ -23,20 +23,15 @@ import nextflow.NextflowMeta
 import nextflow.SysEnv
 import nextflow.config.ConfigMap
 import nextflow.exception.AbortOperationException
-import org.junit.Rule
 import spock.lang.Specification
 import spock.lang.Unroll
-import test.OutputCapture
 
 /**
  *
  * @author Paolo Di Tommaso <[email protected]>
  */
 class CmdRunTest extends Specification {
 
-    @Rule
-    OutputCapture capture = new OutputCapture()
-
     @Unroll
     def 'should parse cmd param=#STR' () {
 
@@ -390,40 +385,6 @@ class CmdRunTest extends Specification {
         CmdRun.detectDslMode(new ConfigMap(), DSL2_SCRIPT, [:]) == '2'
     }
 
-    def 'should warn for invalid config vars' () {
-        given:
-        def ENV = [NXF_ANSI_SUMMARY: 'true']
-
-        when:
-        new CmdRun().checkConfigEnv(new ConfigMap([env:ENV]))
-
-        then:
-        def warning = capture
-                .toString()
-                .readLines()
-                .findResults { line -> line.contains('WARN') ? line : null }
-                .join('\n')
-        and:
-        warning.contains('Nextflow variables must be defined in the launching environment - The following variable set in the config file is going to be ignored: \'NXF_ANSI_SUMMARY\'')
-    }
-
-    def 'should not warn for valid config vars' () {
-        given:
-        def ENV = [FOO: '/something', NXF_DEBUG: 'true']
-
-        when:
-        new CmdRun().checkConfigEnv(new ConfigMap([env:ENV]))
-
-        then:
-        def warning = capture
-                .toString()
-                .readLines()
-                .findResults { line -> line.contains('WARN') ? line : null }
-                .join('\n')
-        and:
-        !warning
-    }
-
     @Unroll
     def 'should detect moduleBinaries' () {
         given:

tests/checks/profiles.nf/.expected_profile_advanced.txt

@@ -1,5 +1,4 @@
-echo = true
-foo = 'bar'
+outputDir = 'results'
 
 process {
    cpus = 8

tests/checks/profiles.nf/.expected_profile_all.txt

@@ -1,5 +1,4 @@
-echo = true
-foo = 'bar'
+outputDir = 'results'
 
 profiles {
    standard {

tests/checks/profiles.nf/.expected_profile_standard.txt

@@ -1,5 +1,4 @@
-echo = true
-foo = 'bar'
+outputDir = 'results'
 
 process {
    cpus = 2