Fix Prevent S3 global option when using custom endpoints (#5779)

Signed-off-by: Matthew Colpus <[email protected]> Co-authored-by: Matthew Colpus <[email protected]>
nextflow-io · Feb 24, 2025 · ed9da46 · ed9da46
1 parent 6a81c01
commit ed9da46
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 14 deletions.
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy
@@ -21,6 +21,7 @@ import static nextflow.processor.TaskProcessor.*
 import java.nio.file.Path
 
 import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
 import nextflow.Const
 import nextflow.ast.NextflowDSLImpl
 import nextflow.exception.AbortOperationException
@@ -40,6 +41,7 @@ import nextflow.util.MemoryUnit
  *
  * @author Paolo Di Tommaso <[email protected]>
  */
+@Slf4j
 @CompileStatic
 class TaskConfig extends LazyMap implements Cloneable {
 
@@ -392,10 +394,14 @@ class TaskConfig extends LazyMap implements Cloneable {
         for( String it : shell ) {
             if( !it )
                 throw new IllegalArgumentException("Directive `process.shell` cannot contain empty values - offending value: ${shell}")
-            if( !it || it.contains('\n') || it.contains('\r') )
-                throw new IllegalArgumentException("Directive `process.shell` cannot contain new-line characters - offending value: ${shell}")
-            if( it.startsWith(' ') || it.endsWith(' '))
-                throw new IllegalArgumentException("Directive `process.shell` cannot contain leading or tralining blanks - offending value: ${shell}")
+            if( !it || it.contains('\n') || it.contains('\r') ) {
+                log.warn1 "Directive `process.shell` cannot contain new-line characters - offending value: ${shell}"
+                break
+            }
+            if( it.startsWith(' ') || it.endsWith(' ')) {
+                log.warn "Directive `process.shell` cannot contain leading or tralining blanks - offending value: ${shell}"
+                break
+            }
         }
         return shell
     }

diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskConfigTest.groovy
@@ -684,14 +684,14 @@ class TaskConfigTest extends Specification {
         then:
         thrown(IllegalArgumentException)
 
-        when:
-        config.validateShell(['bash\nthis\nthat'])
-        then:
-        thrown(IllegalArgumentException)
-
-        when:
-        config.validateShell(['bash', ' -eu '])
-        then:
-        thrown(IllegalArgumentException)
+//        when:
+//        config.validateShell(['bash\nthis\nthat'])
+//        then:
+//        thrown(IllegalArgumentException)
+//
+//        when:
+//        config.validateShell(['bash', ' -eu '])
+//        then:
+//        thrown(IllegalArgumentException)
     }
 }
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java
@@ -835,7 +835,10 @@ protected S3FileSystem createFileSystem(URI uri, AwsConfig awsConfig) {
 		ClientConfiguration clientConfig = createClientConfig(props);
 
 		final String bucketName = S3Path.bucketName(uri);
-		final boolean global = bucketName!=null;
+        // do not use `global` flag for custom endpoint because
+        // when enabling that flag, it overrides S3 endpoints with AWS global endpoint
+        // see https://github.com/nextflow-io/nextflow/pull/5779
+		final boolean global = bucketName!=null && !awsConfig.getS3Config().isCustomEndpoint();
 		final AwsClientFactory factory = new AwsClientFactory(awsConfig, globalRegion(awsConfig));
 		client = new S3Client(factory.getS3Client(clientConfig, global));