[GR-48205] Improved Maven Testing; Migrate native-image configuration…

… for tools, espresso and sulong

PullRequest: graal/15339

compiler/src/jdk.internal.vm.compiler.test/src/org/graalvm/compiler/truffle/test/LazyClassLoadingTest.java

@@ -41,10 +41,14 @@
 import org.graalvm.compiler.core.GraalCompilerOptions;
 import org.graalvm.compiler.core.common.util.Util;
 import org.graalvm.compiler.debug.Assertions;
+import org.graalvm.compiler.hotspot.CommunityCompilerConfigurationFactory;
+import org.graalvm.compiler.hotspot.CompilerConfigurationFactory;
+import org.graalvm.compiler.hotspot.EconomyCompilerConfigurationFactory;
 import org.graalvm.compiler.nodes.Cancellable;
 import org.graalvm.compiler.options.OptionDescriptor;
 import org.graalvm.compiler.options.OptionDescriptors;
 import org.graalvm.compiler.options.OptionKey;
+import org.graalvm.compiler.options.OptionStability;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.options.OptionsParser;
 import org.graalvm.compiler.test.SubprocessUtil;
@@ -206,6 +210,14 @@ private List<String> filterGraalCompilerClasses(List<String> loadedGraalClassNam
             }
         }
 
+        // classes needed to find out whether enterprise is installed in the JDK
+        allowList.add(OptionStability.class);
+        allowList.add(CompilerConfigurationFactory.class);
+        allowList.add(CompilerConfigurationFactory.Options.class);
+        allowList.add(CompilerConfigurationFactory.ShowConfigurationLevel.class);
+        allowList.add(EconomyCompilerConfigurationFactory.class);
+        allowList.add(CommunityCompilerConfigurationFactory.class);
+
         allowList.add(Cancellable.class);
 
         List<String> forbiddenClasses = new ArrayList<>();

compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java

@@ -60,13 +60,13 @@
  */
 public abstract class CompilerConfigurationFactory implements Comparable<CompilerConfigurationFactory> {
 
-    enum ShowConfigurationLevel {
+    public enum ShowConfigurationLevel {
         none,
         info,
         verbose
     }
 
-    static class Options {
+    public static class Options {
         // @formatter:off
         @Option(help = "Names the compiler configuration to use. If omitted, the compiler configuration " +
                        "with the highest auto-selection priority is used. To see the set of available configurations, " +

sdk/mx.sdk/mx_sdk.py

@@ -277,9 +277,12 @@ def maven_deploy_public_repo_dir():
     return os.path.join(_suite.get_mx_output_dir(), 'public-maven-repo')
 
 @mx.command(_suite.name, 'maven-deploy-public')
-def maven_deploy_public(args, licenses=None):
+def maven_deploy_public(args, licenses=None, deploy_snapshots=True):
     """Helper to simplify deploying all public Maven dependendencies into the mxbuild directory"""
-    artifact_version = mx_sdk_vm_impl.graalvm_version('graalvm')
+    if deploy_snapshots:
+        artifact_version = f'{mx_sdk_vm_impl.graalvm_version("graalvm")}-SNAPSHOT'
+    else:
+        artifact_version = f'{mx_sdk_vm_impl.graalvm_version("graalvm")}'
     path = maven_deploy_public_repo_dir()
     mx.rmtree(path, ignore_errors=True)
     os.mkdir(path)

sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java

@@ -1632,6 +1632,11 @@ private static AbstractPolyglotImpl loadAndValidateProviders(Iterator<? extends
         Collections.sort(impls, Comparator.comparing(AbstractPolyglotImpl::getPriority));
         AbstractPolyglotImpl prev = null;
         for (AbstractPolyglotImpl impl : impls) {
+            if (impl.getPriority() == Integer.MIN_VALUE) {
+                // disabled
+                continue;
+            }
+
             impl.setNext(prev);
             try {
                 impl.setConstructors(APIAccessImpl.INSTANCE);
@@ -1703,6 +1708,9 @@ private Iterator<? extends AbstractPolyglotImpl> searchServiceLoader() throws In
     private static class ClassPathIsolation {
 
         private static final String TRUFFLE_MODULE_NAME = "org.graalvm.truffle";
+        private static final String TRUFFLE_RUNTIME_MODULE_NAME = "org.graalvm.truffle.runtime";
+        private static final String JVMCI_MODULE_NAME = "jdk.internal.vm.ci";
+        private static final String TRUFFLE_ENTERPRISE_MODULE_NAME = "com.oracle.truffle.enterprise";
         private static final String POLYGLOT_MODULE_NAME = "org.graalvm.polyglot";
         private static final String OPTION_DISABLE_CLASS_PATH_ISOLATION = "polyglotimpl.DisableClassPathIsolation";
         private static final String OPTION_TRACE_CLASS_PATH_ISOLATION = "polyglotimpl.TraceClassPathIsolation";
@@ -1781,7 +1789,6 @@ static Module createIsolatedTruffleModule(Class<?> polyglotClass) {
                     trace("Class-path entry: %s", p);
                 }
             }
-
             List<Path> relevantPaths = filterClasspath(parent, classpath);
             if (relevantPaths == null) {
                 if (TRACE_CLASS_PATH_ISOLATION) {
@@ -1864,6 +1871,10 @@ record ParsedModule(Path p, Set<ModuleReference> modules) {
                     Set<ModuleReference> modules = finder.findAll();
                     if (modules.size() > 0) {
                         parsedModules.add(new ParsedModule(path, modules));
+                    } else {
+                        if (TRACE_CLASS_PATH_ISOLATION) {
+                            trace("No modules found in class-path entry %s", path);
+                        }
                     }
                 } catch (FindException t) {
                     // error in module finding -> not a valid module descriptor ignore
@@ -1897,6 +1908,16 @@ record ParsedModule(Path p, Set<ModuleReference> modules) {
                 return null;
             }
 
+            Set<String> excludedModules;
+            if (ModuleLayer.boot().findModule(JVMCI_MODULE_NAME).isPresent()) {
+                excludedModules = Set.of();
+            } else {
+                /*
+                 * if jdk.internal.vm.ci is not enabled we can't load these modules on the layer.
+                 */
+                excludedModules = Set.of(TRUFFLE_RUNTIME_MODULE_NAME, TRUFFLE_ENTERPRISE_MODULE_NAME);
+            }
+
             // now iteratively resolve modules until no more modules are included
             List<ParsedModule> toProcess = new ArrayList<>(parsedModules);
             Set<String> usedServices = new HashSet<>();
@@ -1909,6 +1930,11 @@ record ParsedModule(Path p, Set<ModuleReference> modules) {
                     ParsedModule module = modules.next();
                     for (ModuleReference m : module.modules) {
                         ModuleDescriptor d = m.descriptor();
+                        if (excludedModules.contains(d.name())) {
+                            modules.remove();
+                            continue;
+                        }
+
                         for (Provides p : d.provides()) {
                             if (usedServices.contains(p.service())) {
                                 if (includedModules.add(d.name())) {
@@ -1922,7 +1948,15 @@ record ParsedModule(Path p, Set<ModuleReference> modules) {
                         if (includedModules.contains(d.name())) {
                             usedServices.addAll(d.uses());
                             for (Requires r : d.requires()) {
-                                if (!r.modifiers().contains(Requires.Modifier.STATIC) && includedModules.add(r.name())) {
+                                /*
+                                 * We deliberately follow static resources here, even though they
+                                 * are not real dependencies in the module-graph. If we don't follow
+                                 * static requires we might fallback to the parent class-loader for
+                                 * these classes causing weird problems. So if the module is on the
+                                 * class-path and there is a static requires we pick it up into the
+                                 * module-graph well.
+                                 */
+                                if (includedModules.add(r.name())) {
                                     if (TRACE_CLASS_PATH_ISOLATION) {
                                         trace("Include module '%s' because it is required by '%s'.", r.name(), d.name());
                                     }
@@ -1940,6 +1974,9 @@ record ParsedModule(Path p, Set<ModuleReference> modules) {
                 for (ModuleReference ref : module.modules()) {
                     String name = ref.descriptor().name();
                     if (!includedModules.contains(name)) {
+                        if (TRACE_CLASS_PATH_ISOLATION) {
+                            trace("Filter module '%s' because not reachable on the module graph.", name);
+                        }
                         continue;
                     }
 
@@ -1950,10 +1987,8 @@ record ParsedModule(Path p, Set<ModuleReference> modules) {
                         continue;
                     }
 
-                    if (includedModules.contains(name)) {
-                        filteredList.add(module.p());
-                        break;
-                    }
+                    filteredList.add(module.p());
+                    break;
                 }
             }
 
@@ -2008,7 +2043,8 @@ private static class PolyglotInvalid extends AbstractPolyglotImpl {
 
         @Override
         public int getPriority() {
-            return Integer.MIN_VALUE;
+            // make sure polyglot invalid has lowest priority but is not filtered (hence + 1)
+            return Integer.MIN_VALUE + 1;
         }
 
         @Override

sulong/projects/com.oracle.truffle.llvm.parser/src/com/oracle/truffle/llvm/parser/coff/WindowsLibraryLocator.java

@@ -42,7 +42,16 @@ public final class WindowsLibraryLocator extends LibraryLocator {
     Path libraryPath;
 
     public WindowsLibraryLocator(Source source) {
-        libraryPath = source != null ? Paths.get(source.getPath()).getParent() : null;
+        if (source == null || source.isInternal()) {
+            /*
+             * No need to search the current file directory for internal sources. Those can always
+             * be located anyway. Also Paths.get doesn't work if the source path is an internal
+             * resources inside a jar.
+             */
+            libraryPath = null;
+        } else {
+            libraryPath = Paths.get(source.getPath()).getParent();
+        }
     }
 
     @Override

tools/mx.tools/tools-chromeinspector.properties

@@ -1,4 +1 @@
 Requires = tool:profiler language:truffle-json
-Args = --initialize-at-build-time=com.oracle.truffle.tools.chromeinspector \
-       -H:MaxRuntimeCompileMethods=300
-

tools/mx.tools/tools-coverage.properties

@@ -1,3 +1 @@
 Requires = macro:truffle language:truffle-json
-Args = --initialize-at-build-time=com.oracle.truffle.tools.coverage \
-       -H:MaxRuntimeCompileMethods=100

tools/mx.tools/tools-dap.properties

@@ -1,3 +1 @@
 Requires = macro:truffle language:truffle-json
-Args = --initialize-at-build-time=com.oracle.truffle.tools.dap \
-       -H:MaxRuntimeCompileMethods=100

tools/mx.tools/tools-insight.properties

@@ -1,3 +0,0 @@
-Args = --initialize-at-build-time=com.oracle.truffle.tools.agentscript \
-       --initialize-at-build-time=org.graalvm.tools.insight \
-       -H:MaxRuntimeCompileMethods=300

tools/mx.tools/tools-profiler.properties

@@ -1,3 +1 @@
 Requires = macro:truffle language:truffle-json
-Args = --initialize-at-build-time=com.oracle.truffle.tools.profiler \
-       -H:MaxRuntimeCompileMethods=100

tools/src/com.oracle.truffle.tools.chromeinspector/src/META-INF/native-image/com.oracle.truffle.tools.chromeinspector/native-image.properties

@@ -0,0 +1,3 @@
+Args = --initialize-at-build-time=com.oracle.truffle.tools.chromeinspector \
+       -H:MaxRuntimeCompileMethods=300
+

tools/src/com.oracle.truffle.tools.coverage/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties

@@ -0,0 +1,2 @@
+Args = --initialize-at-build-time=com.oracle.truffle.tools.profiler \
+       -H:MaxRuntimeCompileMethods=100

tools/src/com.oracle.truffle.tools.dap/src/META-INF/native-image/com.oracle.truffle.tools.dap/native-image.properties

@@ -0,0 +1,2 @@
+Args = --initialize-at-build-time=com.oracle.truffle.tools.dap \
+       -H:MaxRuntimeCompileMethods=100

tools/src/com.oracle.truffle.tools.profiler/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties

@@ -0,0 +1,2 @@
+Args = --initialize-at-build-time=com.oracle.truffle.tools.profiler \
+       -H:MaxRuntimeCompileMethods=100

tools/src/org.graalvm.tools.insight.heap/src/META-INF/native-image/org.graalvm.tools.insight.heap/native-image.properties

@@ -0,0 +1,2 @@
+Args = --initialize-at-build-time=org.graalvm.tools.insight.heap \
+       -H:MaxRuntimeCompileMethods=100

tools/src/org.graalvm.tools.insight/src/META-INF/native-image/org.graalvm.tools.insight/native-image.properties

@@ -0,0 +1,3 @@
+Args = --initialize-at-build-time=com.oracle.truffle.tools.agentscript \
+       --initialize-at-build-time=org.graalvm.tools.insight \
+       -H:MaxRuntimeCompileMethods=300

tools/src/org.graalvm.tools.lsp/src/META-INF/native-image/org.graalvm.tools.lsp/native-image.properties

@@ -0,0 +1,2 @@
+Args = --initialize-at-build-time=org.graalvm.tools.lsp,org.graalvm.tools.api.lsp \
+       -H:MaxRuntimeCompileMethods=100

truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/Truffle.java

@@ -144,7 +144,14 @@ private static TruffleRuntime createRuntime() throws InternalError {
                 return runtime;
             }
         }
-        return new DefaultTruffleRuntime("No optimizing Truffle runtime found on the module-path.");
+
+        String reason;
+        if (ModuleLayer.boot().findModule("jdk.internal.vm.ci").isPresent()) {
+            reason = "No optimizing Truffle runtime found on the module or class-path.";
+        } else {
+            reason = "JVMCI is required to enable optimizations. Pass -XX:+EnableJVMCI as a virtual machine argument to the java executable to resolve this.";
+        }
+        return new DefaultTruffleRuntime(reason);
     }
 
     private static void maybeExportJVMCITo(Class<?> runtimeClass) throws ReflectiveOperationException {

truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java

@@ -303,7 +303,11 @@ private static Path findCacheRootOnNativeImage() {
         if (res == null) {
             assert ImageInfo.inImageRuntimeCode() : "Can be called only in the native-image execution time.";
             Path executable = Path.of(ProcessProperties.getExecutableName());
-            Path cache = executable.resolve("resources");
+            Path parent = executable.getParent();
+            if (parent == null) {
+                throw new IllegalStateException("Could not locate native-image resources.");
+            }
+            Path cache = parent.resolve("resources");
             res = Pair.create(cache, false);
             cacheRoot = res;
         }

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/OptimizedTruffleRuntime.java

@@ -262,7 +262,7 @@ public final Iterable<Class<?>> getLookupTypes() {
      * the {@link TruffleCompiler} with {@link #getTruffleCompiler(TruffleCompilable)
      * getTruffleCompiler}.
      */
-    protected final String getCompilerConfigurationName() {
+    public final String getCompilerConfigurationName() {
         return compilationSupport.getCompilerConfigurationName(this);
     }
 

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/HotSpotTruffleRuntimeAccess.java

@@ -79,7 +79,8 @@ protected static TruffleRuntime createRuntime() {
         } catch (Error e) {
             // hack to detect the exact error that is thrown when
             if (e.getClass() == Error.class && e.getMessage().startsWith("The EnableJVMCI VM option must be true")) {
-                return new DefaultTruffleRuntime("JVMCI is not enabled on this JVM. JVMCI may be enabled using -XX:+EnableJVMCI. This is necessary on JVMs that do not enable JVMCI by default.");
+                return new DefaultTruffleRuntime(
+                                "JVMCI is required to enable optimizations. Pass -XX:+EnableJVMCI as a virtual machine argument to the java executable to resolve this. This is necessary on JVMs that do not enable JVMCI by default.");
             }
             throw e;
         }
@@ -88,7 +89,7 @@ protected static TruffleRuntime createRuntime() {
         boolean useCompiler = config.getFlag("UseCompiler", Boolean.class);
         if (!useCompiler) {
             // compilation disabled in host VM -> fallback to default runtime
-            return new DefaultTruffleRuntime("JVMCI compilation was disabled on this JVM. JVMCI may be enabled using -XX:+EnableJVMCI.");
+            return new DefaultTruffleRuntime("JVMCI compilation was disabled on this JVM. Pass -XX:+EnableJVMCI as a virtual machine argument to the java executable to resolve this.");
         }
         /*
          * Module integrity rules ignore qualified exports from a parent module layer to a child
@@ -117,7 +118,7 @@ protected static TruffleRuntime createRuntime() {
                 if (compilerModule == null) {
                     // jargraal compiler module not found -> fallback to default runtime
                     return new DefaultTruffleRuntime(
-                                    "Libgraal compilation is not available on this JVM. Alternatively, the compiler module jdk.internal.vm.compiler was not found on the --upgrade-module-path.");
+                                    "Libgraal compilation is not available on this JVM. Alternatively, the org.graalvm.compiler:compiler module can be put on the --upgrade-module-path.");
                 }
                 /*
                  * That the compiler has a qualified export to Truffle may not be enough if truffle

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/libgraal/LibGraalObject.java

@@ -53,12 +53,17 @@ public class LibGraalObject {
     static {
         if (LibGraal.isAvailable()) {
             LibGraal.registerNativeMethods(LibGraalObject.class);
-            Runtime.getRuntime().addShutdownHook(new Thread() {
-                @Override
-                public void run() {
-                    exiting = true;
-                }
-            });
+            try {
+                Runtime.getRuntime().addShutdownHook(new Thread() {
+                    @Override
+                    public void run() {
+                        exiting = true;
+                    }
+                });
+            } catch (IllegalStateException e) {
+                // shutdown already in progress
+                // catching the exception is the only way to detect this.
+            }
         }
     }
 

vm/ci/ci_common/common.jsonnet

@@ -359,6 +359,10 @@ local devkits = graal_common.devkits;
   weekly_vm_windows: self.vm_windows + {
     targets+: ['weekly'],
   },
+
+  weekly_vm_windows_amd64: self.vm_windows + {
+    targets+: ['weekly'],
+  },
 
   weekly_vm_windows_jdk17: self.vm_windows_jdk17 + {
     targets+: ['weekly'],

vm/mx.vm/mx_vm_gate.py

@@ -781,7 +781,7 @@ def gate_maven_downloader(tasks):
     with Task('Maven Downloader prepare maven repo', tasks, tags=[VmGateTasks.maven_downloader]) as t:
         if t:
             mx.suite('sulong')
-            mx_sdk.maven_deploy_public([], licenses=['EPL-2.0', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-new', 'UPL', 'MIT'])
+            mx_sdk.maven_deploy_public([], licenses=['EPL-2.0', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-new', 'UPL', 'MIT'], deploy_snapshots=False)
             mx.build(["--dep", "MAVEN_DOWNLOADER"])
             jdk = mx.get_jdk()
             mvnDownloader = mx.distribution("MAVEN_DOWNLOADER")

vm/mx.vm/suite.py

@@ -65,7 +65,7 @@
             },
             {
                 "name": "graalpython",
-                "version": "f42da6d958171905ddc00e26bd7b0ce52dca732a",
+                "version": "cbb9985782bf0f68969ae6f2eb6101fd02eb4db4",
                 "dynamic": True,
                 "urls": [
                     {"url": "https://github.com/graalvm/graalpython.git", "kind": "git"},