From b912cd9f838493214679bbecd6436184c7177ad2 Mon Sep 17 00:00:00 2001 From: Loic Ottet Date: Mon, 28 Aug 2023 15:32:16 +0200 Subject: [PATCH 01/29] Fix SVM tests --- .../src/com/oracle/svm/hosted/annotation/AnnotationFeature.java | 1 + 1 file changed, 1 insertion(+) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/annotation/AnnotationFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/annotation/AnnotationFeature.java index 420406307d08..1bd3a09661ac 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/annotation/AnnotationFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/annotation/AnnotationFeature.java @@ -59,6 +59,7 @@ private Object registerDeclaredMethods(Object obj) { Class annotationType = annotation.annotationType(); if (processedTypes.add(annotationType)) { RuntimeReflection.registerAllDeclaredMethods(annotationType); + RuntimeReflection.register(annotationType.getDeclaredMethods()); } } return obj; From 27716d7cf3131f5d8d0d6cbdd069ce747aab2a09 Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Tue, 22 Aug 2023 19:59:25 +0200 Subject: [PATCH 02/29] Internal resource for polyglot isolates. --- sdk/mx.sdk/mx_sdk_vm.py | 3 +- sdk/mx.sdk/mx_sdk_vm_impl.py | 34 ++++++- truffle/mx.truffle/mx_truffle.py | 91 +++++++++++++++++++ .../com/oracle/truffle/api/impl/Accessor.java | 5 +- .../api/impl/DefaultRuntimeAccessor.java | 8 -- .../truffle/polyglot/EngineAccessor.java | 30 +++++- .../polyglot/InternalResourceCache.java | 47 +++++++--- .../runtime/LibTruffleAttachResource.java | 6 +- .../truffle/runtime/ModulesSupport.java | 2 +- .../runtime/OptimizedRuntimeSupport.java | 8 -- 10 files changed, 190 insertions(+), 44 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm.py b/sdk/mx.sdk/mx_sdk_vm.py index 97b5546ea4c2..beeaecf335df 100644 --- a/sdk/mx.sdk/mx_sdk_vm.py +++ b/sdk/mx.sdk/mx_sdk_vm.py @@ -197,7 +197,7 @@ def __init__(self, destination, jar_distributions, build_args, jvm_library=False class LanguageLibraryConfig(LibraryConfig): - def __init__(self, jar_distributions, build_args, language, main_class=None, is_sdk_launcher=True, launchers=None, option_vars=None, default_vm_args=None, headers=False, set_default_relative_home_path=True, **kwargs): + def __init__(self, jar_distributions, build_args, language, main_class=None, is_sdk_launcher=True, launchers=None, option_vars=None, default_vm_args=None, headers=False, set_default_relative_home_path=True, isolate_library_layout_distribution=None, **kwargs): """ :param str language :param str main_class @@ -219,6 +219,7 @@ def __init__(self, jar_distributions, build_args, language, main_class=None, is_ if set_default_relative_home_path: # Ensure the language launcher can always find the language home self.add_relative_home_path(language, relpath('.', dirname(self.destination))) + self.isolate_library_layout_distribution = isolate_library_layout_distribution class GraalVmComponent(object): def __init__(self, diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 309699e45be0..9a692d1d0a03 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -3485,6 +3485,7 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi else: with_non_rebuildable_configs = True for library_config in _get_library_configs(component): + library_project = None if with_svm: library_project = GraalVmLibrary(component, GraalVmNativeImage.project_name(library_config), [], library_config) register_project(library_project) @@ -3496,11 +3497,34 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi jmod_file = library_config.add_to_module + ('' if library_config.add_to_module.endswith('.jmod') else '.jmod') modified_jmods.setdefault(jmod_file, []).append(library_project) needs_stage1 = True # library configs need a stage1 even when they are skipped - if isinstance(library_config, mx_sdk.LanguageLibraryConfig) and library_config.launchers: - launcher_project = NativeLibraryLauncherProject(component, library_config) - register_project(launcher_project) - polyglot_config_project = PolyglotConfig(component, library_config) - register_project(polyglot_config_project) + if isinstance(library_config, mx_sdk.LanguageLibraryConfig): + if library_config.launchers: + launcher_project = NativeLibraryLauncherProject(component, library_config) + register_project(launcher_project) + polyglot_config_project = PolyglotConfig(component, library_config) + register_project(polyglot_config_project) + if with_svm and library_config.isolate_library_layout_distribution and not library_project.is_skipped() and has_component('tfle', stage1=True): + # Create a layout distribution with the resulting language library that can be consumed into the + # isolate resources jar distribution, + resource_base_folder = f'META-INF/resources/engine/{library_config.language}-isolate///libvm' + attrs = { + 'description': f'Contains {library_config.language} language library resources', + 'hashEntry': f'{resource_base_folder}/sha256', + 'fileListEntry': f'{resource_base_folder}/files', + 'maven': False, + } + register_distribution(mx.LayoutDirDistribution( + suite=_suite, + name=library_config.isolate_library_layout_distribution, + deps=[], + layout={ + f'{resource_base_folder}/': f'dependency:{library_project.name}' + }, + path=None, + platformDependent=True, + theLicense=None, + **attrs + )) if isinstance(component, mx_sdk.GraalVmLanguage) and component.support_distributions: ni_resources_components = dir_name_to_ni_resources_components.get(component.dir_name) if not ni_resources_components: diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index 31a240671eb5..9d87bce45520 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -56,6 +56,8 @@ import mx_sdk import mx_sdk_vm import mx_unittest +import mx_jardistribution +import mx_pomdistribution import tck from mx_gate import Task from mx_javamodules import as_java_module, get_module_name @@ -905,6 +907,95 @@ def read_file(path): f"{diff}{nl}" + "Make sure the grammar files are up to date with the generated code. You can regenerate the generated code using mx.") + +def register_polyglot_isolate_distributions(register_distribution, language_id, language_distribution, isolate_library_layout_distribution, internal_resource_project): + """ + Registers the polyglot isolate resource distribution and isolate resource meta-POM distribution. + The created polyglot isolate resource distribution is named `_ISOLATE_RESOURCES`, inheriting the Maven group ID + from the given `language_distribution`, and the Maven artifact ID is `-isolate`. + The meta-POM distribution is named `_ISOLATE`, having the Maven group ID `org.graalvm.polyglot`, + and the Maven artifact ID is `-isolate`. + + :param register_distribution: A callback to dynamically register the distribution, obtained as a parameter from `mx_register_dynamic_suite_constituents`. + :type register_distribution: (mx.Distribution) -> None + :param language_id: The language ID. + :param language_distribution: The language distribution used to inherit distribution properties. + :param isolate_library_layout_distribution: The layout distribution with polyglot isolate library. + :param internal_resource_project: The internal resource project used for unpacking the polyglot isolate library. + """ + assert language_distribution + assert isolate_library_layout_distribution + assert internal_resource_project + owner_suite = language_distribution.suite + resources_dist_name = f'{language_id.upper()}_ISOLATE_RESOURCES' + isolate_dist_name = f'{language_id.upper()}_ISOLATE' + layout_dist_qualified_name = f'{isolate_library_layout_distribution.suite.name}:{isolate_library_layout_distribution.name}' + maven_group_id = language_distribution.maven_group_id() + maven_artifact_id = f'{language_id}-isolate' + module_name = f'{get_module_name(language_distribution)}.isolate' + licenses = set() + licenses.update(language_distribution.theLicense) + licenses.update(owner_suite.defaultLicense) + attrs = { + 'description': f'Polyglot isolate resources for {language_id}.', + 'moduleInfo': { + 'name': module_name, + }, + 'maven': { + 'groupId': maven_group_id, + 'artifactId': maven_artifact_id, + 'tag': ['default', 'public'], + }, + 'mavenNoJavadoc': True, + 'mavenNoSources': True, + } + isolate_library_dist = mx_jardistribution.JARDistribution( + suite=owner_suite, + name=resources_dist_name, + subDir=language_distribution.subDir, + path=None, + sourcesPath=None, + deps=[ + internal_resource_project.name, + layout_dist_qualified_name, + ], + mainClass=None, + excludedLibs=[], + distDependencies=['truffle:TRUFFLE_API'], + javaCompliance=str(internal_resource_project.javaCompliance)+'+', + platformDependent=True, + theLicense=list(licenses), + compress=True, + **attrs + ) + register_distribution(isolate_library_dist) + attrs = { + 'description': f'The {language_id} polyglot isolate.', + 'maven': { + 'groupId': 'org.graalvm.polyglot', + 'artifactId': maven_artifact_id, + 'tag': ['default', 'public'], + }, + } + # the graal-enterprise may not be fully loaded + # we need to use suiteDict to get the license + graal_enterprise = mx.suite('graal-enterprise') + graal_enterprise_license = graal_enterprise.suiteDict.get('defaultLicense') + if isinstance(graal_enterprise_license, str): + graal_enterprise_license = [graal_enterprise_license] + licenses.update(graal_enterprise_license) + meta_pom_dist = mx_pomdistribution.POMDistribution( + suite=owner_suite, + name=isolate_dist_name, + distDependencies=[], + runtimeDependencies=[ + resources_dist_name, + 'graal-enterprise:TRUFFLE_ENTERPRISE', + ], + theLicense=sorted(list(licenses)), + **attrs) + register_distribution(meta_pom_dist) + class LibffiBuilderProject(mx.AbstractNativeProject, mx_native.NativeDependency): # pylint: disable=too-many-ancestors """Project for building libffi from source. diff --git a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/Accessor.java b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/Accessor.java index e6127833869a..181c9add1d9e 100644 --- a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/Accessor.java +++ b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/Accessor.java @@ -753,6 +753,8 @@ public abstract Iterator mergeHostGuestFrames(Object polyglotEngine, S public abstract TruffleFile getInternalResource(Object owner, Class resourceType) throws IOException; public abstract TruffleFile getInternalResource(Object owner, String resourceId) throws IOException; + + public abstract Collection getResourceIds(String componentId); } public abstract static class LanguageSupport extends Support { @@ -1232,9 +1234,6 @@ public AbstractFastThreadLocal getContextThreadLocal() { public abstract boolean isLegacyCompilerOption(String key); public abstract ThreadLocal createTerminatingThreadLocal(Supplier initialValue, Consumer onThreadTermination); - - public abstract Collection getInternalResources(); - } public abstract static class LanguageProviderSupport extends Support { diff --git a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/DefaultRuntimeAccessor.java b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/DefaultRuntimeAccessor.java index afafdfa43d9f..097c43aeba48 100644 --- a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/DefaultRuntimeAccessor.java +++ b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/DefaultRuntimeAccessor.java @@ -40,13 +40,10 @@ */ package com.oracle.truffle.api.impl; -import java.util.Collection; -import java.util.List; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; -import com.oracle.truffle.api.InternalResource; import org.graalvm.options.OptionDescriptors; import org.graalvm.options.OptionValues; @@ -280,11 +277,6 @@ public AbstractFastThreadLocal getContextThreadLocal() { public ThreadLocal createTerminatingThreadLocal(Supplier initialValue, Consumer onThreadTermination) { return ThreadLocal.withInitial(initialValue); } - - @Override - public Collection getInternalResources() { - return List.of(); - } } } diff --git a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/EngineAccessor.java b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/EngineAccessor.java index df8a9799adbe..82bd35560852 100644 --- a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/EngineAccessor.java +++ b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/EngineAccessor.java @@ -2072,6 +2072,11 @@ private static TruffleFile getInternalResource(Object owner, String resourceId, resourceCache = cache.getResourceCache(resourceId); componentId = cache.getId(); supportedResourceIds = cache::getResourceIds; + } else if (owner instanceof PolyglotEngineImpl) { + cachedRoots = null; + resourceCache = InternalResourceCache.getEngineResource(resourceId); + componentId = PolyglotEngineImpl.ENGINE_ID; + supportedResourceIds = InternalResourceCache::getEngineResourceIds; } else { throw CompilerDirectives.shouldNotReachHere("Unsupported owner " + owner); } @@ -2083,16 +2088,35 @@ private static TruffleFile getInternalResource(Object owner, String resourceId, return null; } } - TruffleFile root = cachedRoots.get(resourceId); + TruffleFile root = cachedRoots != null ? cachedRoots.get(resourceId) : null; if (root == null) { PolyglotEngineImpl polyglotEngine = ((VMObject) owner).getEngine(); Object fsContext = EngineAccessor.LANGUAGE.createFileSystemContext(polyglotEngine, resourceCache.getResourceFileSystem(polyglotEngine)); root = EngineAccessor.LANGUAGE.getTruffleFile(".", fsContext); - var prevValue = cachedRoots.putIfAbsent(resourceId, root); - root = prevValue != null ? prevValue : root; + if (cachedRoots != null) { + var prevValue = cachedRoots.putIfAbsent(resourceId, root); + root = prevValue != null ? prevValue : root; + } } return root; } + + @Override + public Collection getResourceIds(String componentId) { + if (PolyglotEngineImpl.ENGINE_ID.equals(componentId)) { + return InternalResourceCache.getEngineResourceIds(); + } + LanguageCache languageCache = LanguageCache.languages().get(componentId); + if (languageCache != null) { + return languageCache.getResourceIds(); + } + for (InstrumentCache instrumentCache : InstrumentCache.load()) { + if (instrumentCache.getId().equals(componentId)) { + return instrumentCache.getResourceIds(); + } + } + throw new IllegalArgumentException(componentId); + } } abstract static class AbstractClassLoaderSupplier implements Supplier { diff --git a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java index 3227dd0abe04..0d496e68637b 100644 --- a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java +++ b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java @@ -395,9 +395,10 @@ static boolean copyResourcesForNativeImage(Path target, String... components) th result |= cache.copyResourcesForNativeImage(target); } } - // Always install Truffle runtime resource caches - for (InternalResource resource : EngineAccessor.RUNTIME.getInternalResources()) { - result |= createRuntimeResourceCache(resource).copyResourcesForNativeImage(target); + // Always install engine resources + for (String resourceId : getEngineResourceIds()) { + InternalResourceCache cache = getEngineResource(resourceId); + result |= cache.copyResourcesForNativeImage(target); } return result; } @@ -417,6 +418,17 @@ private boolean copyResourcesForNativeImage(Path target) throws IOException { } } + static Collection getEngineResourceIds() { + Map> engineResources = loadOptionalInternalResources(EngineAccessor.locatorOrDefaultLoaders()).get(PolyglotEngineImpl.ENGINE_ID); + return engineResources != null ? engineResources.keySet() : List.of(); + } + + static InternalResourceCache getEngineResource(String resourceId) { + Map> engineResources = loadOptionalInternalResources(EngineAccessor.locatorOrDefaultLoaders()).get(PolyglotEngineImpl.ENGINE_ID); + Supplier resourceSupplier = engineResources != null ? engineResources.get(resourceId) : null; + return resourceSupplier != null ? resourceSupplier.get() : null; + } + static Map>> loadOptionalInternalResources(List suppliers) { if (TruffleOptions.AOT) { assert nativeImageCache != null; @@ -455,7 +467,7 @@ private static Map>> collect } private static boolean hasSameCodeSource(OptionalResourceSupplier first, OptionalResourceSupplier second) { - return first.optionalResource.getClass() == second.optionalResource.getClass(); + return first.optionalResourceProvider.getClass() == second.optionalResourceProvider.getClass(); } private static boolean isValidLoader(ClassLoader loader) { @@ -569,18 +581,31 @@ public Path get() { } } - private record OptionalResourceSupplier(InternalResourceProvider optionalResource) implements Supplier { + private static final class OptionalResourceSupplier implements Supplier { + private final InternalResourceProvider optionalResourceProvider; + private volatile InternalResourceCache cachedResource; - private OptionalResourceSupplier { - Objects.requireNonNull(optionalResource, "OptionalResource must be non null"); + private OptionalResourceSupplier(InternalResourceProvider optionalResourceProvider) { + Objects.requireNonNull(optionalResourceProvider, "OptionalResourceProvider must be non null"); + this.optionalResourceProvider = optionalResourceProvider; } @Override public InternalResourceCache get() { - return new InternalResourceCache( - EngineAccessor.LANGUAGE_PROVIDER.getInternalResourceComponentId(optionalResource), - EngineAccessor.LANGUAGE_PROVIDER.getInternalResourceId(optionalResource), - () -> EngineAccessor.LANGUAGE_PROVIDER.createInternalResource(optionalResource)); + InternalResourceCache res = cachedResource; + if (res == null) { + synchronized (this) { + res = cachedResource; + if (res == null) { + res = new InternalResourceCache( + EngineAccessor.LANGUAGE_PROVIDER.getInternalResourceComponentId(optionalResourceProvider), + EngineAccessor.LANGUAGE_PROVIDER.getInternalResourceId(optionalResourceProvider), + () -> EngineAccessor.LANGUAGE_PROVIDER.createInternalResource(optionalResourceProvider)); + cachedResource = res; + } + } + } + return res; } } } diff --git a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/LibTruffleAttachResource.java b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/LibTruffleAttachResource.java index 4f19eb8782fe..6d471d8f8972 100644 --- a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/LibTruffleAttachResource.java +++ b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/LibTruffleAttachResource.java @@ -46,14 +46,12 @@ import java.io.IOException; import java.nio.file.Path; -@InternalResource.Id(LibTruffleAttachResource.ID) +@InternalResource.Id(value = LibTruffleAttachResource.ID, componentId = "engine", optional = true) final class LibTruffleAttachResource implements InternalResource { static final String ID = "libtruffleattach"; - static final LibTruffleAttachResource INSTANCE = new LibTruffleAttachResource(); - - private LibTruffleAttachResource() { + LibTruffleAttachResource() { } @Override diff --git a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/ModulesSupport.java b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/ModulesSupport.java index af3331be62d1..d3a3c43849e7 100644 --- a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/ModulesSupport.java +++ b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/ModulesSupport.java @@ -137,7 +137,7 @@ private static boolean loadModulesSupportLibrary() { Class resourceCacheClass = Class.forName("com.oracle.truffle.polyglot.InternalResourceCache", false, ModulesSupport.class.getClassLoader()); Method installRuntimeResource = resourceCacheClass.getDeclaredMethod("installRuntimeResource", InternalResource.class); installRuntimeResource.setAccessible(true); - Path root = (Path) installRuntimeResource.invoke(null, LibTruffleAttachResource.INSTANCE); + Path root = (Path) installRuntimeResource.invoke(null, new LibTruffleAttachResource()); Path libAttach = root.resolve("bin").resolve(System.mapLibraryName("truffleattach")); attachLibPath = libAttach.toString(); } diff --git a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/OptimizedRuntimeSupport.java b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/OptimizedRuntimeSupport.java index 4a5de39ec6a2..dfdf4cd57417 100644 --- a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/OptimizedRuntimeSupport.java +++ b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/OptimizedRuntimeSupport.java @@ -40,13 +40,10 @@ */ package com.oracle.truffle.runtime; -import java.util.Collection; -import java.util.List; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; -import com.oracle.truffle.api.InternalResource; import org.graalvm.options.OptionDescriptors; import org.graalvm.options.OptionValues; @@ -376,9 +373,4 @@ public AbstractFastThreadLocal getContextThreadLocal() { public ThreadLocal createTerminatingThreadLocal(Supplier initialValue, Consumer onThreadTermination) { return OptimizedTruffleRuntime.createTerminatingThreadLocal(initialValue, onThreadTermination); } - - @Override - public Collection getInternalResources() { - return List.of(LibTruffleAttachResource.INSTANCE); - } } From a84f91ce1a286fc56cd6c9fba02feac416409e7d Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Wed, 30 Aug 2023 15:08:00 +0200 Subject: [PATCH 03/29] Remove all Sulong jdk17 jobs. --- sulong/ci/ci.jsonnet | 1 - 1 file changed, 1 deletion(-) diff --git a/sulong/ci/ci.jsonnet b/sulong/ci/ci.jsonnet index 22bba711fe37..b9a4c9c718d5 100644 --- a/sulong/ci/ci.jsonnet +++ b/sulong/ci/ci.jsonnet @@ -41,7 +41,6 @@ local sc = (import "ci_common/sulong-common.jsonnet"); sc.weekly + $.sulong + sc.labsjdk21 + sc.darwin_amd64 + sc.llvm4 + sc.gateTags(basicTags) + { name: "weekly-sulong-basic-nwcc-llvm-v40-jdk21-darwin-amd64", timelimit: "0:45:00" }, sc.gate + $.sulong + sc.labsjdk21 + sc.darwin_amd64 + sc.llvmBundled + sc.gateTags(basicTagsToolchain) + { name: "gate-sulong-basic-nwcc-llvm-toolchain-jdk21-darwin-amd64", timelimit: "0:45:00" }, - sc.gate + $.sulong + sc.labsjdk_ee_17 + sc.linux_amd64 + sc.llvmBundled + sc.requireGMP + sc.requireGCC + sc.gateTags(basicTagsToolchain) + { name: "gate-sulong-basic-nwcc-llvm-toolchain-jdk17-linux-amd64" }, sc.gate + $.sulong + sc.labsjdk21 + sc.linux_amd64 + sc.llvmBundled + sc.requireGMP + sc.requireGCC + sc.gateTags(basicTagsToolchain) + { name: "gate-sulong-basic-nwcc-llvm-toolchain-jdk21-linux-amd64" }, sc.gate + $.sulong + sc.labsjdk21 + sc.linux_aarch64 + sc.llvmBundled + sc.requireGMP + sc.gateTags(basicTagsNoNWCC) + { name: "gate-sulong-basic-llvm-jdk21-linux-aarch64", timelimit: "30:00" }, From efdb234880d147c2517194421ce390fb7b25cbe8 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Fri, 4 Aug 2023 16:41:20 +0200 Subject: [PATCH 04/29] Refactor use of AnalysisField.isValueAvailable(). --- .../graal/pointsto/heap/ImageHeapScanner.java | 4 +- .../ameta/AnalysisConstantFieldProvider.java | 2 +- .../AnalysisConstantReflectionProvider.java | 38 ++++++++++--------- .../svm/hosted/ameta/ReadableJavaField.java | 7 ++++ .../StaticFinalFieldFoldingNodePlugin.java | 2 +- .../svm/hosted/heap/SVMImageHeapScanner.java | 6 +-- .../svm/hosted/image/NativeImageHeap.java | 7 +++- .../meta/SharedConstantFieldProvider.java | 6 +-- 8 files changed, 40 insertions(+), 32 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java index 5980d4a99e62..752dba946779 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java @@ -477,8 +477,8 @@ protected void onObjectReachable(ImageHeapConstant imageHeapConstant, ScanReason } } - public boolean isValueAvailable(@SuppressWarnings("unused") AnalysisField field) { - return true; + public boolean isValueAvailable(AnalysisField field) { + return field.isValueAvailable(); } protected String formatReason(String message, ScanReason reason) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantFieldProvider.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantFieldProvider.java index b2182aa39fdf..5b61e6ea32e4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantFieldProvider.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantFieldProvider.java @@ -45,7 +45,7 @@ public AnalysisConstantFieldProvider(MetaAccessProvider metaAccess, SVMHost host @Override public T readConstantField(ResolvedJavaField f, ConstantFieldTool analysisTool) { AnalysisField field = (AnalysisField) f; - if (!field.isValueAvailable()) { + if (!ReadableJavaField.isValueAvailable(field)) { return null; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java index dc3aa3afcd73..e539ed8702cb 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java @@ -42,6 +42,7 @@ import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.graal.pointsto.meta.AnalysisUniverse; +import com.oracle.graal.pointsto.util.AnalysisError; import com.oracle.svm.core.RuntimeAssertionsSupport; import com.oracle.svm.core.annotate.InjectAccessors; import com.oracle.svm.core.graal.meta.SharedConstantReflectionProvider; @@ -199,11 +200,12 @@ public JavaConstant readValue(UniverseMetaAccess suppliedMetaAccess, AnalysisFie value = readSimulatedValue(field); } if (value == null && receiver instanceof ImageHeapConstant) { + AnalysisError.guarantee(field.isValueAvailable(), "Value not yet available for %s", field); ImageHeapInstance heapObject = (ImageHeapInstance) receiver; value = heapObject.readFieldValue(field); } if (value == null) { - value = universe.lookup(ReadableJavaField.readFieldValue(suppliedMetaAccess, classInitializationSupport, field.wrapped, universe.toHosted(receiver))); + value = doReadValue(field, universe.toHosted(receiver), suppliedMetaAccess); } return interceptValue(suppliedMetaAccess, field, value); } @@ -217,24 +219,24 @@ public ValueSupplier readHostedFieldValue(AnalysisField field, Hos } } - if (field.wrapped instanceof ReadableJavaField) { - ReadableJavaField readableField = (ReadableJavaField) field.wrapped; - if (readableField.isValueAvailableBeforeAnalysis()) { - /* Materialize and return the value. */ - return ValueSupplier.eagerValue(universe.lookup(readableField.readValue(metaAccess, classInitializationSupport, receiver))); - } else { - /* - * Return a lazy value. This applies to RecomputeFieldValue.Kind.FieldOffset and - * RecomputeFieldValue.Kind.Custom. The value becomes available during hosted - * universe building and is installed by calling - * ComputedValueField.processSubstrate() or by ComputedValueField.readValue(). - * Attempts to materialize the value earlier will result in an error. - */ - return ValueSupplier.lazyValue(() -> universe.lookup(readableField.readValue(hMetaAccess, classInitializationSupport, receiver)), - readableField::isValueAvailable); - } + if (ReadableJavaField.isValueAvailable(field)) { + /* Materialize and return the value. */ + return ValueSupplier.eagerValue(doReadValue(field, receiver, metaAccess)); } - return ValueSupplier.eagerValue(universe.lookup(ReadableJavaField.readFieldValue(metaAccess, classInitializationSupport, field.wrapped, receiver))); + /* + * Return a lazy value. First, this applies to fields annotated with + * RecomputeFieldValue.Kind.FieldOffset and RecomputeFieldValue.Kind.Custom whose value + * becomes available during hosted universe building and is installed by calling + * ComputedValueField.processSubstrate() or ComputedValueField.readValue(). Secondly, this + * applies to fields annotated with @UnknownObjectField whose value is set directly either + * during analysis or in a later phase. Attempts to materialize the value before it becomes + * available will result in an error. + */ + return ValueSupplier.lazyValue(() -> doReadValue(field, receiver, hMetaAccess), () -> ReadableJavaField.isValueAvailable(field)); + } + + private JavaConstant doReadValue(AnalysisField field, JavaConstant receiver, UniverseMetaAccess access) { + return universe.lookup(ReadableJavaField.readFieldValue(access, classInitializationSupport, field.wrapped, receiver)); } /** diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/ReadableJavaField.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/ReadableJavaField.java index 6888e97cd445..c183cefdf031 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/ReadableJavaField.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/ReadableJavaField.java @@ -38,6 +38,13 @@ public interface ReadableJavaField extends ResolvedJavaField { + static boolean isValueAvailable(AnalysisField field) { + if (field.wrapped instanceof ReadableJavaField readableField) { + return readableField.isValueAvailable(); + } + return field.isValueAvailable(); + } + static JavaConstant readFieldValue(MetaAccessProvider metaAccess, ClassInitializationSupport classInitializationSupport, ResolvedJavaField field, JavaConstant receiver) { assert !(field instanceof AnalysisField) && !(field instanceof HostedField) : "must have been unwrapped"; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/fieldfolding/StaticFinalFieldFoldingNodePlugin.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/fieldfolding/StaticFinalFieldFoldingNodePlugin.java index 4d9f18270834..6810029380c5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/fieldfolding/StaticFinalFieldFoldingNodePlugin.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/fieldfolding/StaticFinalFieldFoldingNodePlugin.java @@ -80,7 +80,7 @@ public boolean handleLoadStaticField(GraphBuilderContext b, ResolvedJavaField fi return false; } - if (aField.wrapped instanceof ReadableJavaField && !((ReadableJavaField) aField.wrapped).isValueAvailable()) { + if (!ReadableJavaField.isValueAvailable(aField)) { /* * Cannot optimize static field whose value is recomputed and is not yet available, * i.e., it may depend on analysis/compilation derived data. diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java index 40280dd3d0bd..1b2725da4a0f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java @@ -111,11 +111,7 @@ protected ImageHeapConstant getOrCreateImageHeapConstant(JavaConstant javaConsta @Override public boolean isValueAvailable(AnalysisField field) { - if (field.wrapped instanceof ReadableJavaField) { - ReadableJavaField readableField = (ReadableJavaField) field.wrapped; - return readableField.isValueAvailable(); - } - return super.isValueAvailable(field); + return ReadableJavaField.isValueAvailable(field); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java index 05dfbee81738..0ef281182c94 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java @@ -504,7 +504,12 @@ private void addObjectToImageHeap(final JavaConstant constant, boolean immutable final boolean fieldsAreImmutable = hMetaAccess.isInstanceOf(constant, String.class); for (HostedField field : clazz.getInstanceFields(true)) { boolean fieldRelocatable = false; - if (field.isRead() && + /* + * Fields that are only available after heap layout, such as + * StringInternSupport.imageInternedStrings and all ImageHeapInfo fields will + * not be processed. + */ + if (field.isRead() && field.isValueAvailable() && !field.equals(hybridArrayField) && !field.equals(hybridTypeIDSlotsField)) { if (field.getJavaKind() == JavaKind.Object) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/SharedConstantFieldProvider.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/SharedConstantFieldProvider.java index f6a7a2c50470..b3fc97e3bccd 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/SharedConstantFieldProvider.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/SharedConstantFieldProvider.java @@ -53,10 +53,8 @@ public SharedConstantFieldProvider(MetaAccessProvider metaAccess, SVMHost hostVM @Override public T readConstantField(ResolvedJavaField field, ConstantFieldTool analysisTool) { - if (asAnalysisField(field).getWrapped() instanceof ReadableJavaField readableField) { - if (!readableField.isValueAvailable()) { - return null; - } + if (!ReadableJavaField.isValueAvailable(asAnalysisField(field))) { + return null; } return super.readConstantField(field, analysisTool); } From 70c177af76f929438d4b09c12247ce89efc1ad23 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Mon, 7 Aug 2023 15:41:14 +0200 Subject: [PATCH 05/29] Check availability via ReadableJavaField.isValueAvailable(). --- .../svm/hosted/ameta/AnalysisConstantReflectionProvider.java | 2 +- .../src/com/oracle/svm/hosted/meta/HostedField.java | 3 ++- .../src/com/oracle/svm/hosted/meta/UniverseBuilder.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java index e539ed8702cb..bc9188cfb1e6 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java @@ -200,7 +200,7 @@ public JavaConstant readValue(UniverseMetaAccess suppliedMetaAccess, AnalysisFie value = readSimulatedValue(field); } if (value == null && receiver instanceof ImageHeapConstant) { - AnalysisError.guarantee(field.isValueAvailable(), "Value not yet available for %s", field); + AnalysisError.guarantee(ReadableJavaField.isValueAvailable(field), "Value not yet available for %s", field); ImageHeapInstance heapObject = (ImageHeapInstance) receiver; value = heapObject.readFieldValue(field); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedField.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedField.java index add91cf1890f..2737b32b2a43 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedField.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedField.java @@ -30,6 +30,7 @@ import com.oracle.graal.pointsto.infrastructure.WrappedJavaField; import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.svm.core.meta.SharedField; +import com.oracle.svm.hosted.ameta.ReadableJavaField; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.JavaTypeProfile; @@ -119,7 +120,7 @@ public boolean isWritten() { @Override public boolean isValueAvailable() { - return wrapped.isValueAvailable(); + return ReadableJavaField.isValueAvailable(wrapped); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java index 39baf9f87627..dd7fd73d3b9d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java @@ -94,6 +94,7 @@ import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl; import com.oracle.svm.hosted.HostedConfiguration; import com.oracle.svm.hosted.NativeImageOptions; +import com.oracle.svm.hosted.ameta.ReadableJavaField; import com.oracle.svm.hosted.annotation.CustomSubstitutionMethod; import com.oracle.svm.hosted.config.HybridLayout; import com.oracle.svm.hosted.heap.PodSupport; @@ -1116,7 +1117,7 @@ private void processFieldLocations() { ((ComputedValueField) aField.wrapped).processSubstrate(hMetaAccess); } - if (!hField.hasLocation() && Modifier.isStatic(hField.getModifiers()) && !aField.isWritten() && aField.isValueAvailable()) { + if (!hField.hasLocation() && Modifier.isStatic(hField.getModifiers()) && !aField.isWritten() && ReadableJavaField.isValueAvailable(aField)) { hField.setUnmaterializedStaticConstant(); } } From 283b2afb3ac14b2741a3d2563bb6db9d78764a5e Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Fri, 4 Aug 2023 12:14:09 +0200 Subject: [PATCH 06/29] Refactor Graal object scanning during analysis. Cannot rescan unknown fields during analysis, they are not available. Just rescan their value if they reference objects. --- .../com/oracle/svm/core/hub/DynamicHub.java | 1 + .../com/oracle/svm/graal/GraalSupport.java | 32 +++++++++++-------- .../hosted/GraalGraphObjectReplacer.java | 19 +++-------- .../oracle/svm/graal/meta/SubstrateField.java | 4 +-- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index 9714ea3de02a..51388df7948a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -314,6 +314,7 @@ public final class DynamicHub implements AnnotatedElement, java.lang.reflect.Typ * Back link to the SubstrateType used by the substrate meta access. Only used for the subset of * types for which a SubstrateType exists. */ + @UnknownObjectField(fullyQualifiedTypes = "com.oracle.svm.graal.meta.SubstrateType")// private SharedType metaType; /** diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/GraalSupport.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/GraalSupport.java index b2c7e18efb8e..c42a0b1a95b8 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/GraalSupport.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/GraalSupport.java @@ -27,7 +27,6 @@ import static org.graalvm.word.LocationIdentity.ANY_LOCATION; import java.io.PrintStream; -import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; @@ -73,6 +72,7 @@ import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import com.oracle.graal.pointsto.meta.AnalysisUniverse; import com.oracle.svm.core.c.CGlobalData; import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.config.ConfigurationValues; @@ -86,7 +86,6 @@ import com.oracle.svm.core.util.ImageHeapMap; import com.oracle.svm.graal.meta.SubstrateMethod; import com.oracle.svm.hosted.FeatureImpl.DuringAnalysisAccessImpl; -import com.oracle.svm.util.ReflectionUtil; /** * Holds data that is pre-computed during native image generation and accessed at run time during a @@ -129,11 +128,6 @@ public class GraalSupport { protected final DiagnosticsOutputDirectory outputDirectory = new DiagnosticsOutputDirectory(RuntimeOptionValues.singleton()); protected final Map compilationProblemsPerAction = new EnumMap<>(ExceptionAction.class); - private static final Field graphEncodingField = ReflectionUtil.lookupField(GraalSupport.class, "graphEncoding"); - private static final Field graphObjectsField = ReflectionUtil.lookupField(GraalSupport.class, "graphObjects"); - private static final Field graphNodeTypesField = ReflectionUtil.lookupField(GraalSupport.class, "graphNodeTypes"); - private static final Field methodsToCompileField = ReflectionUtil.lookupField(GraalSupport.class, "methodsToCompile"); - private static final CGlobalData nextIsolateId = CGlobalDataFactory.createWord((Pointer) WordFactory.unsigned(1L)); private volatile long isolateId = 0; @@ -234,7 +228,7 @@ public static boolean setMethodsToCompile(DuringAnalysisAccessImpl config, Subst GraalSupport support = get(); if (!Arrays.equals(support.methodsToCompile, methodsToCompile)) { support.methodsToCompile = methodsToCompile; - GraalSupport.rescan(config, support, methodsToCompileField); + GraalSupport.rescan(config, methodsToCompile); result = true; } return result; @@ -274,28 +268,38 @@ public static boolean setGraphEncoding(FeatureAccess a, byte[] graphEncoding, Ob boolean result = false; if (!Arrays.equals(support.graphEncoding, graphEncoding)) { support.graphEncoding = graphEncoding; - GraalSupport.rescan(a, support, graphEncodingField); result = true; } if (!Arrays.deepEquals(support.graphObjects, graphObjects)) { support.graphObjects = graphObjects; - GraalSupport.rescan(a, support, graphObjectsField); + GraalSupport.rescan(a, graphObjects); result = true; } if (!Arrays.equals(support.graphNodeTypes, graphNodeTypes)) { support.graphNodeTypes = graphNodeTypes; - GraalSupport.rescan(a, support, graphNodeTypesField); + GraalSupport.rescan(a, graphNodeTypes); result = true; } return result; } - private static void rescan(FeatureAccess a, GraalSupport support, Field field) { - if (a instanceof DuringAnalysisAccessImpl) { - ((DuringAnalysisAccessImpl) a).rescanField(support, field); + private static void rescan(FeatureAccess a, Object object) { + if (a instanceof DuringAnalysisAccessImpl access) { + rescan(access.getUniverse(), object); } } + /** + * Rescan Graal objects during analysis. The fields that point to these objects are annotated + * with {@link UnknownObjectField} so their value is not processed during analysis, only their + * declared type is injected in the type flow graphs. Their eventual value becomes available + * after analysis. Later when the field is read the lazy value supplier scans the final value + * and patches the shadow heap. + */ + public static void rescan(AnalysisUniverse universe, Object object) { + universe.getHeapScanner().rescanObject(object); + } + @Platforms(Platform.HOSTED_ONLY.class) public static void registerImmutableObjects(CompilationAccess access) { access.registerAsImmutable(get().graphEncoding); diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalGraphObjectReplacer.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalGraphObjectReplacer.java index c79525081653..133b13edc964 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalGraphObjectReplacer.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalGraphObjectReplacer.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.graal.hosted; -import java.lang.reflect.Field; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -56,6 +55,7 @@ import com.oracle.svm.core.util.HostedStringDeduplication; import com.oracle.svm.core.util.ObservableImageHeapMapProvider; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.graal.GraalSupport; import com.oracle.svm.graal.SubstrateGraalRuntime; import com.oracle.svm.graal.meta.SubstrateField; import com.oracle.svm.graal.meta.SubstrateMethod; @@ -107,11 +107,6 @@ public class GraalGraphObjectReplacer implements Function { private final HostedStringDeduplication stringTable; - private final Field substrateFieldTypeField; - private final Field substrateFieldDeclaringClassField; - private final Field dynamicHubMetaTypeField; - private final Field substrateTypeRawAllInstanceFieldsField; - private final Class jvmciCleanerClass = ReflectionUtil.lookupClass(false, "jdk.vm.ci.hotspot.Cleaner"); /** @@ -125,10 +120,6 @@ public GraalGraphObjectReplacer(AnalysisUniverse aUniverse, SubstrateProviders s this.sProviders = sProviders; this.universeFactory = universeFactory; this.stringTable = HostedStringDeduplication.singleton(); - substrateFieldTypeField = ReflectionUtil.lookupField(SubstrateField.class, "type"); - substrateFieldDeclaringClassField = ReflectionUtil.lookupField(SubstrateField.class, "declaringClass"); - dynamicHubMetaTypeField = ReflectionUtil.lookupField(DynamicHub.class, "metaType"); - substrateTypeRawAllInstanceFieldsField = ReflectionUtil.lookupField(SubstrateType.class, "rawAllInstanceFields"); } public void setGraalRuntime(SubstrateGraalRuntime sGraalRuntime) { @@ -289,8 +280,8 @@ public synchronized SubstrateField createField(ResolvedJavaField original) { sField = newField; sField.setLinks(createType(aField.getType()), createType(aField.getDeclaringClass())); - aUniverse.getHeapScanner().rescanField(sField, substrateFieldTypeField); - aUniverse.getHeapScanner().rescanField(sField, substrateFieldDeclaringClassField); + GraalSupport.rescan(aUniverse, sField.getType()); + GraalSupport.rescan(aUniverse, sField.getDeclaringClass()); } } return sField; @@ -338,10 +329,10 @@ public synchronized SubstrateType createType(JavaType original) { if (sType == null) { sType = newType; hub.setMetaType(sType); - aUniverse.getHeapScanner().rescanField(hub, dynamicHubMetaTypeField); + GraalSupport.rescan(aUniverse, hub.getMetaType()); sType.setRawAllInstanceFields(createAllInstanceFields(aType)); - aUniverse.getHeapScanner().rescanField(sType, substrateTypeRawAllInstanceFieldsField); + GraalSupport.rescan(aUniverse, sType.getRawAllInstanceFields()); createType(aType.getSuperclass()); createType(aType.getComponentType()); for (AnalysisType aInterface : aType.getInterfaces()) { diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateField.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateField.java index 46ee9d5cb435..fcaca720e75d 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateField.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateField.java @@ -50,8 +50,8 @@ public class SubstrateField implements SharedField { protected static final SubstrateField[] EMPTY_ARRAY = new SubstrateField[0]; - SubstrateType type; - SubstrateType declaringClass; + @UnknownObjectField SubstrateType type; + @UnknownObjectField SubstrateType declaringClass; private final String name; private final int modifiers; private int hashCode; From 8a37b19505115ed2138575653937aae13b77bf4e Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Tue, 1 Aug 2023 19:24:42 +0200 Subject: [PATCH 07/29] Fix @UnkownObjectField availability. --- .../src/com/oracle/svm/core/heapdump/HeapDumpUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpUtils.java index 40b7c984f9de..385c917fc1ae 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpUtils.java @@ -37,6 +37,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.BuildPhaseProvider.AfterCompilation; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.code.CodeInfo; @@ -64,7 +65,7 @@ */ public class HeapDumpUtils { - @UnknownObjectField private byte[] fieldsMap; + @UnknownObjectField(availability = AfterCompilation.class) private byte[] fieldsMap; /** Extra methods for testing. */ private final TestingBackDoor testingBackDoor; From 3f74e7e5723b5228c23c777638f76b98313c2ffb Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Wed, 2 Aug 2023 17:10:12 +0200 Subject: [PATCH 08/29] Propagate `null` values even when field value is not yet available. --- .../graal/pointsto/heap/ImageHeapScanner.java | 36 +++++++++++-------- .../svm/core/genscavenge/ImageHeapInfo.java | 32 ++++++++--------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java index 752dba946779..4f5fcde13628 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java @@ -118,17 +118,19 @@ public void scanEmbeddedRoot(JavaConstant root, BytecodePosition position) { public void onFieldRead(AnalysisField field) { assert field.isRead(); /* Check if the value is available before accessing it. */ - if (isValueAvailable(field)) { - FieldScan reason = new FieldScan(field); - AnalysisType declaringClass = field.getDeclaringClass(); - if (field.isStatic()) { + FieldScan reason = new FieldScan(field); + AnalysisType declaringClass = field.getDeclaringClass(); + if (field.isStatic()) { + if (isValueAvailable(field)) { JavaConstant fieldValue = declaringClass.getOrComputeData().readFieldValue(field); markReachable(fieldValue, reason); notifyAnalysis(field, null, fieldValue, reason); - } else { - /* Trigger field scanning for the already processed objects. */ - postTask(() -> onInstanceFieldRead(field, declaringClass, reason)); + } else if (field.canBeNull()) { + notifyAnalysis(field, null, JavaConstant.NULL_POINTER, reason); } + } else { + /* Trigger field scanning for the already processed objects. */ + postTask(() -> onInstanceFieldRead(field, declaringClass, reason)); } } @@ -136,9 +138,7 @@ private void onInstanceFieldRead(AnalysisField field, AnalysisType type, FieldSc for (AnalysisType subtype : type.getSubTypes()) { for (ImageHeapConstant imageHeapConstant : imageHeap.getReachableObjects(subtype)) { ImageHeapInstance imageHeapInstance = (ImageHeapInstance) imageHeapConstant; - JavaConstant fieldValue = imageHeapInstance.readFieldValue(field); - markReachable(fieldValue, reason); - notifyAnalysis(field, imageHeapInstance, fieldValue, reason); + updateInstanceField(field, imageHeapInstance, reason, null); } /* Subtypes include this type itself. */ if (!subtype.equals(type)) { @@ -468,15 +468,23 @@ protected void onObjectReachable(ImageHeapConstant imageHeapConstant, ScanReason } else if (imageHeapConstant instanceof ImageHeapInstance imageHeapInstance) { for (ResolvedJavaField javaField : objectType.getInstanceFields(true)) { AnalysisField field = (AnalysisField) javaField; - if (field.isRead() && isValueAvailable(field)) { - JavaConstant fieldValue = imageHeapInstance.readFieldValue(field); - markReachable(fieldValue, reason, onAnalysisModified); - notifyAnalysis(field, imageHeapInstance, fieldValue, reason, onAnalysisModified); + if (field.isRead()) { + updateInstanceField(field, imageHeapInstance, reason, onAnalysisModified); } } } } + private void updateInstanceField(AnalysisField field, ImageHeapInstance imageHeapInstance, ScanReason reason, Consumer onAnalysisModified) { + if (isValueAvailable(field)) { + JavaConstant fieldValue = imageHeapInstance.readFieldValue(field); + markReachable(fieldValue, reason, onAnalysisModified); + notifyAnalysis(field, imageHeapInstance, fieldValue, reason, onAnalysisModified); + } else if (field.canBeNull()) { + notifyAnalysis(field, imageHeapInstance, JavaConstant.NULL_POINTER, reason, onAnalysisModified); + } + } + public boolean isValueAvailable(AnalysisField field) { return field.isValueAvailable(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java index 191748d82cc1..d6819af09d97 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java @@ -47,29 +47,29 @@ public final class ImageHeapInfo { /** Indicates no chunk with {@link #initialize} chunk offset parameters. */ public static final long NO_CHUNK = -1; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object firstReadOnlyPrimitiveObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object lastReadOnlyPrimitiveObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstReadOnlyPrimitiveObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastReadOnlyPrimitiveObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object firstReadOnlyReferenceObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object lastReadOnlyReferenceObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstReadOnlyReferenceObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastReadOnlyReferenceObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object firstReadOnlyRelocatableObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object lastReadOnlyRelocatableObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstReadOnlyRelocatableObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastReadOnlyRelocatableObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object firstWritablePrimitiveObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object lastWritablePrimitiveObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstWritablePrimitiveObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastWritablePrimitiveObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object firstWritableReferenceObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object lastWritableReferenceObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstWritableReferenceObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastWritableReferenceObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object firstWritableHugeObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object lastWritableHugeObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstWritableHugeObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastWritableHugeObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object firstReadOnlyHugeObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object lastReadOnlyHugeObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstReadOnlyHugeObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastReadOnlyHugeObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object firstObject; - @UnknownObjectField(availability = AfterHeapLayout.class) public Object lastObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastObject; // All offsets are relative to the heap base. @UnknownPrimitiveField(availability = AfterHeapLayout.class) public long offsetOfFirstWritableAlignedChunk; From a9ec7794c4c2601e5657f200f12ec31c5dab42dd Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Thu, 3 Aug 2023 23:22:16 +0200 Subject: [PATCH 09/29] Scan RuntimeModuleSupport.bootLayer prototype object. --- .../oracle/svm/hosted/ModuleLayerFeature.java | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java index a85de883ffa6..e2a5f40568d1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java @@ -66,9 +66,11 @@ import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.heap.UnknownObjectField; import com.oracle.svm.core.jdk.Resources; import com.oracle.svm.core.jdk.RuntimeModuleSupport; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -124,17 +126,6 @@ public void duringSetup(DuringSetupAccess access) { FeatureImpl.DuringSetupAccessImpl accessImpl = (FeatureImpl.DuringSetupAccessImpl) access; moduleLayerFeatureUtils = new ModuleLayerFeatureUtils(accessImpl.imageClassLoader); - /* - * Generate a temporary module layer to serve as a runtime boot module layer until the - * analysis is finished. - */ - Set baseModules = ModuleLayer.boot().modules() - .stream() - .map(Module::getName) - .collect(Collectors.toSet()); - Function clf = moduleLayerFeatureUtils::getClassLoaderForBootLayerModule; - ModuleLayer runtimeBootLayer = synthesizeRuntimeModuleLayer(new ArrayList<>(List.of(ModuleLayer.empty())), accessImpl.imageClassLoader, baseModules, Set.of(), clf, null); - RuntimeModuleSupport.instance().setBootLayer(runtimeBootLayer); RuntimeModuleSupport.instance().setHostedToRuntimeModuleMapper(moduleLayerFeatureUtils::getOrCreateRuntimeModuleForHostedModule); /* @@ -168,6 +159,28 @@ public void afterRegistration(AfterRegistrationAccess access) { } } + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + scanRuntimeBootLayerPrototype((BeforeAnalysisAccessImpl) access); + } + + /** + * Generate a temporary module layer to serve as a prototype object for + * {@link RuntimeModuleSupport}.bootLayer. The value doesn't need to actually be set in the + * field, just scanned such that the analysis sees the deep type hierarchy. The field value + * wouldn't be processed during analysis anyway since the field is annotated with + * {@link UnknownObjectField}, only the field declared type is injected in the type flow graphs. + * The concrete value is set in {@link ModuleLayerFeature#afterAnalysis}. Later when the field + * is read the lazy value supplier scans the concrete value and patches the shadow heap. + */ + private void scanRuntimeBootLayerPrototype(BeforeAnalysisAccessImpl accessImpl) { + Set baseModules = ModuleLayer.boot().modules().stream().map(Module::getName).collect(Collectors.toSet()); + Function clf = moduleLayerFeatureUtils::getClassLoaderForBootLayerModule; + ModuleLayer runtimeBootLayer = synthesizeRuntimeModuleLayer(new ArrayList<>(List.of(ModuleLayer.empty())), accessImpl.imageClassLoader, baseModules, Set.of(), clf, null); + /* Only scan the value if module support is enabled and bootLayer field is reachable. */ + accessImpl.registerReachabilityHandler((a) -> accessImpl.rescanObject(runtimeBootLayer), ReflectionUtil.lookupField(RuntimeModuleSupport.class, "bootLayer")); + } + @Override public void afterAnalysis(AfterAnalysisAccess access) { FeatureImpl.AfterAnalysisAccessImpl accessImpl = (FeatureImpl.AfterAnalysisAccessImpl) access; From bbb7ac469abc8b7c2efa59ff755948eeda76e76f Mon Sep 17 00:00:00 2001 From: Andreas Woess Date: Thu, 31 Aug 2023 01:15:40 +0200 Subject: [PATCH 10/29] [GR-47958] Fix assertions in PolyglotWrapper host code cache lookups. --- .../src/com/oracle/truffle/polyglot/PolyglotIterable.java | 4 ++-- .../src/com/oracle/truffle/polyglot/PolyglotList.java | 4 ++-- .../src/com/oracle/truffle/polyglot/PolyglotMap.java | 6 +++--- .../src/com/oracle/truffle/polyglot/PolyglotMapEntry.java | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotIterable.java b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotIterable.java index e257ef934477..c8273f718a57 100644 --- a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotIterable.java +++ b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotIterable.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -138,7 +138,7 @@ static Cache lookup(PolyglotLanguageContext languageContext, Class receiverCl } assert cache.receiverClass == receiverClass; assert cache.valueClass == valueClass; - assert cache.valueType == valueType; + assert Objects.equals(cache.valueType, valueType); return cache; } diff --git a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotList.java b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotList.java index 91a6dd54d6f8..03983eb4d2a1 100644 --- a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotList.java +++ b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotList.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -191,7 +191,7 @@ static Cache lookup(PolyglotLanguageContext languageContext, Class receiverCl } assert cache.receiverClass == receiverClass; assert cache.valueClass == valueClass; - assert cache.valueType == valueType; + assert Objects.equals(cache.valueType, valueType); return cache; } diff --git a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotMap.java b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotMap.java index 9e61e94820e6..571aeadad80c 100644 --- a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotMap.java +++ b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -427,9 +427,9 @@ static Cache lookup(PolyglotLanguageContext languageContext, Class receiverCl } assert cache.receiverClass == receiverClass; assert cache.keyClass == keyClass; - assert cache.keyType == keyType; + assert Objects.equals(cache.keyType, keyType); assert cache.valueClass == valueClass; - assert cache.valueType == valueType; + assert Objects.equals(cache.valueType, valueType); return cache; } diff --git a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotMapEntry.java b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotMapEntry.java index be30599d0b55..c9bb3983a160 100644 --- a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotMapEntry.java +++ b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotMapEntry.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -164,9 +164,9 @@ static PolyglotMapEntry.Cache lookup(PolyglotLanguageContext languageContext, Cl } assert cache.receiverClass == receiverClass; assert cache.keyClass == keyClass; - assert cache.keyType == keyType; + assert Objects.equals(cache.keyType, keyType); assert cache.valueClass == valueClass; - assert cache.valueType == valueType; + assert Objects.equals(cache.valueType, valueType); return cache; } From 45cb38765981da5ae4ea372c6e81b752bd6f09ff Mon Sep 17 00:00:00 2001 From: Andreas Woess Date: Thu, 31 Aug 2023 01:42:30 +0200 Subject: [PATCH 11/29] [GR-47958] Add regression test. --- .../oracle/truffle/api/test/host/GR47958.java | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/host/GR47958.java diff --git a/truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/host/GR47958.java b/truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/host/GR47958.java new file mode 100644 index 000000000000..08cc926d387b --- /dev/null +++ b/truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/host/GR47958.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.oracle.truffle.api.test.host; + +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.HostAccess; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.proxy.ProxyArray; +import org.graalvm.polyglot.proxy.ProxyHashMap; +import org.junit.Assert; +import org.junit.Test; + +import com.oracle.truffle.api.test.host.AsCollectionsTest.ListBasedTO; +import com.oracle.truffle.api.test.host.AsCollectionsTest.MapBasedTO; + +/** + * Reproducer for https://github.com/oracle/graal/issues/7082. + */ +public class GR47958 { + + @FunctionalInterface + public interface GenericListConsumer { + @HostAccess.Export + String call(List> arg); + } + + @FunctionalInterface + public interface GenericMapConsumer { + @HostAccess.Export + String call(Map> arg); + } + + @SuppressWarnings("static-method") + private String toString(List> arg) { + return String.valueOf(arg.stream().flatMap(List::stream).collect(Collectors.joining(", "))); + } + + @SuppressWarnings("static-method") + private String toString(Map> arg) { + return String.valueOf(arg.entrySet().stream().map(e -> e.getKey() + ", " + String.join(", ", e.getValue())).collect(Collectors.joining(", "))); + } + + @Test + public void testPolyglotListReproducer() { + HostAccess hostAccess = HostAccess.newBuilder(HostAccess.EXPLICIT).allowAccessInheritance(true).build(); + try (Context context = Context.newBuilder().allowHostAccess(hostAccess).build()) { + for (var invoker : List.> of( + (target, array) -> target.invokeMember("call", array), + (target, array) -> target.execute(array))) { + Value target; + Value result; + Object array; + + target = context.asValue((GenericListConsumer) this::toString); + array = new ListBasedTO(List.of(new ListBasedTO(List.of("hello")), new ListBasedTO(List.of("world")), new ListBasedTO(List.of()))); + result = invoker.apply(target, array); + Assert.assertEquals("hello, world", result.toString()); + + target = context.asValue((GenericListConsumer) this::toString); + array = new ListBasedTO(List.of(new ListBasedTO(List.of("hello")))); + result = invoker.apply(target, array); + Assert.assertEquals("hello", result.toString()); + + target = context.asValue((GenericListConsumer) this::toString); + array = ProxyArray.fromArray(ProxyArray.fromArray("hello"), ProxyArray.fromArray("world"), ProxyArray.fromArray()); + result = invoker.apply(target, array); + Assert.assertEquals("hello, world", result.toString()); + + target = context.asValue((GenericListConsumer) this::toString); + array = ProxyArray.fromArray(ProxyArray.fromArray("hello")); + result = invoker.apply(target, array); + Assert.assertEquals("hello", result.toString()); + } + } + } + + @Test + public void testPolyglotMapReproducer() { + HostAccess hostAccess = HostAccess.newBuilder(HostAccess.EXPLICIT).allowAccessInheritance(true).build(); + try (Context context = Context.newBuilder().allowHostAccess(hostAccess).build()) { + for (var invoker : List.> of( + (target, array) -> target.invokeMember("call", array), + (target, array) -> target.execute(array))) { + Value target; + Value result; + Object map; + + target = context.asValue((GenericMapConsumer) this::toString); + map = new MapBasedTO(Map.of("hello", new ListBasedTO(List.of("world")))); + result = invoker.apply(target, map); + Assert.assertEquals("hello, world", result.toString()); + + target = context.asValue((GenericMapConsumer) this::toString); + map = new MapBasedTO(Map.of("hi", new ListBasedTO(List.of("there")))); + result = invoker.apply(target, map); + Assert.assertEquals("hi, there", result.toString()); + + target = context.asValue((GenericMapConsumer) this::toString); + map = ProxyHashMap.from(Map.of("hello", ProxyArray.fromList(List.of("world")))); + result = invoker.apply(target, map); + Assert.assertEquals("hello, world", result.toString()); + + target = context.asValue((GenericMapConsumer) this::toString); + map = ProxyHashMap.from(Map.of("hi", ProxyArray.fromList(List.of("there")))); + result = invoker.apply(target, map); + Assert.assertEquals("hi, there", result.toString()); + } + } + } +} From ebd85ff1ec60375c1a05cdb94fad6969770c4885 Mon Sep 17 00:00:00 2001 From: Andreas Woess Date: Thu, 31 Aug 2023 01:59:46 +0200 Subject: [PATCH 12/29] Use Java 16 InvocationHandler.invokeDefault. --- .../PolyglotFunctionProxyHandler.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotFunctionProxyHandler.java b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotFunctionProxyHandler.java index 607289f09f20..b07e1a06d247 100644 --- a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotFunctionProxyHandler.java +++ b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotFunctionProxyHandler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -40,9 +40,6 @@ */ package com.oracle.truffle.polyglot; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -53,7 +50,6 @@ import com.oracle.truffle.api.CallTarget; import com.oracle.truffle.api.CompilerAsserts; import com.oracle.truffle.api.CompilerDirectives; -import com.oracle.truffle.api.Truffle; import com.oracle.truffle.api.TruffleOptions; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.ImportStatic; @@ -191,18 +187,7 @@ static Object invokeDefault(PolyglotWrapper host, Object proxy, Method method, O if (TruffleOptions.AOT) { throw new UnsupportedOperationException("calling default method " + method.getName() + " is not yet supported on SubstrateVM"); } - - // default method; requires Java 9 (JEP 274) - Class declaringClass = method.getDeclaringClass(); - assert declaringClass.isInterface() : declaringClass; - MethodHandle mh; - try { - Truffle.class.getModule().addReads(declaringClass.getModule()); - mh = MethodHandles.lookup().findSpecial(declaringClass, method.getName(), MethodType.methodType(method.getReturnType(), method.getParameterTypes()), declaringClass); - } catch (IllegalAccessException e) { - throw new UnsupportedOperationException(method.getName(), e); - } - return mh.bindTo(proxy).invokeWithArguments(arguments); + return InvocationHandler.invokeDefault(proxy, method, arguments); } @ImportStatic(ProxyInvokeNode.class) From babc773551362599b7990f81953cc7b6f349264a Mon Sep 17 00:00:00 2001 From: Andreas Woess Date: Thu, 31 Aug 2023 05:03:27 +0200 Subject: [PATCH 13/29] Unignore test and use assumeNotAOT() instead. --- .../truffle/api/test/polyglot/ValueHostInteropTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/polyglot/ValueHostInteropTest.java b/truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/polyglot/ValueHostInteropTest.java index decceaa27a7b..6db6765e0635 100644 --- a/truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/polyglot/ValueHostInteropTest.java +++ b/truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/polyglot/ValueHostInteropTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -78,7 +78,6 @@ import org.graalvm.polyglot.proxy.ProxyObject; import org.hamcrest.CoreMatchers; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -537,9 +536,9 @@ public void executableAsFunctionalInterface2() throws Exception { f.toString(); } - @Ignore("Interface not accessible") @Test public void executableAsFunctionalInterface3() throws Exception { + TruffleTestAssumptions.assumeNotAOT(); TruffleTestAssumptions.assumeNoClassLoaderEncapsulation(); TruffleObject executable = new FunctionObject(); FunctionalWithDefaults f = context.asValue(executable).as(FunctionalWithDefaults.class); From 5c57b4a1dc056e078cb8f61acc64bcd2af3757f7 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 23 Aug 2023 13:31:12 +0200 Subject: [PATCH 14/29] add maven downloader --- sdk/mx.sdk/mx_sdk_vm_impl.py | 4 +- vm/mx.vm/suite.py | 24 ++ .../maven.downloader/native-image.properties | 4 + .../maven/downloader/MVNDownloader.java | 274 ++++++++++++++++++ .../org/graalvm/maven/downloader/Main.java | 188 ++++++++++++ .../maven/downloader/OptionProperties.java | 86 ++++++ 6 files changed, 578 insertions(+), 2 deletions(-) create mode 100644 vm/src/org.graalvm.maven.downloader/src/META-INF/native-image/org.graalvm/maven.downloader/native-image.properties create mode 100644 vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/MVNDownloader.java create mode 100644 vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/Main.java create mode 100644 vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/OptionProperties.java diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 309699e45be0..4c19a536296a 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -2816,7 +2816,7 @@ def add_files_from_component(comp, is_main, path_prefix, excluded_paths, force_m 'path': None, }) - assert not self.is_jvm or not is_main or len(launcher_configs) == 0, "JVM standalones do not yet support main components with launcher configs, only thin launchers. Found: '{}'".format(name) + assert not self.is_jvm or not is_main or all(type(lc) == mx_sdk.LauncherConfig for lc in launcher_configs), "JVM standalones do not yet support main components with language launcher configs, only thin launchers and plain NI launchers. Found: '{}'".format(name) # pylint: disable=unidiomatic-typecheck for launcher_config in launcher_configs: launcher_dest = path_prefix + launcher_config.destination if launcher_config.destination not in excluded_paths: @@ -3560,7 +3560,7 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi mx.warn("Skipping standalone of '{}' because the following components are excluded:\n * {}".format(main_component.name, '\n * '.join(missing_dependency_names))) else: # JVM standalones - if len(main_component.launcher_configs): + if any(type(lc) != mx_sdk.LauncherConfig for lc in main_component.launcher_configs): # pylint: disable=unidiomatic-typecheck if mx.get_opts().verbose: mx.warn("Skipping JVM standalone of '{}' because it contains launcher configs that are not yet supported".format(main_component.name)) else: diff --git a/vm/mx.vm/suite.py b/vm/mx.vm/suite.py index 01cb5b5a2cb9..254cdff3e842 100644 --- a/vm/mx.vm/suite.py +++ b/vm/mx.vm/suite.py @@ -83,6 +83,19 @@ }, "projects": { + "org.graalvm.maven.downloader" : { + "subDir" : "src", + "sourceDirs" : ["src"], + "javaCompliance" : "17+", + "license" : "UPL", + "dependencies": [ + "sdk:NATIVEIMAGE", + ], + "requires": [ + "java.logging", + "java.xml", + ], + }, "org.graalvm.component.installer" : { "subDir" : "src", "sourceDirs" : ["src"], @@ -191,6 +204,17 @@ }, "distributions": { + "MAVEN_DOWNLOADER": { + "defaultBuild": False, + "mainClass": "org.graalvm.maven.downloader.Main", + "dependencies": [ + "org.graalvm.maven.downloader", + ], + "distDependencies": [ + "sdk:NATIVEIMAGE", + ], + "maven": False, + }, "INSTALLER": { "subDir": "src", "mainClass": "org.graalvm.component.installer.ComponentInstaller", diff --git a/vm/src/org.graalvm.maven.downloader/src/META-INF/native-image/org.graalvm/maven.downloader/native-image.properties b/vm/src/org.graalvm.maven.downloader/src/META-INF/native-image/org.graalvm/maven.downloader/native-image.properties new file mode 100644 index 000000000000..16455e30dde5 --- /dev/null +++ b/vm/src/org.graalvm.maven.downloader/src/META-INF/native-image/org.graalvm/maven.downloader/native-image.properties @@ -0,0 +1,4 @@ +Args = --enable-url-protocols=http,https \ + --install-exit-handlers \ + --initialize-at-build-time=org.graalvm.maven.downloader.OptionProperties \ + --initialize-at-run-time=org.graalvm.maven.downloader.Main,org.graalvm.maven.downloader.MVNDownloader,org.graalvm.maven.downloader.Main$Arguments,org.graalvm.maven.downloader.MVNDownloader$DeleteDownloadDir diff --git a/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/MVNDownloader.java b/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/MVNDownloader.java new file mode 100644 index 000000000000..6eb771c4a608 --- /dev/null +++ b/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/MVNDownloader.java @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.maven.downloader; + +import static org.graalvm.maven.downloader.Main.LOGGER; +import static org.graalvm.maven.downloader.OptionProperties.DEFAULT_MAVEN_REPO; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.module.ModuleFinder; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.NoSuchAlgorithmException; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +public class MVNDownloader { + + private final Set downloadedJars = new HashSet<>(); + private final Set existingModules; + private final Path outputDir; + private final Path downloadDir; + + private final class DeleteDownloadDir implements Runnable { + @Override + public void run() { + try { + Files.deleteIfExists(downloadDir); + } catch (IOException e) { + // Nothing we can do now + } + } + } + + public MVNDownloader(String outputDir) throws IOException { + this.outputDir = Paths.get(outputDir); + this.existingModules = getModuleNamesInDirectory(this.outputDir); + this.downloadDir = Files.createTempDirectory(MVNDownloader.class.getSimpleName()); + Runtime.getRuntime().addShutdownHook(new Thread(new DeleteDownloadDir())); + } + + private static Set getModuleNamesInDirectory(Path dir) { + return ModuleFinder.of(dir).findAll().stream().map(mr -> mr.descriptor().name()).collect(Collectors.toUnmodifiableSet()); + } + + public void downloadDependencies(String repoUrl, String groupId, String artifactId, String version) + throws IOException, URISyntaxException, ParserConfigurationException, SAXException, ClassCastException, NoSuchAlgorithmException { + String artifactName = toMavenPath(groupId, artifactId, version, "pom"); + boolean mvnCentralFallback = !repoUrl.startsWith(DEFAULT_MAVEN_REPO); + byte[] bytes = downloadMavenFile(repoUrl, artifactName, mvnCentralFallback); + + var factory = DocumentBuilderFactory.newInstance(); + var builder = factory.newDocumentBuilder(); + var document = builder.parse(new ByteArrayInputStream(bytes)); + + // We care only about a very small subset of the POM, and accept even malformed POMs if the + // required tags are there. Since this is supposed to access an actual repository, real + // Maven tools wouldn't work if the POMs are really malformed, so we do minimal error + // checking. + + var projectNode = (Element) document.getElementsByTagName("project").item(0); + if (projectNode == null) { + LOGGER.severe(String.format("Malformed pom %s does not have tag", artifactName)); + System.exit(1); + } + LOGGER.fine(String.format("loaded model for %s", artifactName)); + + var packagingNode = projectNode.getElementsByTagName("packaging").item(0); + var packaging = packagingNode == null ? "jar" : packagingNode.getTextContent(); + + if (!packaging.equals("pom")) { + artifactName = toMavenPath(groupId, artifactId, version, packaging); + if (downloadedJars.contains(artifactName)) { + LOGGER.finer(String.format("skipped already downloaded artifact %s", artifactName)); + return; + } + bytes = downloadMavenFile(repoUrl, artifactName, mvnCentralFallback); + var tmpfile = store(downloadDir, groupId, artifactId, version, bytes, packaging); + var definedModules = getModuleNamesInDirectory(downloadDir); + if (definedModules.size() > 1) { + LOGGER.severe(String.format("Internal error: more than one module in temporary directory %s", downloadDir)); + System.exit(1); + } else if (definedModules.size() == 1 && existingModules.containsAll(definedModules)) { + LOGGER.finer(String.format("skipped artifact %s, which defines module %s, because it matches an existing module in the output dir %s", + artifactName, definedModules.toArray()[0], outputDir)); + Files.delete(tmpfile); + // if the module is already there, we assume its dependencies must be too. + return; + } else { + Files.move(tmpfile, outputDir.resolve(tmpfile.getFileName()), StandardCopyOption.REPLACE_EXISTING); + } + downloadedJars.add(artifactName); + } + + var dependenciesNodes = document.getDocumentElement().getElementsByTagName("dependencies"); + for (int i = 0; i < dependenciesNodes.getLength(); i++) { + var dependenciesNode = (Element) dependenciesNodes.item(i); + var dependencyNodes = dependenciesNode.getElementsByTagName("dependency"); + for (int j = 0; j < dependencyNodes.getLength(); j++) { + var dependencyNode = (Element) dependencyNodes.item(j); + var gidNode = dependencyNode.getElementsByTagName("groupId").item(0); + if (gidNode == null) { + LOGGER.severe(String.format("Malformed pom %s, dependency does not have tag", artifactName)); + System.exit(1); + } + var gid = gidNode.getTextContent(); + var aidNode = dependencyNode.getElementsByTagName("artifactId").item(0); + if (aidNode == null) { + LOGGER.severe(String.format("Malformed pom %s, dependency does not have tag", artifactName)); + System.exit(1); + } + var aid = aidNode.getTextContent(); + var versionNode = dependencyNode.getElementsByTagName("version").item(0); + if (versionNode == null) { + LOGGER.severe(String.format("missing version for dependency %s:%s in %s", gid, aid, artifactName)); + System.exit(1); + } + var ver = versionNode.getTextContent(); + var scopeNode = dependencyNode.getElementsByTagName("scope").item(0); + if (scopeNode != null) { + var scope = scopeNode.getTextContent(); + if ("test".equals(scope) || "provided".equals(scope)) { + continue; + } + } + downloadDependencies(repoUrl, gid, aid, ver); + } + } + } + + private static byte[] downloadMavenFile(String repoUrl, String artefactName, boolean fallback) throws IOException, URISyntaxException { + String url = repoUrl + artefactName; + try { + if (url.startsWith("file:")) { + File file = new File(new URI(url)); + if (!file.exists()) { + throw new IOException(String.format("does not exist %s", url)); + } + } + URL u = new URI(url).toURL(); + byte[] bytes = downloadFromServer(u); + LOGGER.info(String.format("downloaded file %s", url)); + return bytes; + } catch (IOException ioe) { + if (fallback) { + LOGGER.log(Level.WARNING, String.format("could not download maven file from %s, because of: %s. Falling back on %s", url, ioe, DEFAULT_MAVEN_REPO)); + return downloadMavenFile(DEFAULT_MAVEN_REPO, artefactName, false); + } + LOGGER.log(Level.SEVERE, String.format("exception while downloading maven file from %s", url), ioe); + throw ioe; + } catch (URISyntaxException ex) { + LOGGER.log(Level.SEVERE, String.format("wrong url", url), ex); + throw ex; + } + } + + private static byte[] downloadFromServer(URL url) throws IOException { + URLConnection conn = url.openConnection(getProxy()); + int code = HttpURLConnection.HTTP_OK; + if (conn instanceof HttpURLConnection) { + code = ((HttpURLConnection) conn).getResponseCode(); + } + if (code != HttpURLConnection.HTTP_OK) { + throw new IOException("Skipping download from " + url + " due to response code " + code); + } + try { + try (InputStream is = conn.getInputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + byte[] buf = new byte[4096]; + int read; + while ((read = is.read(buf)) != -1) { + baos.write(buf, 0, read); + } + return baos.toByteArray(); + } + } catch (IOException ex) { + throw new IOException("Cannot download: " + url + " due to: " + ex, ex); + } + } + + private static Proxy getProxy() { + String httpProxy = getProxyVar("https_proxy"); + if (httpProxy == null || "".equals(httpProxy.trim())) { + httpProxy = getProxyVar("http_proxy"); + } + if (httpProxy == null || "".equals(httpProxy.trim())) { + LOGGER.info(String.format("using no proxy")); + return Proxy.NO_PROXY; + } + httpProxy = httpProxy.trim(); + int idx = httpProxy.lastIndexOf(":"); + if (idx < 0 || idx > httpProxy.length() - 1) { + LOGGER.warning(String.format("http_proxy env variable has to be in format host:url, but was '%s'", httpProxy)); + return Proxy.NO_PROXY; + } + String host = httpProxy.substring(0, idx); + int port; + try { + port = Integer.parseInt(httpProxy.substring(idx + 1)); + } catch (NumberFormatException e) { + LOGGER.severe(String.format("can't parse port number in '%s'", httpProxy)); + throw e; + } + LOGGER.info(String.format("using proxy '%s:%s'", host, port)); + + InetSocketAddress address = InetSocketAddress.createUnresolved(host, port); + return new Proxy(Proxy.Type.HTTP, address); + } + + private static String getProxyVar(String key) { + String var = System.getenv(key); + if (var == null) { + var = System.getenv(key.toUpperCase(Locale.ROOT)); + } + return var; + } + + private static String toMavenPath(String groupId, String artifactId, String version, String extension) { + return String.format("%s/%s/%s/%s", groupId.replace(".", "/"), artifactId, version, toArtifactFilename(artifactId, version, extension)); + } + + private static String toArtifactFilename(String artifactId, String version, String extension) { + return String.format("%s-%s.%s", artifactId, version, extension); + } + + private static Path store(Path dir, String groupId, String artifactId, String version, byte[] bytes, String extension) throws IOException { + String fileName = String.format("%s-%s", groupId, toArtifactFilename(artifactId, version, extension)); + Path path = dir.resolve(fileName); + Files.write(path, bytes); + return path; + } +} diff --git a/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/Main.java b/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/Main.java new file mode 100644 index 000000000000..c3ffbbe7e28a --- /dev/null +++ b/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/Main.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.maven.downloader; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.xml.parsers.ParserConfigurationException; + +import org.xml.sax.SAXException; + +public class Main { + + static final Logger LOGGER = Logger.getLogger(Main.class.getName()); + static final String LOGGING_PROP = "org.graalvm.maven.downloader.logLevel"; + static { + String logLevel = System.getenv(LOGGING_PROP); + if (logLevel != null) { + Level level = Level.parse(logLevel.toUpperCase()); + LOGGER.setLevel(level); + Handler[] handlers = Logger.getLogger("").getHandlers(); + for (Handler h : handlers) { + h.setLevel(level); + } + } + } + + static record Arguments(String repoUrl, String outputDir, String groupId, String artifactId, String version) { + } + + static Arguments parseArguments(String... args) { + if (args.length == 0 || "--help".equals(args[0])) { + printHelp(); + System.exit(0); + } + + String repoUrl = null; + String outputDir = null; + String groupId = null; + String artifactId = null; + String version = null; + + for (int i = 0; i < args.length; i++) { + if ('-' == args[i].charAt(0)) { + if (args[i].length() < 2) { + exitErr("Invalid argument: " + args[i]); + } + if (args.length - 1 == i) { + exitErr("Expected argument value after: " + args[i]); + } + switch (args[i].charAt(1)) { + case 'r': + repoUrl = args[i + 1]; + break; + case 'o': + outputDir = args[i + 1]; + break; + case 'g': + groupId = args[i + 1]; + break; + case 'a': + artifactId = args[i + 1]; + break; + case 'v': + version = args[i + 1]; + break; + default: + exitErr("Invalid argument: " + args[i]); + } + i++; + } else if (args.length == 1) { + artifactId = args[i]; + } else { + exitErr("Invalid argument: " + args[i]); + } + } + + if (repoUrl == null) { + repoUrl = OptionProperties.getDefaultRepo(); + } else if (!repoUrl.endsWith("/")) { + repoUrl = repoUrl + "/"; + } + if (outputDir == null) { + outputDir = OptionProperties.getDefaultOutputDir(); + } + if (groupId == null) { + groupId = OptionProperties.getDefaultGroup(); + } + if (artifactId == null) { + exitErr("Missing artifactId argument -a"); + } + if (version == null) { + version = OptionProperties.getDefaultVersion(); + } + + return new Arguments(repoUrl, outputDir, groupId, artifactId, version); + } + + public static void main(String... args) throws IOException, URISyntaxException, ClassCastException, ParserConfigurationException, SAXException, NoSuchAlgorithmException { + Arguments parsedArgs = parseArguments(args); + + if (parsedArgs.outputDir == null) { + exitErr("Missing output directory argument -o"); + } + Path outputPath = Paths.get(parsedArgs.outputDir); + if (!Files.exists(outputPath)) { + Files.createDirectory(outputPath); + } else { + if (!Files.isDirectory(outputPath)) { + exitErr("Output folder has to be a directory."); + } + } + + LOGGER.info(String.format("downloading %s:%s:%s from %s to %s", parsedArgs.groupId, parsedArgs.artifactId, parsedArgs.version, parsedArgs.repoUrl, parsedArgs.outputDir)); + + new MVNDownloader(parsedArgs.outputDir).downloadDependencies(parsedArgs.repoUrl, parsedArgs.groupId, parsedArgs.artifactId, parsedArgs.version); + } + + private static void exitErr(String err) { + System.err.println(err + "\n"); + printHelp(); + System.exit(1); + } + + private static void printHelp() { + System.out.println(String.format(""" + Usage: %s [OPTION...] + + This tool downloads maven artifacts and their dependencies to a single directory. + + There are two operating modes. + + Specifying everything needed for download and resolution: + -r maven-repository-url [default: %s] + -o output-directory [default: %s] + -g group-id [default: %s] + -v artifact-version [default: %s] + -a artifact-id + + Passing just a single argument: + This is treated as an artifact ID. The other options then use their defaults. + + + Environment options: + 'http_proxy' and 'https_proxy' are observed for proxy configuration + %s can be used to set the default version + %s is observed to configure the default maven repo + %s is observed to configure logging level + """, + OptionProperties.getExeName(), + OptionProperties.getDefaultRepo(), + OptionProperties.getDefaultOutputDir(), + OptionProperties.getDefaultGroup(), + OptionProperties.getDefaultVersion(), + OptionProperties.VERSION_PROP, + OptionProperties.MAVEN_PROP, + LOGGING_PROP)); + } +} diff --git a/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/OptionProperties.java b/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/OptionProperties.java new file mode 100644 index 000000000000..e633c8d0881b --- /dev/null +++ b/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/OptionProperties.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.maven.downloader; + +import java.nio.file.Paths; + +import org.graalvm.nativeimage.ImageInfo; +import org.graalvm.nativeimage.ProcessProperties; + +class OptionProperties { + static final String RELATIVE_OUTPUT_DIR = System.getProperty("org.graalvm.maven.downloader.relative_output_dir"); + static final String DEFAULT_VERSION = System.getProperty("org.graalvm.maven.downloader.default_version"); + static final String VERSION_PROP = "org.graalvm.maven.downloader.version"; + static final String DEFAULT_MAVEN_REPO = "https://repo1.maven.org/maven2/"; + static final String MAVEN_PROP = "org.graalvm.maven.downloader.repository"; + static final String DEFAULT_GROUP_ID = "org.graalvm.polyglot"; + + public static String getDefaultGroup() { + return DEFAULT_GROUP_ID; + } + + public static String getDefaultRepo() { + var repo = System.getenv(MAVEN_PROP); + if (repo == null) { + return DEFAULT_MAVEN_REPO; + } else { + return repo; + } + } + + public static String getDefaultVersion() { + var ver = System.getenv(VERSION_PROP); + if (ver == null) { + if (DEFAULT_VERSION != null) { + return DEFAULT_VERSION; + } else { + return "not specified"; + } + } else { + return ver; + } + } + + public static String getExeName() { + if (ImageInfo.inImageRuntimeCode()) { + if (ProcessProperties.getArgumentVectorBlockSize() > 0) { + return ProcessProperties.getArgumentVectorProgramName(); + } + } + return "maven downloader"; + } + + public static String getDefaultOutputDir() { + if (RELATIVE_OUTPUT_DIR != null) { + if (ImageInfo.inImageRuntimeCode()) { + if (ProcessProperties.getArgumentVectorBlockSize() > 0) { + String progName = ProcessProperties.getArgumentVectorProgramName(); + return Paths.get(progName).resolve("..").resolve(RELATIVE_OUTPUT_DIR).normalize().toString(); + } + } + } + return "maven downloader output"; + } +} From 2915a1059bb55d01564212190017fd36176dc5c9 Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Sun, 27 Aug 2023 16:17:40 +0200 Subject: [PATCH 15/29] Add maven deploy command. --- sdk/mx.sdk/mx_sdk.py | 36 ++++++++++++++++++++++++++++++++++++ vm/mx.vm/mx_vm.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/sdk/mx.sdk/mx_sdk.py b/sdk/mx.sdk/mx_sdk.py index f3302e6aafd5..06a51b97bfe6 100644 --- a/sdk/mx.sdk/mx_sdk.py +++ b/sdk/mx.sdk/mx_sdk.py @@ -46,6 +46,7 @@ import mx_gate import mx_sdk_vm import mx_sdk_vm_impl +import pathlib import mx_sdk_benchmark # pylint: disable=unused-import import mx_sdk_clangformat # pylint: disable=unused-import import datetime @@ -270,3 +271,38 @@ def description(self): return "GraalVM JDK" mx.addJDKFactory('graalvm', mx.get_jdk(tag='default').javaCompliance, GraalVMJDK()) + + +def maven_deploy_public_repo_dir(): + return os.path.join(_suite.get_mx_output_dir(), 'public-maven-repo') + +@mx.command(_suite.name, 'maven-deploy-public') +def maven_deploy_public(args, licenses=None): + """Helper to simplify deploying all public Maven dependendencies into the mxbuild directory""" + artifact_version = mx_sdk_vm_impl.graalvm_version('graalvm') + path = maven_deploy_public_repo_dir() + mx.rmtree(path, ignore_errors=True) + os.mkdir(path) + + if not licenses: + # default licenses used + licenses = ['GFTC', 'EPL-2.0', 'PSF-License', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-simplified', 'BSD-new', 'UPL', 'MIT'] + + deploy_args = [ + '--tags=public', + '--all-suites', + '--all-distribution-types', + f'--version-string={artifact_version}', + '--validate=full', + '--licenses', ','.join(licenses), + 'local', + pathlib.Path(path).as_uri(), + ] + if mx.get_jdk().javaCompliance > '17': + mx.warn("Javadoc won't be deployed as a JDK > 17 is not yet compatible with javadoc doclets. In order to deploy javadoc use a JDK 17 as a JDK on JAVA_HOME.") + deploy_args += ["--suppress-javadoc"] + + mx.log(f'mx maven-deploy {" ".join(deploy_args)}') + mx.maven_deploy(deploy_args) + mx.log(f'Deployed Maven artefacts to {path}') + return path diff --git a/vm/mx.vm/mx_vm.py b/vm/mx.vm/mx_vm.py index 2581f62d2144..e404b2dcfb74 100644 --- a/vm/mx.vm/mx_vm.py +++ b/vm/mx.vm/mx_vm.py @@ -249,8 +249,8 @@ def mx_post_parse_cmd_line(args): # When this list is changed, the _enterprise_languages_meta_distributions # in the mx_vm_enterprise.py must also be updated. _community_languages_meta_distributions = [ - 'espresso:JAVA_COMMUNITY', 'graal-js:JS_COMMUNITY', + 'espresso:JAVA_COMMUNITY', 'graalpython:PYTHON_COMMUNITY', 'sulong:LLVM_NATIVE_COMMUNITY', 'sulong:LLVM_COMMUNITY', From dbbf92465444f726ab95a32a1a4d819e5671e7ed Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 25 Aug 2023 13:13:30 +0200 Subject: [PATCH 16/29] add test for maven downloader --- vm/ci/ci_includes/vm-native.jsonnet | 11 +++++ vm/mx.vm/mx_vm_gate.py | 48 +++++++++++++++++++ .../maven/downloader/MVNDownloader.java | 6 ++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/vm/ci/ci_includes/vm-native.jsonnet b/vm/ci/ci_includes/vm-native.jsonnet index dcaa7692525a..ca19bb6178ae 100644 --- a/vm/ci/ci_includes/vm-native.jsonnet +++ b/vm/ci/ci_includes/vm-native.jsonnet @@ -43,6 +43,17 @@ local vm_common = import '../ci_common/common.jsonnet'; timelimit: '30:00', name: 'gate-vm-ce-truffle-unchained-linux-amd64', }, + vm.vm_java_21 + vm_common.svm_common_linux_amd64 + vm_common.sulong_linux + vm_common.gate_vm_linux_amd64 + { + run+: [ + ['export', 'SVM_SUITE=' + vm.svm_suite], + ['mx', '--env', 'ce-llvm', '--native-images=', 'gate', '--no-warning-as-error', '--tags', 'build,maven-downloader'], + ], + timelimit: '30:00', + name: 'gate-vm-ce-truffle-maven-downloader-linux-amd64', + packages+: { + maven: '>=3.3.9', + }, + }, ], builds: [{'defined_in': std.thisFile} + b for b in builds], diff --git a/vm/mx.vm/mx_vm_gate.py b/vm/mx.vm/mx_vm_gate.py index 86fa9a0eacd5..939a562be24b 100644 --- a/vm/mx.vm/mx_vm_gate.py +++ b/vm/mx.vm/mx_vm_gate.py @@ -30,6 +30,7 @@ import mx import mx_subst import mx_unittest +import mx_sdk import mx_sdk_vm import mx_sdk_vm_impl @@ -73,6 +74,7 @@ class VmGateTasks: svm_truffle_tck_js = 'svm-truffle-tck-js' svm_truffle_tck_python = 'svm-truffle-tck-python' truffle_unchained = 'truffle-unchained' + maven_downloader = 'maven-downloader' def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config @@ -553,6 +555,7 @@ def gate_body(args, tasks): gate_svm_truffle_tck_js(tasks) gate_svm_truffle_tck_python(tasks) gate_truffle_unchained(tasks) + gate_maven_downloader(tasks) def graalvm_svm(): """ @@ -774,6 +777,51 @@ def gate_truffle_unchained(tasks): mx.abort("Cannot resolve truffle suite.") mx_truffle.sl_native_optimized_gate_tests() +def gate_maven_downloader(tasks): + with Task('Maven Downloader prepare maven repo', tasks, tags=[VmGateTasks.maven_downloader]) as t: + if t: + mx.suite('sulong') + mx_sdk.maven_deploy_public([], licenses=['EPL-2.0', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-new', 'UPL', 'MIT']) + mx.build(["--dep", "MAVEN_DOWNLOADER"]) + jdk = mx.get_jdk() + mvnDownloader = mx.distribution("MAVEN_DOWNLOADER") + vm_args = mx.get_runtime_jvm_args([mvnDownloader], jdk=jdk) + vm_args.append(mvnDownloader.mainClass) + output_dir = os.path.join(_suite.get_mx_output_dir(), 'downloaded-mvn-modules') + shutil.rmtree(output_dir, ignore_errors=True) + env = os.environ.copy() + if mx._opts.verbose: + env["org.graalvm.maven.downloader.logLevel"] = "ALL" + mvntoolargs = [ + "-r", f"file://{abspath(mx_sdk.maven_deploy_public_repo_dir())}", + "-o", output_dir, + "-v", mx_sdk_vm_impl.graalvm_version('graalvm'), + "-a", + ] + mx.run_java(vm_args + mvntoolargs + ["polyglot"], jdk=jdk, env=env) + # now we should have all jars for llvm-language in the output dir + entries = os.listdir(output_dir) + for entry in entries: + if isdir(entry): + mx.abort("We do not expect the maven downloader to create directories") + os.rename(join(output_dir, entry), join(output_dir, f"first_download_{entry}")) + if not any(re.search("polyglot.*\\.jar", entry) for entry in entries): + mx.abort("We expected polyglot to have been downloaded") + if any(re.search("llvm.*\\.jar", entry) for entry in entries): + mx.abort("We did not expect llvm to have been downloaded") + # now we download something higher up the dependency tree, to check + # that existing files (even though renamed) are not clobbered and + # no duplicate modules are placed in the output directory + mx.run_java(vm_args + mvntoolargs + ["llvm-community"], jdk=jdk, env=env) + entries = os.listdir(output_dir) + for entry in entries: + if isdir(entry): + mx.abort("We do not expect the maven downloader to create directories") + if f"first_download_{entry}" in entries: + mx.abort("We do not expect the maven downloader to download modules that are already there") + if not any(re.search("llvm-language-native.*\\.jar", entry) for entry in entries): + mx.abort("We did not expect llvm-language-native to have been downloaded via llvm-community meta pom") + def build_tests_image(image_dir, options, unit_tests=None, additional_deps=None, shared_lib=False): native_image_context, svm = graalvm_svm() with native_image_context(svm.IMAGE_ASSERTION_FLAGS) as native_image: diff --git a/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/MVNDownloader.java b/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/MVNDownloader.java index 6eb771c4a608..532eb2bf6437 100644 --- a/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/MVNDownloader.java +++ b/vm/src/org.graalvm.maven.downloader/src/org/graalvm/maven/downloader/MVNDownloader.java @@ -244,7 +244,11 @@ private static Proxy getProxy() { throw e; } LOGGER.info(String.format("using proxy '%s:%s'", host, port)); - + if (host.startsWith("http://")) { + host = host.substring("http://".length()); + } else if (host.startsWith("https://")) { + host = host.substring("https://".length()); + } InetSocketAddress address = InetSocketAddress.createUnresolved(host, port); return new Proxy(Proxy.Type.HTTP, address); } From 57f2e105cf55fa4b4ef4c5e4b80be2d553d85238 Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Thu, 24 Aug 2023 23:42:55 +0200 Subject: [PATCH 17/29] Improve tracing in class isolation. --- .../src/org/graalvm/polyglot/Engine.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java index 47b4d1f00823..5d748a595bc3 100644 --- a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java +++ b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java @@ -1864,6 +1864,10 @@ record ParsedModule(Path p, Set modules) { Set modules = finder.findAll(); if (modules.size() > 0) { parsedModules.add(new ParsedModule(path, modules)); + } else { + if (TRACE_CLASS_PATH_ISOLATION) { + trace("No modules found in class-path entry %s", path); + } } } catch (FindException t) { // error in module finding -> not a valid module descriptor ignore @@ -1922,7 +1926,15 @@ record ParsedModule(Path p, Set modules) { if (includedModules.contains(d.name())) { usedServices.addAll(d.uses()); for (Requires r : d.requires()) { - if (!r.modifiers().contains(Requires.Modifier.STATIC) && includedModules.add(r.name())) { + /* + * We deliberately follow static resources here, even though they + * are not real dependencies in the module-graph. If we don't follow + * static requires we might fallback to the parent class-loader for + * these classes causing weird problems. So if the module is on the + * class-path and there is a static requires we pick it up into the + * module-graph well. + */ + if (includedModules.add(r.name())) { if (TRACE_CLASS_PATH_ISOLATION) { trace("Include module '%s' because it is required by '%s'.", r.name(), d.name()); } @@ -1940,6 +1952,9 @@ record ParsedModule(Path p, Set modules) { for (ModuleReference ref : module.modules()) { String name = ref.descriptor().name(); if (!includedModules.contains(name)) { + if (TRACE_CLASS_PATH_ISOLATION) { + trace("Filter module '%s' because not reachable on the module graph.", name); + } continue; } @@ -1950,10 +1965,8 @@ record ParsedModule(Path p, Set modules) { continue; } - if (includedModules.contains(name)) { - filteredList.add(module.p()); - break; - } + filteredList.add(module.p()); + break; } } From c7c40249428c122de348771659bc45f055c06a73 Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Fri, 1 Sep 2023 11:23:26 +0200 Subject: [PATCH 18/29] Fix invalid resolution relative path resources. --- .../com/oracle/truffle/polyglot/InternalResourceCache.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java index 0d496e68637b..c11c4e98c098 100644 --- a/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java +++ b/truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/InternalResourceCache.java @@ -303,7 +303,11 @@ private static Path findCacheRootOnNativeImage() { if (res == null) { assert ImageInfo.inImageRuntimeCode() : "Can be called only in the native-image execution time."; Path executable = Path.of(ProcessProperties.getExecutableName()); - Path cache = executable.resolve("resources"); + Path parent = executable.getParent(); + if (parent == null) { + throw new IllegalStateException("Could not locate native-image resources."); + } + Path cache = parent.resolve("resources"); res = Pair.create(cache, false); cacheRoot = res; } From 3e64f82e52480be84ffe4d17066d37843dc06475 Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Fri, 25 Aug 2023 00:02:07 +0200 Subject: [PATCH 19/29] Migrate tools native-image configuration. --- tools/mx.tools/tools-chromeinspector.properties | 3 --- tools/mx.tools/tools-coverage.properties | 2 -- tools/mx.tools/tools-dap.properties | 2 -- tools/mx.tools/tools-insight.properties | 3 --- tools/mx.tools/tools-profiler.properties | 2 -- .../native-image.properties | 3 +++ .../com.oracle.truffle.tools.profiler/native-image.properties | 2 ++ .../com.oracle.truffle.tools.dap/native-image.properties | 2 ++ .../com.oracle.truffle.tools.profiler/native-image.properties | 2 ++ .../org.graalvm.tools.insight.heap/native-image.properties | 2 ++ .../org.graalvm.tools.insight/native-image.properties | 3 +++ .../native-image/org.graalvm.tools.lsp/native-image.properties | 2 ++ 12 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 tools/src/com.oracle.truffle.tools.chromeinspector/src/META-INF/native-image/com.oracle.truffle.tools.chromeinspector/native-image.properties create mode 100644 tools/src/com.oracle.truffle.tools.coverage/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties create mode 100644 tools/src/com.oracle.truffle.tools.dap/src/META-INF/native-image/com.oracle.truffle.tools.dap/native-image.properties create mode 100644 tools/src/com.oracle.truffle.tools.profiler/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties create mode 100644 tools/src/org.graalvm.tools.insight.heap/src/META-INF/native-image/org.graalvm.tools.insight.heap/native-image.properties create mode 100644 tools/src/org.graalvm.tools.insight/src/META-INF/native-image/org.graalvm.tools.insight/native-image.properties create mode 100644 tools/src/org.graalvm.tools.lsp/src/META-INF/native-image/org.graalvm.tools.lsp/native-image.properties diff --git a/tools/mx.tools/tools-chromeinspector.properties b/tools/mx.tools/tools-chromeinspector.properties index 59daf868f260..4af1899e35b6 100644 --- a/tools/mx.tools/tools-chromeinspector.properties +++ b/tools/mx.tools/tools-chromeinspector.properties @@ -1,4 +1 @@ Requires = tool:profiler language:truffle-json -Args = --initialize-at-build-time=com.oracle.truffle.tools.chromeinspector \ - -H:MaxRuntimeCompileMethods=300 - diff --git a/tools/mx.tools/tools-coverage.properties b/tools/mx.tools/tools-coverage.properties index 24049163e862..ebe4c73b37d1 100644 --- a/tools/mx.tools/tools-coverage.properties +++ b/tools/mx.tools/tools-coverage.properties @@ -1,3 +1 @@ Requires = macro:truffle language:truffle-json -Args = --initialize-at-build-time=com.oracle.truffle.tools.coverage \ - -H:MaxRuntimeCompileMethods=100 diff --git a/tools/mx.tools/tools-dap.properties b/tools/mx.tools/tools-dap.properties index f00d3e764608..ebe4c73b37d1 100644 --- a/tools/mx.tools/tools-dap.properties +++ b/tools/mx.tools/tools-dap.properties @@ -1,3 +1 @@ Requires = macro:truffle language:truffle-json -Args = --initialize-at-build-time=com.oracle.truffle.tools.dap \ - -H:MaxRuntimeCompileMethods=100 diff --git a/tools/mx.tools/tools-insight.properties b/tools/mx.tools/tools-insight.properties index 3af290d61d56..e69de29bb2d1 100644 --- a/tools/mx.tools/tools-insight.properties +++ b/tools/mx.tools/tools-insight.properties @@ -1,3 +0,0 @@ -Args = --initialize-at-build-time=com.oracle.truffle.tools.agentscript \ - --initialize-at-build-time=org.graalvm.tools.insight \ - -H:MaxRuntimeCompileMethods=300 diff --git a/tools/mx.tools/tools-profiler.properties b/tools/mx.tools/tools-profiler.properties index 21d928095401..ebe4c73b37d1 100644 --- a/tools/mx.tools/tools-profiler.properties +++ b/tools/mx.tools/tools-profiler.properties @@ -1,3 +1 @@ Requires = macro:truffle language:truffle-json -Args = --initialize-at-build-time=com.oracle.truffle.tools.profiler \ - -H:MaxRuntimeCompileMethods=100 diff --git a/tools/src/com.oracle.truffle.tools.chromeinspector/src/META-INF/native-image/com.oracle.truffle.tools.chromeinspector/native-image.properties b/tools/src/com.oracle.truffle.tools.chromeinspector/src/META-INF/native-image/com.oracle.truffle.tools.chromeinspector/native-image.properties new file mode 100644 index 000000000000..f27837ccc580 --- /dev/null +++ b/tools/src/com.oracle.truffle.tools.chromeinspector/src/META-INF/native-image/com.oracle.truffle.tools.chromeinspector/native-image.properties @@ -0,0 +1,3 @@ +Args = --initialize-at-build-time=com.oracle.truffle.tools.chromeinspector \ + -H:MaxRuntimeCompileMethods=300 + diff --git a/tools/src/com.oracle.truffle.tools.coverage/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties b/tools/src/com.oracle.truffle.tools.coverage/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties new file mode 100644 index 000000000000..59b62dadd5d8 --- /dev/null +++ b/tools/src/com.oracle.truffle.tools.coverage/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties @@ -0,0 +1,2 @@ +Args = --initialize-at-build-time=com.oracle.truffle.tools.profiler \ + -H:MaxRuntimeCompileMethods=100 \ No newline at end of file diff --git a/tools/src/com.oracle.truffle.tools.dap/src/META-INF/native-image/com.oracle.truffle.tools.dap/native-image.properties b/tools/src/com.oracle.truffle.tools.dap/src/META-INF/native-image/com.oracle.truffle.tools.dap/native-image.properties new file mode 100644 index 000000000000..e4e83e2efeab --- /dev/null +++ b/tools/src/com.oracle.truffle.tools.dap/src/META-INF/native-image/com.oracle.truffle.tools.dap/native-image.properties @@ -0,0 +1,2 @@ +Args = --initialize-at-build-time=com.oracle.truffle.tools.dap \ + -H:MaxRuntimeCompileMethods=100 diff --git a/tools/src/com.oracle.truffle.tools.profiler/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties b/tools/src/com.oracle.truffle.tools.profiler/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties new file mode 100644 index 000000000000..59b62dadd5d8 --- /dev/null +++ b/tools/src/com.oracle.truffle.tools.profiler/src/META-INF/native-image/com.oracle.truffle.tools.profiler/native-image.properties @@ -0,0 +1,2 @@ +Args = --initialize-at-build-time=com.oracle.truffle.tools.profiler \ + -H:MaxRuntimeCompileMethods=100 \ No newline at end of file diff --git a/tools/src/org.graalvm.tools.insight.heap/src/META-INF/native-image/org.graalvm.tools.insight.heap/native-image.properties b/tools/src/org.graalvm.tools.insight.heap/src/META-INF/native-image/org.graalvm.tools.insight.heap/native-image.properties new file mode 100644 index 000000000000..fbf28b18e7ce --- /dev/null +++ b/tools/src/org.graalvm.tools.insight.heap/src/META-INF/native-image/org.graalvm.tools.insight.heap/native-image.properties @@ -0,0 +1,2 @@ +Args = --initialize-at-build-time=org.graalvm.tools.insight.heap \ + -H:MaxRuntimeCompileMethods=100 diff --git a/tools/src/org.graalvm.tools.insight/src/META-INF/native-image/org.graalvm.tools.insight/native-image.properties b/tools/src/org.graalvm.tools.insight/src/META-INF/native-image/org.graalvm.tools.insight/native-image.properties new file mode 100644 index 000000000000..3af290d61d56 --- /dev/null +++ b/tools/src/org.graalvm.tools.insight/src/META-INF/native-image/org.graalvm.tools.insight/native-image.properties @@ -0,0 +1,3 @@ +Args = --initialize-at-build-time=com.oracle.truffle.tools.agentscript \ + --initialize-at-build-time=org.graalvm.tools.insight \ + -H:MaxRuntimeCompileMethods=300 diff --git a/tools/src/org.graalvm.tools.lsp/src/META-INF/native-image/org.graalvm.tools.lsp/native-image.properties b/tools/src/org.graalvm.tools.lsp/src/META-INF/native-image/org.graalvm.tools.lsp/native-image.properties new file mode 100644 index 000000000000..066b4e4be66b --- /dev/null +++ b/tools/src/org.graalvm.tools.lsp/src/META-INF/native-image/org.graalvm.tools.lsp/native-image.properties @@ -0,0 +1,2 @@ +Args = --initialize-at-build-time=org.graalvm.tools.lsp,org.graalvm.tools.api.lsp \ + -H:MaxRuntimeCompileMethods=100 From 6812f70ec081791f7b59bee77b9c6a5eb3fe400d Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Mon, 28 Aug 2023 19:04:28 +0200 Subject: [PATCH 20/29] Allow disabling of polyglot implementations. --- .../src/org/graalvm/polyglot/Engine.java | 5 +++++ .../com/oracle/truffle/runtime/OptimizedTruffleRuntime.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java index 5d748a595bc3..2095439c4dfa 100644 --- a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java +++ b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java @@ -1632,6 +1632,11 @@ private static AbstractPolyglotImpl loadAndValidateProviders(Iterator> getLookupTypes() { * the {@link TruffleCompiler} with {@link #getTruffleCompiler(TruffleCompilable) * getTruffleCompiler}. */ - protected final String getCompilerConfigurationName() { + public final String getCompilerConfigurationName() { return compilationSupport.getCompilerConfigurationName(this); } From 13b00c4cbf12cc40e9ceb66f24a499589d5a212e Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Wed, 30 Aug 2023 14:23:21 +0200 Subject: [PATCH 21/29] Update python. --- vm/mx.vm/suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/mx.vm/suite.py b/vm/mx.vm/suite.py index 51cca1ab39fd..65ee51cb84c4 100644 --- a/vm/mx.vm/suite.py +++ b/vm/mx.vm/suite.py @@ -65,7 +65,7 @@ }, { "name": "graalpython", - "version": "f42da6d958171905ddc00e26bd7b0ce52dca732a", + "version": "cbb9985782bf0f68969ae6f2eb6101fd02eb4db4", "dynamic": True, "urls": [ {"url": "https://github.com/graalvm/graalpython.git", "kind": "git"}, From 4f45801bcd55827b1806c014d829b1d75046eadc Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Thu, 31 Aug 2023 23:34:43 +0200 Subject: [PATCH 22/29] Fix problem with WindowsLibraryLocator when library is an internal resource. --- .../llvm/parser/coff/WindowsLibraryLocator.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sulong/projects/com.oracle.truffle.llvm.parser/src/com/oracle/truffle/llvm/parser/coff/WindowsLibraryLocator.java b/sulong/projects/com.oracle.truffle.llvm.parser/src/com/oracle/truffle/llvm/parser/coff/WindowsLibraryLocator.java index ffc31b8ed9a3..c0904f915272 100644 --- a/sulong/projects/com.oracle.truffle.llvm.parser/src/com/oracle/truffle/llvm/parser/coff/WindowsLibraryLocator.java +++ b/sulong/projects/com.oracle.truffle.llvm.parser/src/com/oracle/truffle/llvm/parser/coff/WindowsLibraryLocator.java @@ -42,7 +42,16 @@ public final class WindowsLibraryLocator extends LibraryLocator { Path libraryPath; public WindowsLibraryLocator(Source source) { - libraryPath = source != null ? Paths.get(source.getPath()).getParent() : null; + if (source == null || source.isInternal()) { + /* + * No need to search the current file directory for internal sources. Those can always + * be located anyway. Also Paths.get doesn't work if the source path is an internal + * resources inside a jar. + */ + libraryPath = null; + } else { + libraryPath = Paths.get(source.getPath()).getParent(); + } } @Override From 15f6d3b1ae378bb8bb29ec1a63661e8bee472247 Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Fri, 1 Sep 2023 14:22:34 +0200 Subject: [PATCH 23/29] Fix some limited eager class loading might now be triggered if enterprise truffle is on the classpath. --- .../compiler/truffle/test/LazyClassLoadingTest.java | 12 ++++++++++++ .../hotspot/CompilerConfigurationFactory.java | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/compiler/src/jdk.internal.vm.compiler.test/src/org/graalvm/compiler/truffle/test/LazyClassLoadingTest.java b/compiler/src/jdk.internal.vm.compiler.test/src/org/graalvm/compiler/truffle/test/LazyClassLoadingTest.java index 65d2a6c37ff2..a42a9c283623 100644 --- a/compiler/src/jdk.internal.vm.compiler.test/src/org/graalvm/compiler/truffle/test/LazyClassLoadingTest.java +++ b/compiler/src/jdk.internal.vm.compiler.test/src/org/graalvm/compiler/truffle/test/LazyClassLoadingTest.java @@ -41,10 +41,14 @@ import org.graalvm.compiler.core.GraalCompilerOptions; import org.graalvm.compiler.core.common.util.Util; import org.graalvm.compiler.debug.Assertions; +import org.graalvm.compiler.hotspot.CommunityCompilerConfigurationFactory; +import org.graalvm.compiler.hotspot.CompilerConfigurationFactory; +import org.graalvm.compiler.hotspot.EconomyCompilerConfigurationFactory; import org.graalvm.compiler.nodes.Cancellable; import org.graalvm.compiler.options.OptionDescriptor; import org.graalvm.compiler.options.OptionDescriptors; import org.graalvm.compiler.options.OptionKey; +import org.graalvm.compiler.options.OptionStability; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.options.OptionsParser; import org.graalvm.compiler.test.SubprocessUtil; @@ -206,6 +210,14 @@ private List filterGraalCompilerClasses(List loadedGraalClassNam } } + // classes needed to find out whether enterprise is installed in the JDK + allowList.add(OptionStability.class); + allowList.add(CompilerConfigurationFactory.class); + allowList.add(CompilerConfigurationFactory.Options.class); + allowList.add(CompilerConfigurationFactory.ShowConfigurationLevel.class); + allowList.add(EconomyCompilerConfigurationFactory.class); + allowList.add(CommunityCompilerConfigurationFactory.class); + allowList.add(Cancellable.class); List forbiddenClasses = new ArrayList<>(); diff --git a/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java b/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java index ed724bf32ea3..fed9e284ab02 100644 --- a/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java +++ b/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java @@ -60,13 +60,13 @@ */ public abstract class CompilerConfigurationFactory implements Comparable { - enum ShowConfigurationLevel { + public enum ShowConfigurationLevel { none, info, verbose } - static class Options { + public static class Options { // @formatter:off @Option(help = "Names the compiler configuration to use. If omitted, the compiler configuration " + "with the highest auto-selection priority is used. To see the set of available configurations, " + From aeaae1715e08aa57d5f689ea5751b040e77583ee Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Fri, 1 Sep 2023 15:41:20 +0200 Subject: [PATCH 24/29] Fix PolyglotInvalid should not use Integer.MIN_VALUE to avoid getting filtered. --- .../org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java index 2095439c4dfa..ef687a978cad 100644 --- a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java +++ b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java @@ -2026,7 +2026,8 @@ private static class PolyglotInvalid extends AbstractPolyglotImpl { @Override public int getPriority() { - return Integer.MIN_VALUE; + // make sure polyglot invalid has lowest priority but is not filtered (hence + 1) + return Integer.MIN_VALUE + 1; } @Override From 33475f54be7a69a7ca2e3fde29cfd2812b644fc8 Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Fri, 1 Sep 2023 19:21:47 +0200 Subject: [PATCH 25/29] Fix use snapshot for maven deployment. --- sdk/mx.sdk/mx_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/mx.sdk/mx_sdk.py b/sdk/mx.sdk/mx_sdk.py index 06a51b97bfe6..a1210e8469d4 100644 --- a/sdk/mx.sdk/mx_sdk.py +++ b/sdk/mx.sdk/mx_sdk.py @@ -279,7 +279,7 @@ def maven_deploy_public_repo_dir(): @mx.command(_suite.name, 'maven-deploy-public') def maven_deploy_public(args, licenses=None): """Helper to simplify deploying all public Maven dependendencies into the mxbuild directory""" - artifact_version = mx_sdk_vm_impl.graalvm_version('graalvm') + artifact_version = mx_sdk_vm_impl.graalvm_version('graalvm') + '-SNAPSHOT' path = maven_deploy_public_repo_dir() mx.rmtree(path, ignore_errors=True) os.mkdir(path) From f41efe80401590fbd58d342c6108fa1239f14ef7 Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Fri, 1 Sep 2023 19:22:25 +0200 Subject: [PATCH 26/29] Allow truffle to be loaded without a jvmci JDK. --- .../src/org/graalvm/polyglot/Engine.java | 19 ++++++++++++++++++- .../src/com/oracle/truffle/api/Truffle.java | 9 ++++++++- .../hotspot/HotSpotTruffleRuntimeAccess.java | 7 ++++--- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java index ef687a978cad..63745fcae96e 100644 --- a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java +++ b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/Engine.java @@ -1708,6 +1708,9 @@ private Iterator searchServiceLoader() throws In private static class ClassPathIsolation { private static final String TRUFFLE_MODULE_NAME = "org.graalvm.truffle"; + private static final String TRUFFLE_RUNTIME_MODULE_NAME = "org.graalvm.truffle.runtime"; + private static final String JVMCI_MODULE_NAME = "jdk.internal.vm.ci"; + private static final String TRUFFLE_ENTERPRISE_MODULE_NAME = "com.oracle.truffle.enterprise"; private static final String POLYGLOT_MODULE_NAME = "org.graalvm.polyglot"; private static final String OPTION_DISABLE_CLASS_PATH_ISOLATION = "polyglotimpl.DisableClassPathIsolation"; private static final String OPTION_TRACE_CLASS_PATH_ISOLATION = "polyglotimpl.TraceClassPathIsolation"; @@ -1786,7 +1789,6 @@ static Module createIsolatedTruffleModule(Class polyglotClass) { trace("Class-path entry: %s", p); } } - List relevantPaths = filterClasspath(parent, classpath); if (relevantPaths == null) { if (TRACE_CLASS_PATH_ISOLATION) { @@ -1906,6 +1908,16 @@ record ParsedModule(Path p, Set modules) { return null; } + Set excludedModules; + if (ModuleLayer.boot().findModule(JVMCI_MODULE_NAME).isPresent()) { + excludedModules = Set.of(); + } else { + /* + * if jdk.internal.vm.ci is not enabled we can't load these modules on the layer. + */ + excludedModules = Set.of(TRUFFLE_RUNTIME_MODULE_NAME, TRUFFLE_ENTERPRISE_MODULE_NAME); + } + // now iteratively resolve modules until no more modules are included List toProcess = new ArrayList<>(parsedModules); Set usedServices = new HashSet<>(); @@ -1918,6 +1930,11 @@ record ParsedModule(Path p, Set modules) { ParsedModule module = modules.next(); for (ModuleReference m : module.modules) { ModuleDescriptor d = m.descriptor(); + if (excludedModules.contains(d.name())) { + modules.remove(); + continue; + } + for (Provides p : d.provides()) { if (usedServices.contains(p.service())) { if (includedModules.add(d.name())) { diff --git a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/Truffle.java b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/Truffle.java index 258e70f88931..97a7cb50a359 100644 --- a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/Truffle.java +++ b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/Truffle.java @@ -144,7 +144,14 @@ private static TruffleRuntime createRuntime() throws InternalError { return runtime; } } - return new DefaultTruffleRuntime("No optimizing Truffle runtime found on the module-path."); + + String reason; + if (ModuleLayer.boot().findModule("jdk.internal.vm.ci").isPresent()) { + reason = "No optimizing Truffle runtime found on the module or class-path."; + } else { + reason = "JVMCI is required to enable optimizations. Pass -XX:+EnableJVMCI as a virtual machine argument to the java executable to resolve this."; + } + return new DefaultTruffleRuntime(reason); } private static void maybeExportJVMCITo(Class runtimeClass) throws ReflectiveOperationException { diff --git a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/HotSpotTruffleRuntimeAccess.java b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/HotSpotTruffleRuntimeAccess.java index 52ded49dffcf..b15389ffc8fd 100644 --- a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/HotSpotTruffleRuntimeAccess.java +++ b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/HotSpotTruffleRuntimeAccess.java @@ -79,7 +79,8 @@ protected static TruffleRuntime createRuntime() { } catch (Error e) { // hack to detect the exact error that is thrown when if (e.getClass() == Error.class && e.getMessage().startsWith("The EnableJVMCI VM option must be true")) { - return new DefaultTruffleRuntime("JVMCI is not enabled on this JVM. JVMCI may be enabled using -XX:+EnableJVMCI. This is necessary on JVMs that do not enable JVMCI by default."); + return new DefaultTruffleRuntime( + "JVMCI is required to enable optimizations. Pass -XX:+EnableJVMCI as a virtual machine argument to the java executable to resolve this. This is necessary on JVMs that do not enable JVMCI by default."); } throw e; } @@ -88,7 +89,7 @@ protected static TruffleRuntime createRuntime() { boolean useCompiler = config.getFlag("UseCompiler", Boolean.class); if (!useCompiler) { // compilation disabled in host VM -> fallback to default runtime - return new DefaultTruffleRuntime("JVMCI compilation was disabled on this JVM. JVMCI may be enabled using -XX:+EnableJVMCI."); + return new DefaultTruffleRuntime("JVMCI compilation was disabled on this JVM. Pass -XX:+EnableJVMCI as a virtual machine argument to the java executable to resolve this."); } /* * Module integrity rules ignore qualified exports from a parent module layer to a child @@ -117,7 +118,7 @@ protected static TruffleRuntime createRuntime() { if (compilerModule == null) { // jargraal compiler module not found -> fallback to default runtime return new DefaultTruffleRuntime( - "Libgraal compilation is not available on this JVM. Alternatively, the compiler module jdk.internal.vm.compiler was not found on the --upgrade-module-path."); + "Libgraal compilation is not available on this JVM. Alternatively, the org.graalvm.compiler:compiler module can be put on the --upgrade-module-path."); } /* * That the compiler has a qualified export to Truffle may not be enough if truffle From 3e1b1ad86672704fba0f858c2a4502221b73de6c Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Fri, 1 Sep 2023 23:42:18 +0200 Subject: [PATCH 27/29] Sync common.jssonnet. --- vm/ci/ci_common/common.jsonnet | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 5b76e51cc6cb..fe48e662e569 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -359,6 +359,10 @@ local devkits = graal_common.devkits; weekly_vm_windows: self.vm_windows + { targets+: ['weekly'], }, + + weekly_vm_windows_amd64: self.vm_windows + { + targets+: ['weekly'], + }, weekly_vm_windows_jdk17: self.vm_windows_jdk17 + { targets+: ['weekly'], From 1a19ae7b224f7788078dea7b182ad7527d9e53a4 Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Sat, 2 Sep 2023 10:48:13 +0200 Subject: [PATCH 28/29] Fix potential shutdown in progress. --- .../hotspot/libgraal/LibGraalObject.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/libgraal/LibGraalObject.java b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/libgraal/LibGraalObject.java index 2be214f75e43..7081586763b2 100644 --- a/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/libgraal/LibGraalObject.java +++ b/truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/libgraal/LibGraalObject.java @@ -53,12 +53,17 @@ public class LibGraalObject { static { if (LibGraal.isAvailable()) { LibGraal.registerNativeMethods(LibGraalObject.class); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - exiting = true; - } - }); + try { + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + exiting = true; + } + }); + } catch (IllegalStateException e) { + // shutdown already in progress + // catching the exception is the only way to detect this. + } } } From af188fd04a35906cf2efb29586faf2989985633e Mon Sep 17 00:00:00 2001 From: Christian Humer Date: Sun, 3 Sep 2023 13:24:00 +0200 Subject: [PATCH 29/29] Support deploying regular versions in addition to snapshots for the maven downloader. --- sdk/mx.sdk/mx_sdk.py | 7 +++++-- vm/mx.vm/mx_vm_gate.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk.py b/sdk/mx.sdk/mx_sdk.py index a1210e8469d4..f1d7af815f51 100644 --- a/sdk/mx.sdk/mx_sdk.py +++ b/sdk/mx.sdk/mx_sdk.py @@ -277,9 +277,12 @@ def maven_deploy_public_repo_dir(): return os.path.join(_suite.get_mx_output_dir(), 'public-maven-repo') @mx.command(_suite.name, 'maven-deploy-public') -def maven_deploy_public(args, licenses=None): +def maven_deploy_public(args, licenses=None, deploy_snapshots=True): """Helper to simplify deploying all public Maven dependendencies into the mxbuild directory""" - artifact_version = mx_sdk_vm_impl.graalvm_version('graalvm') + '-SNAPSHOT' + if deploy_snapshots: + artifact_version = f'{mx_sdk_vm_impl.graalvm_version("graalvm")}-SNAPSHOT' + else: + artifact_version = f'{mx_sdk_vm_impl.graalvm_version("graalvm")}' path = maven_deploy_public_repo_dir() mx.rmtree(path, ignore_errors=True) os.mkdir(path) diff --git a/vm/mx.vm/mx_vm_gate.py b/vm/mx.vm/mx_vm_gate.py index 939a562be24b..3752d9f13eeb 100644 --- a/vm/mx.vm/mx_vm_gate.py +++ b/vm/mx.vm/mx_vm_gate.py @@ -781,7 +781,7 @@ def gate_maven_downloader(tasks): with Task('Maven Downloader prepare maven repo', tasks, tags=[VmGateTasks.maven_downloader]) as t: if t: mx.suite('sulong') - mx_sdk.maven_deploy_public([], licenses=['EPL-2.0', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-new', 'UPL', 'MIT']) + mx_sdk.maven_deploy_public([], licenses=['EPL-2.0', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-new', 'UPL', 'MIT'], deploy_snapshots=False) mx.build(["--dep", "MAVEN_DOWNLOADER"]) jdk = mx.get_jdk() mvnDownloader = mx.distribution("MAVEN_DOWNLOADER")