From f6c50f0da5904f8c75398259874f373faa11e778 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 17 Oct 2023 16:51:27 +0200 Subject: [PATCH 01/78] Add add_defined_in utility function --- ci/ci_common/common-utils.libsonnet | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/ci_common/common-utils.libsonnet b/ci/ci_common/common-utils.libsonnet index 19bc64d3d9bc..673bb3865830 100644 --- a/ci/ci_common/common-utils.libsonnet +++ b/ci/ci_common/common-utils.libsonnet @@ -26,6 +26,11 @@ else build, + # Adds a 'defined_in' key to all jobs in the list. + # Due to the nature of std.thisFile, the file name has to be explicitly passed. + # Usage: add_defined_in(builds, std.thisFile) + add_defined_in(builds, file):: [{ defined_in: file } + b for b in builds], + # Returns true if `str` contains `needle` as a substring. contains(str, needle):: std.findSubstr(needle, str) != [], From bec5f1baa31211c0c4f4b67dea3b967349e0b4ae Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 17 Oct 2023 18:02:39 +0200 Subject: [PATCH 02/78] Use add_defined_in utility in all CI jobs --- ci_includes/publish-javadoc.jsonnet | 2 +- compiler/ci/ci_common/benchmark-builders.jsonnet | 3 ++- compiler/ci/ci_includes/baseline-benchmarks.jsonnet | 3 ++- substratevm/ci/ci.jsonnet | 2 +- vm/ci/ci_common/common-bench.jsonnet | 3 ++- vm/ci/ci_common/common.jsonnet | 3 ++- vm/ci/ci_includes/vm-bench.jsonnet | 5 +++-- vm/ci/ci_includes/vm-native.jsonnet | 3 ++- vm/ci/ci_includes/vm.jsonnet | 2 +- 9 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ci_includes/publish-javadoc.jsonnet b/ci_includes/publish-javadoc.jsonnet index a64e0eef6eaa..65be4f0c8f89 100644 --- a/ci_includes/publish-javadoc.jsonnet +++ b/ci_includes/publish-javadoc.jsonnet @@ -62,5 +62,5 @@ common.post_merge + linux_amd64 + common.labsjdk21 + javadoc_publisher, ], // adds a "defined_in" field to all builds mentioning the location of this current file - builds:: [{ defined_in: std.thisFile } + b for b in all_builds] + builds:: utils.add_defined_in(all_builds, std.thisFile), } diff --git a/compiler/ci/ci_common/benchmark-builders.jsonnet b/compiler/ci/ci_common/benchmark-builders.jsonnet index b9e21d5a688c..71408f6e67e8 100644 --- a/compiler/ci/ci_common/benchmark-builders.jsonnet +++ b/compiler/ci/ci_common/benchmark-builders.jsonnet @@ -1,5 +1,6 @@ { local c = (import '../../../ci/ci_common/common.jsonnet'), + local utils = (import '../../../ci/ci_common/common-utils.libsonnet'), local bc = (import '../../../ci/ci_common/bench-common.libsonnet'), local cc = (import 'compiler-common.libsonnet'), local bench = (import 'benchmark-suites.libsonnet'), @@ -91,5 +92,5 @@ local all_builds = main_builds + weekly_amd64_forks_builds + weekly_aarch64_forks_builds + profiling_builds + avx_builds + zgc_builds + zgc_avx_builds + aarch64_builds + no_tiered_builds + no_profile_info_builds, local filtered_builds = [b for b in all_builds if b.is_jdk_supported(b.jdk_version) && b.is_arch_supported(b.arch)], // adds a "defined_in" field to all builds mentioning the location of this current file - builds:: [{ defined_in: std.thisFile } + b for b in filtered_builds] + builds:: utils.add_defined_in(filtered_builds, std.thisFile), } diff --git a/compiler/ci/ci_includes/baseline-benchmarks.jsonnet b/compiler/ci/ci_includes/baseline-benchmarks.jsonnet index 16f6b468064a..57f4c503cfb3 100644 --- a/compiler/ci/ci_includes/baseline-benchmarks.jsonnet +++ b/compiler/ci/ci_includes/baseline-benchmarks.jsonnet @@ -1,5 +1,6 @@ { local c = (import '../../../ci/ci_common/common.jsonnet'), + local utils = (import '../../../ci/ci_common/common-utils.libsonnet'), local bc = (import '../../../ci/ci_common/bench-common.libsonnet'), local cc = (import '../ci_common/compiler-common.libsonnet'), local bench = (import '../ci_common/benchmark-suites.libsonnet'), @@ -79,5 +80,5 @@ local filtered_builds = [b for b in all_builds if b.is_jdk_supported(b.jdk_version) && b.is_arch_supported(b.arch)], // adds a "defined_in" field to all builds mentioning the location of this current file - builds:: [{ defined_in: std.thisFile } + b for b in filtered_builds] + builds:: utils.add_defined_in(filtered_builds, std.thisFile), } diff --git a/substratevm/ci/ci.jsonnet b/substratevm/ci/ci.jsonnet index c69709ebeb5a..934381c07417 100644 --- a/substratevm/ci/ci.jsonnet +++ b/substratevm/ci/ci.jsonnet @@ -130,6 +130,6 @@ }, // END MAIN BUILD DEFINITION processed_builds::run_spec.process(task_dict), - builds: [{'defined_in': std.thisFile} + util.add_gate_predicate(b, gate_triggering_suites) for b in self.processed_builds.list], + builds: util.add_defined_in([util.add_gate_predicate(b, gate_triggering_suites) for b in self.processed_builds.list], std.thisFile), assert tools.check_names($.builds), } diff --git a/vm/ci/ci_common/common-bench.jsonnet b/vm/ci/ci_common/common-bench.jsonnet index a3a8a1428f0c..c79e172d5232 100644 --- a/vm/ci/ci_common/common-bench.jsonnet +++ b/vm/ci/ci_common/common-bench.jsonnet @@ -1,5 +1,6 @@ # note: this file needs to be in sync between CE and EE +local utils = import '../../../ci/ci_common/common-utils.libsonnet'; local vm = import '../ci_includes/vm.jsonnet'; local common = import '../../../ci/ci_common/common.jsonnet'; local vm_common = import '../ci_common/common.jsonnet'; @@ -322,5 +323,5 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet'; vm_common.gate_vm_linux_amd64 + self.vm_gate_polybench_linux + {name: 'gate-vm-' + vm.vm_setup.short_name + '-polybench-linux-amd64'}, ], - builds: [{'defined_in': std.thisFile} + b for b in builds], + builds: utils.add_defined_in(builds, std.thisFile), } diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 9dc6d7f375fe..55873d1a6d99 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -1,5 +1,6 @@ local vm = import '../ci_includes/vm.jsonnet'; local graal_common = import '../../../ci/ci_common/common.jsonnet'; +local utils = import '../../../ci/ci_common/common-utils.libsonnet'; local repo_config = import '../../../repo-configuration.libsonnet'; local devkits = graal_common.devkits; @@ -857,5 +858,5 @@ local devkits = graal_common.devkits; vm.vm_java_Latest + sulong_vm_tests, ] + (import 'libgraal.jsonnet').builds, - builds:: [{'defined_in': std.thisFile} + b for b in builds], + builds:: utils.add_defined_in(builds, std.thisFile), } diff --git a/vm/ci/ci_includes/vm-bench.jsonnet b/vm/ci/ci_includes/vm-bench.jsonnet index bdd1215ac269..84d43f916cf8 100644 --- a/vm/ci/ci_includes/vm-bench.jsonnet +++ b/vm/ci/ci_includes/vm-bench.jsonnet @@ -1,3 +1,4 @@ +local utils = import '../../../ci/ci_common/common-utils.libsonnet'; local vm = import 'vm.jsonnet'; local vm_common = import '../ci_common/common.jsonnet'; local vm_common_bench = import '../ci_common/common-bench.jsonnet'; @@ -5,5 +6,5 @@ local vm_common_bench = import '../ci_common/common-bench.jsonnet'; { local builds = [], - builds: [{'defined_in': std.thisFile} + b for b in builds], -} \ No newline at end of file + builds: utils.add_defined_in(builds, std.thisFile) +} diff --git a/vm/ci/ci_includes/vm-native.jsonnet b/vm/ci/ci_includes/vm-native.jsonnet index 1ffd8fdae4d5..d2012f7ef46e 100644 --- a/vm/ci/ci_includes/vm-native.jsonnet +++ b/vm/ci/ci_includes/vm-native.jsonnet @@ -1,3 +1,4 @@ +local utils = import '../../../ci/ci_common/common-utils.libsonnet'; local vm = import 'vm.jsonnet'; local vm_common = import '../ci_common/common.jsonnet'; @@ -74,5 +75,5 @@ local vm_common = import '../ci_common/common.jsonnet'; vm.vm_java_Latest + vm_common.gate_vm_linux_amd64 + truffle_maven_downloader, ], - builds: [{'defined_in': std.thisFile} + b for b in builds], + builds: utils.add_defined_in(builds, std.thisFile), } diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 810ad59fc9d4..6532360042b2 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -224,7 +224,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; self.notify_releaser_build, ], - builds: [vm_common.verify_name(b1) for b1 in vm_common.builds + vm_common_bench.builds + vm_bench.builds + vm_native.builds + [{'defined_in': std.thisFile} + b2 for b2 in builds]], + builds: [vm_common.verify_name(b) for b in vm_common.builds + vm_common_bench.builds + vm_bench.builds + vm_native.builds + utils.add_defined_in(builds, std.thisFile)], compiler_gate:: (import '../../../compiler/ci/ci_common/gate.jsonnet') } From 147686f65a340e4db3c027ff753d4ddec3580992 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 17 Oct 2023 19:15:41 +0200 Subject: [PATCH 03/78] Require notify email for (opt-)post-merge --- ci/ci_common/ci-check.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/ci_common/ci-check.libsonnet b/ci/ci_common/ci-check.libsonnet index 4f945e78b482..0ae1ad090fb3 100644 --- a/ci/ci_common/ci-check.libsonnet +++ b/ci/ci_common/ci-check.libsonnet @@ -3,7 +3,7 @@ local std_get = (import 'common-utils.libsonnet').std_get; { # check that all non [gate, ondemand] entries have notify_emails or notify_groups defined local missing_notify(builds) = { - [x.name]: std_get(x, "defined_in") for x in builds if !std.objectHas(x, "notify_emails") && !std.objectHasAll(x, "notify_groups") && (std.member(x.targets, "daily") || std.member(x.targets, "weekly") || std.member(x.targets, "monthly")) + [x.name]: std_get(x, "defined_in") for x in builds if !std.objectHas(x, "notify_emails") && !std.objectHasAll(x, "notify_groups") && std.length(std.setInter(std.set(x.targets), std.set(["daily", "weekly", "monthly", "post-merge", "opt-post-merge"]))) != 0 }, verify_ci(builds):: From ca37886a18525959323256556f6ae4b07813d861 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 17 Oct 2023 19:21:02 +0200 Subject: [PATCH 04/78] Add notify_group for post-merge-vm-update-stable-mx-branch-linux-amd64 --- vm/ci/ci_includes/vm.jsonnet | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 6532360042b2..31b37dc31999 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -179,7 +179,8 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; ['set-export', 'BRANCH_NAME', ['git', 'rev-parse', '--abbrev-ref', 'HEAD']], ['bash', '-c', 'if [[ ${BRANCH_NAME} == master ]] || [[ ${BRANCH_NAME} == release/* ]] || [[ ${BRANCH_NAME} == cpu/* ]]; then git -C ${MX_HOME} push origin +HEAD:refs/heads/graal/${BRANCH_NAME}; fi'] ], - name: 'post-merge-vm-update-stable-mx-branch-linux-amd64', + name: 'post-merge-vm-update-stable-mx-branch-linux-amd64', + notify_groups:: ['deploy'], }, From 275fd59aa17f2e05313e027c1a3f1e15c3e529ba Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 18 Oct 2023 16:14:07 +0200 Subject: [PATCH 05/78] Update overlay revision --- graal-common.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graal-common.json b/graal-common.json index d97c3c5039d5..fb26b4262ed0 100644 --- a/graal-common.json +++ b/graal-common.json @@ -1,6 +1,6 @@ { "README": "This file contains definitions that are useful for the jsonnet CI files of the graal and graal-enterprise repositories.", "ci": { - "overlay": "f197d31e391f68e62e7b7897e8e161b50e5e880d" + "overlay": "5b25b545821271da526df7d67d88876b89841102" } } From 3d5792e5081c038b6793cab3a7a2dab4c8d4f76b Mon Sep 17 00:00:00 2001 From: Raphael Mosaner Date: Wed, 13 Sep 2023 17:30:44 +0200 Subject: [PATCH 06/78] [GR-48430] Do not let NotCompiledExceptionHandler deopts float. --- .../src/jdk/graal/compiler/nodes/DeoptimizeNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/DeoptimizeNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/DeoptimizeNode.java index 61b72464b3c5..130e67616f95 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/DeoptimizeNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/DeoptimizeNode.java @@ -138,7 +138,7 @@ public boolean canFloat() { * high and would be executed under the wrong conditions. */ public static boolean canFloat(DeoptimizationReason reason, DeoptimizationAction action) { - return action != DeoptimizationAction.None && reason != DeoptimizationReason.Unresolved; + return action != DeoptimizationAction.None && reason != DeoptimizationReason.Unresolved && reason != DeoptimizationReason.NotCompiledExceptionHandler; } @NodeIntrinsic From 9c41a661875b2f5d82211577811e4456bd336168 Mon Sep 17 00:00:00 2001 From: Raphael Mosaner Date: Wed, 13 Sep 2023 17:31:46 +0200 Subject: [PATCH 07/78] [GR-48430] Fix affected unittests. --- .../graal/compiler/truffle/test/FrameHostReadsTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/FrameHostReadsTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/FrameHostReadsTest.java index 1a9a94a0bbc1..46292eae5599 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/FrameHostReadsTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/FrameHostReadsTest.java @@ -70,7 +70,6 @@ public void test0() { int fieldReads = 0; int arrayReads = 0; int arrayLengthReads = 0; - int otherReads = 0; for (ReadNode read : graph.getNodes(ReadNode.TYPE)) { LocationIdentity identity = read.getLocationIdentity(); @@ -81,7 +80,10 @@ public void test0() { } else if (identity == NamedLocationIdentity.ARRAY_LENGTH_LOCATION) { arrayLengthReads++; } else { - otherReads++; + /* + * [GR-48430] Since NotCompiledExceptionHandler deopts are not turned into guards + * anymore, remaining reads belong to the explicit handling of the assertion. + */ } } @@ -105,7 +107,6 @@ public void test0() { * FrameWithoutBoxing.indexedTags and one for FrameWithoutBoxing.indexedPrimitiveLocals. */ Assert.assertEquals(2, arrayLengthReads); - Assert.assertEquals(0, otherReads); } @SuppressWarnings({"serial"}) From 280a22f58ab0e88ef33e8d66f333189a6433654d Mon Sep 17 00:00:00 2001 From: Raphael Mosaner Date: Fri, 15 Sep 2023 13:34:06 +0200 Subject: [PATCH 08/78] [GR-48430] Enable ConvertDeoptimizeToGuard for non-floating deopts. --- .../compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java index 71d2709a4190..ac192632fdd6 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java @@ -110,9 +110,6 @@ protected void run(final StructuredGraph graph, final CoreProviders context) { for (DeoptimizeNode d : graph.getNodes(DeoptimizeNode.TYPE)) { assert d.isAlive(); - if (!d.canFloat()) { - continue; - } try (DebugCloseable closable = d.withNodeSourcePosition()) { propagateFixed(d, d, context, lazyLoops); } From 49df5b1a0b7e4810bf459430d8eae249e49ac228 Mon Sep 17 00:00:00 2001 From: Raphael Mosaner Date: Fri, 15 Sep 2023 13:37:42 +0200 Subject: [PATCH 09/78] [GR-48430] Fix unittests. --- .../hotspot/test/Base64SubstitutionsTest.java | 23 ++----------------- .../truffle/test/FrameHostReadsTest.java | 7 +++--- 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/Base64SubstitutionsTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/Base64SubstitutionsTest.java index c11d10acdcdb..7a5662149e06 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/Base64SubstitutionsTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/Base64SubstitutionsTest.java @@ -38,9 +38,6 @@ import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; - -import jdk.graal.compiler.core.common.GraalOptions; -import jdk.graal.compiler.options.OptionValues; import org.junit.Assert; import org.junit.Test; @@ -141,20 +138,6 @@ public void testEncodeByteArray1() { } } - /** - * The test at - * https://github.com/openjdk/jdk/blob/f2922682688a40529df269e1551246ac8da5d7ee/src/java.base/share/classes/java/util/Base64.java#L878 - * is converted to a guard and (too eagerly) floated outside the loop, leading to deopt even - * when the input byte array has a correct ending. - * - * GR-48430. - */ - private static OptionValues workaroundTooEagerDeopt() { - return new OptionValues(getInitialOptions(), - GraalOptions.OptConvertDeoptsToGuards, false, - GraalOptions.SpeculativeGuardMovement, false); - } - /** * Tests {@link Encoder#encode(byte[], byte[])}. */ @@ -297,7 +280,6 @@ public void testDecodeByteArray1() { */ @Test public void testDecodeByteArray2() { - OptionValues options = workaroundTooEagerDeopt(); ResolvedJavaMethod m = getResolvedJavaMethod(Decoder.class, "decode", byte[].class, byte[].class); for (DecoderTestCase tc : getDecoders()) { for (int i = 0; i < PLAIN_TEXT_BYTES.length; i++) { @@ -305,7 +287,7 @@ public void testDecodeByteArray2() { // JDK-8273108: Test for output buffer overrun byte[] suffix = {0, (byte) 167}; ByteArraySupplier bas = new ByteArraySupplier(srcBytes.length, suffix); - test(options, m, tc.decoder, srcBytes, bas); + test(m, tc.decoder, srcBytes, bas); Assert.assertEquals(bas.supplied.size(), 2); byte[] expect = Arrays.copyOfRange(bas.supplied.get(0), 0, srcBytes.length); byte[] actual = Arrays.copyOfRange(bas.supplied.get(1), 0, srcBytes.length); @@ -328,12 +310,11 @@ public static byte[] decodeByteBufferSnippet(Decoder decoder, ByteBuffer srcBuf) */ @Test public void testDecodeByteBuffer() { - OptionValues options = workaroundTooEagerDeopt(); for (DecoderTestCase tc : getDecoders()) { for (int i = 0; i < PLAIN_TEXT_BYTES.length; i++) { byte[] srcBytes = tc.encoded[i]; ByteBuffer srcBuf = ByteBuffer.wrap(srcBytes); - test(options, "decodeByteBufferSnippet", tc.decoder, srcBuf); + test("decodeByteBufferSnippet", tc.decoder, srcBuf); } } } diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/FrameHostReadsTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/FrameHostReadsTest.java index 46292eae5599..1a9a94a0bbc1 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/FrameHostReadsTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/FrameHostReadsTest.java @@ -70,6 +70,7 @@ public void test0() { int fieldReads = 0; int arrayReads = 0; int arrayLengthReads = 0; + int otherReads = 0; for (ReadNode read : graph.getNodes(ReadNode.TYPE)) { LocationIdentity identity = read.getLocationIdentity(); @@ -80,10 +81,7 @@ public void test0() { } else if (identity == NamedLocationIdentity.ARRAY_LENGTH_LOCATION) { arrayLengthReads++; } else { - /* - * [GR-48430] Since NotCompiledExceptionHandler deopts are not turned into guards - * anymore, remaining reads belong to the explicit handling of the assertion. - */ + otherReads++; } } @@ -107,6 +105,7 @@ public void test0() { * FrameWithoutBoxing.indexedTags and one for FrameWithoutBoxing.indexedPrimitiveLocals. */ Assert.assertEquals(2, arrayLengthReads); + Assert.assertEquals(0, otherReads); } @SuppressWarnings({"serial"}) From b3a9f78d849f89ff76e04beb824b43b852ff16f5 Mon Sep 17 00:00:00 2001 From: Raphael Mosaner Date: Tue, 19 Sep 2023 14:11:41 +0200 Subject: [PATCH 10/78] [GR-48430] Fix unittests affected by allowing non-floatable deopts to be converted to fixed guards. --- .../test/ConditionalEliminationTest11.java | 7 ++++-- .../test/ConditionalEliminationTest9.java | 8 +++---- .../test/IfNodeCanonicalizationTest.java | 9 ++++---- .../test/TransferToInterpreterTest.java | 3 ++- .../compiler/nodes/loop/CountedLoopInfo.java | 22 ++++++++++++++----- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/ConditionalEliminationTest11.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/ConditionalEliminationTest11.java index c081a0c660bb..a2d8b841a632 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/ConditionalEliminationTest11.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/ConditionalEliminationTest11.java @@ -126,7 +126,7 @@ public void test5() { public static int test6Snippet(int a) { if ((a & 8) != 0) { - GraalDirectives.deoptimize(); + GraalDirectives.deoptimizeAndInvalidate(); } if ((a & 15) != 15) { GraalDirectives.deoptimize(); @@ -135,8 +135,11 @@ public static int test6Snippet(int a) { } public static int reference6Snippet(int a) { + /* + * first guard needs higher priority than second guard, otherwise code folds to second guard + */ if ((a & 8) != 0) { - GraalDirectives.deoptimize(); + GraalDirectives.deoptimizeAndInvalidate(); } GraalDirectives.deoptimize(); return 0; diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/ConditionalEliminationTest9.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/ConditionalEliminationTest9.java index 5e7c6f178b68..b3b208bdf426 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/ConditionalEliminationTest9.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/ConditionalEliminationTest9.java @@ -27,8 +27,6 @@ import jdk.graal.compiler.phases.common.ConditionalEliminationPhase; import org.junit.Test; -import jdk.graal.compiler.api.directives.GraalDirectives; - /** * Collection of tests for {@link ConditionalEliminationPhase} including those that triggered bugs * in this phase. @@ -40,7 +38,7 @@ public class ConditionalEliminationTest9 extends ConditionalEliminationTestBase @SuppressWarnings("all") public static int referenceSnippet(int a) { if (a == 0) { - GraalDirectives.deoptimize(); + return 1; } return 0; } @@ -54,10 +52,10 @@ public void test1() { public static int test1Snippet(int a) { if (a == 0) { if (a == 0) { - GraalDirectives.deoptimize(); + return 1; } if (a == 0) { - GraalDirectives.deoptimize(); + return 2; } } return 0; diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/nodes/test/IfNodeCanonicalizationTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/nodes/test/IfNodeCanonicalizationTest.java index 1a4061e2f6a9..57730d3959fc 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/nodes/test/IfNodeCanonicalizationTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/nodes/test/IfNodeCanonicalizationTest.java @@ -24,7 +24,6 @@ */ package jdk.graal.compiler.nodes.test; -import jdk.graal.compiler.api.directives.GraalDirectives; import jdk.graal.compiler.core.test.GraalCompilerTest; import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; @@ -123,10 +122,10 @@ public void test4() { public boolean testSnippet4(int a, int[] limit) { int l = limit.length; if (a < 0) { - GraalDirectives.deoptimize(); + return false; } if (a >= l) { - GraalDirectives.deoptimize(); + return false; } return true; } @@ -140,10 +139,10 @@ public void test5() { public boolean testSnippet5(int a, int[] limit) { int l = limit.length; if (a >= l) { - GraalDirectives.deoptimize(); + return false; } if (a < 0) { - GraalDirectives.deoptimize(); + return false; } return true; } diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/TransferToInterpreterTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/TransferToInterpreterTest.java index 6d33bfb5363f..f111e5fd5ce4 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/TransferToInterpreterTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/TransferToInterpreterTest.java @@ -90,7 +90,8 @@ public Object execute(VirtualFrame frame) { int x = (int) frame.getArguments()[0]; if (x == 0) { CompilerDirectives.transferToInterpreter(); - } else { + } + if (x == 1) { CompilerDirectives.transferToInterpreterAndInvalidate(); } return null; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/CountedLoopInfo.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/CountedLoopInfo.java index a93c026423d4..3954c647136a 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/CountedLoopInfo.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/CountedLoopInfo.java @@ -35,6 +35,7 @@ import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.nodes.AbstractBeginNode; import jdk.graal.compiler.nodes.ConstantNode; +import jdk.graal.compiler.nodes.FixedGuardNode; import jdk.graal.compiler.nodes.GuardNode; import jdk.graal.compiler.nodes.IfNode; import jdk.graal.compiler.nodes.LogicConstantNode; @@ -310,14 +311,23 @@ private ValueNode findOrCreatePositivePi(LogicNode noEntryCheck, ValueNode div, } HIRBlock loopBlock = cfg.blockFor(loop.loopBegin()); for (Node checkUsage : noEntryCheck.usages()) { - if (checkUsage instanceof IfNode) { - IfNode ifCheck = (IfNode) checkUsage; - if (cfg.getNodeToBlock().isNew(ifCheck.falseSuccessor())) { + ValueNode candidateCheck = null; + if (checkUsage instanceof IfNode ifCheck) { + candidateCheck = ifCheck.falseSuccessor(); + } else if (checkUsage instanceof FixedGuardNode guard) { + if (!guard.isNegated()) { continue; } - if (cfg.blockFor(ifCheck.falseSuccessor()).dominates(loopBlock)) { - return graph.addOrUniqueWithInputs(PiNode.create(div, positiveIntStamp.improveWith(div.stamp(NodeView.DEFAULT)), ifCheck.falseSuccessor())); - } + candidateCheck = guard; + } else { + continue; + } + + if (cfg.getNodeToBlock().isNew(candidateCheck)) { + continue; + } + if (cfg.blockFor(candidateCheck).dominates(loopBlock)) { + return graph.addOrUniqueWithInputs(PiNode.create(div, positiveIntStamp.improveWith(div.stamp(NodeView.DEFAULT)), candidateCheck)); } } return null; From 872a4ab695e11ecc7fb35772e3a4504b526c8963 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Tue, 10 Oct 2023 15:43:27 +0200 Subject: [PATCH 11/78] Use common pool for analysis and compilation. --- .../pointsto/standalone/PointsToAnalyzer.java | 5 +- .../StandalonePointsToAnalysis.java | 10 +-- .../pointsto/AbstractAnalysisEngine.java | 5 +- .../graal/pointsto/PointsToAnalysis.java | 30 ++----- .../pointsto/heap/HeapSnapshotVerifier.java | 5 +- .../pointsto/util/CompletionExecutor.java | 60 ++----------- .../ReachabilityAnalysisEngine.java | 5 +- .../src/com/oracle/objectfile/ObjectFile.java | 3 +- .../graal/llvm/LLVMNativeImageCodeCache.java | 8 +- .../core/graal/llvm/LLVMToolchainUtils.java | 5 +- .../graal/llvm/objectfile/LLVMObjectFile.java | 5 +- .../com/oracle/svm/core/SubstrateOptions.java | 11 +++ .../com/oracle/svm/driver/NativeImage.java | 12 +-- .../svm/hosted/HostedConfiguration.java | 5 +- .../svm/hosted/NativeImageGenerator.java | 89 +++++++++---------- .../hosted/NativeImageGeneratorRunner.java | 26 +----- .../oracle/svm/hosted/NativeImageOptions.java | 42 +-------- .../oracle/svm/hosted/ProgressReporter.java | 6 +- .../analysis/NativeImagePointsToAnalysis.java | 6 +- ...NativeImageReachabilityAnalysisEngine.java | 19 ++-- .../oracle/svm/hosted/code/CompileQueue.java | 5 +- .../svm/hosted/image/AbstractImage.java | 31 +++---- .../hosted/image/LIRNativeImageCodeCache.java | 3 +- .../oracle/svm/hosted/image/NativeImage.java | 5 +- .../hosted/image/NativeImageCodeCache.java | 15 ++-- .../svm/hosted/image/NativeImageViaCC.java | 5 +- .../hosted/image/SharedLibraryImageViaCC.java | 5 +- 27 files changed, 140 insertions(+), 286 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/PointsToAnalyzer.java b/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/PointsToAnalyzer.java index 5e5b4dad7517..8785e202b5cf 100644 --- a/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/PointsToAnalyzer.java +++ b/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/PointsToAnalyzer.java @@ -37,7 +37,6 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ForkJoinPool; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.debug.DebugContext; @@ -52,7 +51,6 @@ import com.oracle.graal.pointsto.AnalysisObjectScanningObserver; import com.oracle.graal.pointsto.AnalysisPolicy; -import com.oracle.graal.pointsto.PointsToAnalysis; import com.oracle.graal.pointsto.api.PointstoOptions; import com.oracle.graal.pointsto.flow.context.bytecode.BytecodeSensitiveAnalysisPolicy; import com.oracle.graal.pointsto.heap.HeapSnapshotVerifier; @@ -135,7 +133,6 @@ private PointsToAnalyzer(String mainEntryClass, OptionValues options) { SnippetReflectionProvider snippetReflection = originalProviders.getSnippetReflection(); MetaAccessProvider originalMetaAccess = originalProviders.getMetaAccess(); debugContext = new DebugContext.Builder(options, new GraalDebugHandlersFactory(snippetReflection)).build(); - ForkJoinPool executor = PointsToAnalysis.createExecutor(debugContext, Math.min(Runtime.getRuntime().availableProcessors(), 32)); StandaloneHost standaloneHost = new StandaloneHost(options, analysisClassLoader); int wordSize = getWordSize(); AnalysisPolicy analysisPolicy = PointstoOptions.AllocationSiteSensitiveHeap.getValue(options) ? new BytecodeSensitiveAnalysisPolicy(options) @@ -154,7 +151,7 @@ private PointsToAnalyzer(String mainEntryClass, OptionValues options) { originalProviders.getPlatformConfigurationProvider(), aMetaAccessExtensionProvider, originalProviders.getLoopsDataProvider()); standaloneHost.initializeProviders(aProviders); analysisName = getAnalysisName(mainEntryClass); - bigbang = new StandalonePointsToAnalysis(options, aUniverse, standaloneHost, aMetaAccess, snippetReflection, aConstantReflection, aProviders.getWordTypes(), executor, new TimerCollection()); + bigbang = new StandalonePointsToAnalysis(options, aUniverse, standaloneHost, aMetaAccess, snippetReflection, aConstantReflection, aProviders.getWordTypes(), new TimerCollection()); standaloneHost.setImageName(analysisName); aUniverse.setBigBang(bigbang); ImageHeap heap = new ImageHeap(); diff --git a/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/StandalonePointsToAnalysis.java b/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/StandalonePointsToAnalysis.java index 439ada3ef91a..8e769b0bde7d 100644 --- a/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/StandalonePointsToAnalysis.java +++ b/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/StandalonePointsToAnalysis.java @@ -28,7 +28,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.options.OptionValues; @@ -48,12 +47,9 @@ public class StandalonePointsToAnalysis extends PointsToAnalysis { private final Set addedClinits = ConcurrentHashMap.newKeySet(); - public StandalonePointsToAnalysis(OptionValues options, AnalysisUniverse universe, HostVM hostVM, - AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, - ForkJoinPool executorService, TimerCollection timerCollection) { - super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, executorService, new UnsupportedFeatures(), timerCollection, - true); + public StandalonePointsToAnalysis(OptionValues options, AnalysisUniverse universe, HostVM hostVM, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, TimerCollection timerCollection) { + super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, new UnsupportedFeatures(), timerCollection, true); } @Override diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java index 5bad5d06986f..687995ec9550 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java @@ -27,7 +27,6 @@ import java.io.PrintWriter; import java.util.Collections; import java.util.List; -import java.util.concurrent.ForkJoinPool; import java.util.function.Function; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; @@ -101,7 +100,7 @@ public abstract class AbstractAnalysisEngine implements BigBang { @SuppressWarnings("this-escape") public AbstractAnalysisEngine(OptionValues options, AnalysisUniverse universe, HostVM hostVM, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, ForkJoinPool executorService, UnsupportedFeatures unsupportedFeatures, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, TimerCollection timerCollection) { this.options = options; this.universe = universe; @@ -110,7 +109,7 @@ public AbstractAnalysisEngine(OptionValues options, AnalysisUniverse universe, H this.metaAccess = metaAccess; this.analysisPolicy = universe.analysisPolicy(); this.hostVM = hostVM; - this.executor = new CompletionExecutor(this, executorService); + this.executor = new CompletionExecutor(this); this.unsupportedFeatures = unsupportedFeatures; this.processFeaturesTimer = timerCollection.get(TimerCollection.Registry.FEATURES); diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java index 9b782a19ea96..5c6af4e84765 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java @@ -44,15 +44,6 @@ import java.util.function.Consumer; import java.util.stream.StreamSupport; -import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; -import jdk.graal.compiler.core.common.SuppressFBWarnings; -import jdk.graal.compiler.debug.DebugContext; -import jdk.graal.compiler.debug.Indent; -import jdk.graal.compiler.graph.Node; -import jdk.graal.compiler.graph.NodeList; -import jdk.graal.compiler.options.OptionValues; -import jdk.graal.compiler.word.WordTypes; - import com.oracle.graal.pointsto.api.HostVM; import com.oracle.graal.pointsto.api.PointstoOptions; import com.oracle.graal.pointsto.constraints.UnsupportedFeatures; @@ -86,6 +77,13 @@ import com.oracle.svm.util.ClassUtil; import com.oracle.svm.util.ImageGeneratorThreadMarker; +import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; +import jdk.graal.compiler.debug.DebugContext; +import jdk.graal.compiler.debug.Indent; +import jdk.graal.compiler.graph.Node; +import jdk.graal.compiler.graph.NodeList; +import jdk.graal.compiler.options.OptionValues; +import jdk.graal.compiler.word.WordTypes; import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.JavaType; @@ -111,9 +109,9 @@ public abstract class PointsToAnalysis extends AbstractAnalysisEngine { @SuppressWarnings("this-escape") public PointsToAnalysis(OptionValues options, AnalysisUniverse universe, HostVM hostVM, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, ForkJoinPool executorService, UnsupportedFeatures unsupportedFeatures, TimerCollection timerCollection, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, TimerCollection timerCollection, boolean strengthenGraalGraphs) { - super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, executorService, unsupportedFeatures, timerCollection); + super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, timerCollection); this.typeFlowTimer = timerCollection.createTimer("(typeflow)"); this.strengthenGraalGraphs = strengthenGraalGraphs; @@ -537,16 +535,6 @@ public boolean doTypeflow() throws InterruptedException { return didSomeWork; } - @SuppressFBWarnings(value = "NP_NONNULL_PARAM_VIOLATION", justification = "ForkJoinPool does support null for the exception handler.") - public static ForkJoinPool createExecutor(DebugContext debug, int numberOfThreads) { - ForkJoinPool.ForkJoinWorkerThreadFactory factory = debugThreadFactory(debug.areScopesEnabled() || debug.areMetricsEnabled() ? debug : null); - return new ForkJoinPool(numberOfThreads, factory, null, false); - } - - private static ForkJoinPool.ForkJoinWorkerThreadFactory debugThreadFactory(DebugContext debug) { - return pool -> new SubstrateWorkerThread(pool, debug); - } - @Override public void onTypeInstantiated(AnalysisType type, AnalysisType.UsageKind usageKind) { /* Register the type as instantiated with all its super types. */ diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java index 13f22c94df56..e79f60ca0a6b 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java @@ -27,7 +27,6 @@ import static com.oracle.graal.pointsto.ObjectScanner.ScanReason; import java.util.Objects; -import java.util.concurrent.ForkJoinPool; import java.util.function.Consumer; import jdk.graal.compiler.options.Option; @@ -74,8 +73,8 @@ public HeapSnapshotVerifier(BigBang bb, ImageHeap imageHeap, ImageHeapScanner sc verbosity = Options.HeapVerifierVerbosity.getValue(bb.getOptions()); } - public boolean checkHeapSnapshot(UniverseMetaAccess metaAccess, ForkJoinPool threadPool, String stage) { - CompletionExecutor executor = new CompletionExecutor(bb, threadPool); + public boolean checkHeapSnapshot(UniverseMetaAccess metaAccess, String stage) { + CompletionExecutor executor = new CompletionExecutor(bb); executor.init(); return checkHeapSnapshot(metaAccess, executor, stage, false); } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java index bfbecae57b10..6435a701c420 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -64,7 +63,7 @@ private enum State { private List postedBeforeStart; private final CopyOnWriteArrayList exceptions = new CopyOnWriteArrayList<>(); - private ExecutorService executorService; + private final ForkJoinPool executorService; private final BigBang bb; private Timing timing; @@ -82,9 +81,9 @@ public interface Timing { void print(); } - public CompletionExecutor(BigBang bb, ForkJoinPool forkJoin) { + public CompletionExecutor(BigBang bb) { this.bb = bb; - executorService = forkJoin; + executorService = ForkJoinPool.commonPool(); state = new AtomicReference<>(State.UNUSED); postedOperations = new LongAdder(); completedOperations = new LongAdder(); @@ -146,18 +145,7 @@ public void execute(DebugContextRunnable command) { if (timing != null) { timing.addScheduled(command); } - - if (isSequential()) { - bb.getHostVM().recordActivity(); - try (DebugContext debug = command.getDebug(bb.getOptions(), bb.getDebugHandlerFactories()); - Scope s = debug.scope("Operation")) { - command.run(debug); - } - completedOperations.increment(); - } else { - executeService(command); - } - + executeService(command); break; default: throw JVMCIError.shouldNotReachHere(); @@ -165,9 +153,7 @@ public void execute(DebugContextRunnable command) { } private void executeService(DebugContextRunnable command) { - executorService.execute(() -> { - executeCommand(command); - }); + executorService.execute(() -> executeCommand(command)); } @SuppressWarnings("try") @@ -212,14 +198,6 @@ private void setState(State newState) { } public long complete() throws InterruptedException { - - if (isSequential()) { - long completed = completedOperations.sum(); - long posted = postedOperations.sum(); - assert completed == posted : completed + ", " + posted; - return posted; - } - long lastPrint = 0; if (timing != null) { timing.printHeader(); @@ -230,12 +208,7 @@ public long complete() throws InterruptedException { while (true) { assert state.get() == State.STARTED : state.get(); - boolean quiescent; - if (executorService instanceof ForkJoinPool) { - quiescent = ((ForkJoinPool) executorService).awaitQuiescence(100, TimeUnit.MILLISECONDS); - } else { - quiescent = executorService.awaitTermination(100, TimeUnit.MILLISECONDS); - } + boolean quiescent = executorService.awaitQuiescence(100, TimeUnit.MILLISECONDS); if (timing != null && !quiescent) { long curTime = System.nanoTime(); if (curTime - lastPrint > timing.getPrintIntervalNanos()) { @@ -265,12 +238,8 @@ public long getPostedOperations() { return postedOperations.sum() + (postedBeforeStart == null ? 0 : postedBeforeStart.size()); } - public boolean isSequential() { - return executorService == null; - } - public void shutdown() { - assert isSequential() || !(executorService instanceof ForkJoinPool) || !((ForkJoinPool) executorService).hasQueuedSubmissions() : "There should be no queued submissions on shutdown."; + assert !executorService.hasQueuedSubmissions() : "There should be no queued submissions on shutdown."; assert completedOperations.sum() == postedOperations.sum() : "Posted operations (" + postedOperations.sum() + ") must match completed (" + completedOperations.sum() + ") operations"; setState(State.UNUSED); } @@ -286,19 +255,4 @@ public boolean isStarted() { public State getState() { return state.get(); } - - public int parallelism() { - if (executorService instanceof ForkJoinPool) { - return ((ForkJoinPool) executorService).getParallelism(); - } - return 1; - } - - public ExecutorService getExecutorService() { - return executorService; - } - - public void setExecutorService(ExecutorService executorService) { - this.executorService = executorService; - } } diff --git a/substratevm/src/com.oracle.graal.reachability/src/com/oracle/graal/reachability/ReachabilityAnalysisEngine.java b/substratevm/src/com.oracle.graal.reachability/src/com/oracle/graal/reachability/ReachabilityAnalysisEngine.java index ea0912b41eef..f1eab31b64cc 100644 --- a/substratevm/src/com.oracle.graal.reachability/src/com/oracle/graal/reachability/ReachabilityAnalysisEngine.java +++ b/substratevm/src/com.oracle.graal.reachability/src/com/oracle/graal/reachability/ReachabilityAnalysisEngine.java @@ -30,7 +30,6 @@ import java.util.Deque; import java.util.HashSet; import java.util.Set; -import java.util.concurrent.ForkJoinPool; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.core.common.type.TypedConstant; @@ -80,9 +79,9 @@ public abstract class ReachabilityAnalysisEngine extends AbstractAnalysisEngine @SuppressWarnings("this-escape") public ReachabilityAnalysisEngine(OptionValues options, AnalysisUniverse universe, HostVM hostVM, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, ForkJoinPool executorService, UnsupportedFeatures unsupportedFeatures, TimerCollection timerCollection, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, TimerCollection timerCollection, ReachabilityMethodProcessingHandler reachabilityMethodProcessingHandler) { - super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, executorService, unsupportedFeatures, timerCollection); + super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, timerCollection); this.executor.init(getTiming()); this.reachabilityTimer = timerCollection.createTimer("(reachability)"); diff --git a/substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java b/substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java index 519fa8734266..4f978d4b15d7 100644 --- a/substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java +++ b/substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java @@ -43,7 +43,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.ForkJoinPool; import java.util.function.Consumer; import java.util.stream.StreamSupport; @@ -1268,7 +1267,7 @@ public Element getOffsetBootstrapElement() { private final Map> dependenciesByDependingElement = new IdentityHashMap<>(); private final Map> dependenciesByDependedOnElement = new IdentityHashMap<>(); - public void write(DebugContext context, Path outputFile, @SuppressWarnings("unused") ForkJoinPool forkJoinPool) throws IOException { + public void write(DebugContext context, Path outputFile) throws IOException { try (FileChannel channel = FileChannel.open(outputFile, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE)) { withDebugContext(context, "ObjectFile.write", () -> { write(channel); diff --git a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java index 0c8de6681173..6677d3c29752 100644 --- a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java @@ -42,10 +42,10 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import java.util.stream.IntStream; +import com.oracle.svm.core.SubstrateOptions; import org.graalvm.collections.Pair; import jdk.graal.compiler.code.CompilationResult; import jdk.graal.compiler.core.common.NumUtil; @@ -117,9 +117,9 @@ public int codeSizeFor(HostedMethod method) { @Override @SuppressWarnings({"unused", "try"}) - public void layoutMethods(DebugContext debug, BigBang bb, ForkJoinPool threadPool) { + public void layoutMethods(DebugContext debug, BigBang bb) { try (Indent indent = debug.logAndIndent("layout methods")) { - BatchExecutor executor = new BatchExecutor(bb, threadPool); + BatchExecutor executor = new BatchExecutor(bb); try (StopTimer t = TimerCollection.createTimerAndStart("(bitcode)")) { writeBitcode(executor); } @@ -153,7 +153,7 @@ private void writeBitcode(BatchExecutor executor) { private int createBitcodeBatches(BatchExecutor executor, DebugContext debug) { batchSize = LLVMOptions.LLVMMaxFunctionsPerBatch.getValue(); - int numThreads = executor.getExecutor().parallelism(); + int numThreads = SubstrateOptions.NumberOfThreads.getValue(); int idealSize = NumUtil.divideAndRoundUp(methodIndex.length, numThreads); if (idealSize < batchSize) { batchSize = idealSize; diff --git a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMToolchainUtils.java b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMToolchainUtils.java index 15bb3af478f5..d7b746d668fe 100644 --- a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMToolchainUtils.java +++ b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMToolchainUtils.java @@ -29,7 +29,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ForkJoinPool; import java.util.function.Function; import java.util.function.IntFunction; @@ -170,8 +169,8 @@ public static void llvmCleanupStackMaps(DebugContext debug, String inputPath, Pa public static final class BatchExecutor { private CompletionExecutor executor; - public BatchExecutor(BigBang bb, ForkJoinPool threadPool) { - this.executor = new CompletionExecutor(bb, threadPool); + public BatchExecutor(BigBang bb) { + this.executor = new CompletionExecutor(bb); executor.init(); } diff --git a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/objectfile/LLVMObjectFile.java b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/objectfile/LLVMObjectFile.java index 54a0e6b30775..bcce8a587fea 100644 --- a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/objectfile/LLVMObjectFile.java +++ b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/objectfile/LLVMObjectFile.java @@ -41,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -183,7 +182,7 @@ public SymbolTable getSymbolTable() { @Override @SuppressWarnings("try") - public final void write(DebugContext context, Path outputFile, ForkJoinPool forkJoinPool) throws IOException { + public final void write(DebugContext context, Path outputFile) throws IOException { List sortedObjectFileElements = new ArrayList<>(); bake(sortedObjectFileElements); @@ -192,7 +191,7 @@ public final void write(DebugContext context, Path outputFile, ForkJoinPool fork writeParts(); - BatchExecutor batchExecutor = new BatchExecutor(bb, forkJoinPool); + BatchExecutor batchExecutor = new BatchExecutor(bb); compileBitcodeBatches(batchExecutor, context, numBatches); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 2948aacaa928..03788b94e10a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -58,6 +58,7 @@ import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.thread.VMOperationControl; import com.oracle.svm.core.util.UserError; +import com.oracle.svm.core.util.VMError; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -106,6 +107,16 @@ public static boolean parseOnce() { @Option(help = "Name of the output file to be generated", type = OptionType.User)// public static final HostedOptionKey Name = new HostedOptionKey<>(""); + /** + * Configures the number of threads of the common pool (see driver). + */ + @APIOption(name = "parallelism")// + @Option(help = "The maximum number of threads to use concurrently during native image generation.")// + public static final HostedOptionKey NumberOfThreads = new HostedOptionKey<>(Math.max(1, Math.min(Runtime.getRuntime().availableProcessors(), 32)), key -> { + int numberOfThreads = key.getValue(); + VMError.guarantee(numberOfThreads >= 1, "Number of threads must be at least 1. Validation should have happened in driver."); + }); + @APIOption(name = "shared")// @Option(help = "Build shared library")// public static final HostedOptionKey SharedLibrary = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index ca573188bb4e..5641e7ecf1a6 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -97,7 +97,6 @@ import com.oracle.svm.driver.metainf.NativeImageMetaInfResourceProcessor; import com.oracle.svm.driver.metainf.NativeImageMetaInfWalker; import com.oracle.svm.hosted.NativeImageGeneratorRunner; -import com.oracle.svm.hosted.NativeImageOptions; import com.oracle.svm.hosted.NativeImageSystemClassLoader; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; @@ -269,7 +268,7 @@ private static String oR(OptionKey option) { final String oHInspectServerContentPath = oH(PointstoOptions.InspectServerContentPath); final String oHDeadlockWatchdogInterval = oH(SubstrateOptions.DeadlockWatchdogInterval); - static final String oHNumberOfThreads = oH(NativeImageOptions.NumberOfThreads); + static final String oHNumberOfThreads = oH(SubstrateOptions.NumberOfThreads); final Map imageBuilderEnvironment = new HashMap<>(); private final ArrayList imageBuilderArgs = new ArrayList<>(); @@ -1098,14 +1097,15 @@ private int completeImageBuild() { Integer maxNumberOfThreads = getMaxNumberOfThreads(); if (maxNumberOfThreads != null) { - if (maxNumberOfThreads >= 2) { + if (maxNumberOfThreads >= 1) { /* - * Set number of threads in common pool. Subtract one because the main thread helps - * to process tasks. + * maxNumberOfThreads - 1 because the main thread always helps to process tasks. In + * single-threaded mode (parallelism=0 for common pool), only the main thread + * processes tasks. */ imageBuilderJavaArgs.add("-Djava.util.concurrent.ForkJoinPool.common.parallelism=" + (maxNumberOfThreads - 1)); } else { - throw showError("The number of threads was set to " + maxNumberOfThreads + ". Please set the '--parallelism' option to at least 2."); + throw showError("The number of threads was set to " + maxNumberOfThreads + ". Please set the '--parallelism' option to at least 1."); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java index 0465a396fe21..e7255096bfbf 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java @@ -30,7 +30,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ForkJoinPool; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.core.common.CompressEncoding; @@ -174,9 +173,9 @@ public SVMHost createHostVM(OptionValues options, ClassLoader classLoader, Class } public CompileQueue createCompileQueue(DebugContext debug, FeatureHandler featureHandler, HostedUniverse hostedUniverse, - RuntimeConfiguration runtimeConfiguration, boolean deoptimizeAll, SnippetReflectionProvider aSnippetReflection, ForkJoinPool executor) { + RuntimeConfiguration runtimeConfiguration, boolean deoptimizeAll, SnippetReflectionProvider aSnippetReflection) { - return new CompileQueue(debug, featureHandler, hostedUniverse, runtimeConfiguration, deoptimizeAll, aSnippetReflection, executor); + return new CompileQueue(debug, featureHandler, hostedUniverse, runtimeConfiguration, deoptimizeAll, aSnippetReflection); } public MethodTypeFlowBuilder createMethodTypeFlowBuilder(PointsToAnalysis bb, PointsToAnalysisMethod method, MethodFlowsGraph flowsGraph, MethodFlowsGraph.GraphKind graphKind) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index c159659fdfc2..1714fd92b78b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -53,7 +53,6 @@ import java.util.Map; import java.util.ServiceLoader; import java.util.Set; -import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BooleanSupplier; import java.util.stream.Stream; @@ -519,46 +518,40 @@ public void run(Map entryPoints, JavaMainSupport javaMainSupport, String imageName, NativeImageKind k, SubstitutionProcessor harnessSubstitutions, - ForkJoinPool compilationExecutor, ForkJoinPool analysisExecutor, EconomicSet allOptionNames, TimerCollection timerCollection) { - try { - if (!buildStarted.compareAndSet(false, true)) { - throw UserError.abort("An image build has already been performed with this generator."); - } + if (!buildStarted.compareAndSet(false, true)) { + throw UserError.abort("An image build has already been performed with this generator."); + } - try { - /* - * JVMCI 20.2-b01 introduced new methods for linking and querying whether an - * interface has default methods. Fail early if these methods are missing. - */ - ResolvedJavaType.class.getDeclaredMethod("link"); - } catch (ReflectiveOperationException ex) { - throw UserError.abort("JVMCI version provided %s is missing the 'ResolvedJavaType.link()' method added in jvmci-20.2-b01. " + - "Please use the latest JVMCI JDK from %s.", System.getProperty("java.home"), OPEN_LABSJDK_RELEASE_URL_PATTERN); - } + try { + /* + * JVMCI 20.2-b01 introduced new methods for linking and querying whether an interface + * has default methods. Fail early if these methods are missing. + */ + ResolvedJavaType.class.getDeclaredMethod("link"); + } catch (ReflectiveOperationException ex) { + throw UserError.abort("JVMCI version provided %s is missing the 'ResolvedJavaType.link()' method added in jvmci-20.2-b01. " + + "Please use the latest JVMCI JDK from %s.", System.getProperty("java.home"), OPEN_LABSJDK_RELEASE_URL_PATTERN); + } - setSystemPropertiesForImageLate(k); + setSystemPropertiesForImageLate(k); - ImageSingletonsSupportImpl.HostedManagement.install(new ImageSingletonsSupportImpl.HostedManagement()); + ImageSingletonsSupportImpl.HostedManagement.install(new ImageSingletonsSupportImpl.HostedManagement()); - ImageSingletons.add(ProgressReporter.class, reporter); - ImageSingletons.add(DeadlockWatchdog.class, loader.watchdog); - ImageSingletons.add(TimerCollection.class, timerCollection); - ImageSingletons.add(ImageBuildStatistics.TimerCollectionPrinter.class, timerCollection); - ImageSingletons.add(AnnotationExtractor.class, loader.classLoaderSupport.annotationExtractor); - ImageSingletons.add(BuildArtifacts.class, (type, artifact) -> buildArtifacts.computeIfAbsent(type, t -> new ArrayList<>()).add(artifact)); - ImageSingletons.add(HostedOptionValues.class, new HostedOptionValues(optionProvider.getHostedValues())); - ImageSingletons.add(RuntimeOptionValues.class, new RuntimeOptionValues(optionProvider.getRuntimeValues(), allOptionNames)); + ImageSingletons.add(ProgressReporter.class, reporter); + ImageSingletons.add(DeadlockWatchdog.class, loader.watchdog); + ImageSingletons.add(TimerCollection.class, timerCollection); + ImageSingletons.add(ImageBuildStatistics.TimerCollectionPrinter.class, timerCollection); + ImageSingletons.add(AnnotationExtractor.class, loader.classLoaderSupport.annotationExtractor); + ImageSingletons.add(BuildArtifacts.class, (type, artifact) -> buildArtifacts.computeIfAbsent(type, t -> new ArrayList<>()).add(artifact)); + ImageSingletons.add(HostedOptionValues.class, new HostedOptionValues(optionProvider.getHostedValues())); + ImageSingletons.add(RuntimeOptionValues.class, new RuntimeOptionValues(optionProvider.getRuntimeValues(), allOptionNames)); - try (TemporaryBuildDirectoryProviderImpl tempDirectoryProvider = new TemporaryBuildDirectoryProviderImpl()) { - ImageSingletons.add(TemporaryBuildDirectoryProvider.class, tempDirectoryProvider); - doRun(entryPoints, javaMainSupport, imageName, k, harnessSubstitutions, compilationExecutor, analysisExecutor); - } finally { - reporter.ensureCreationStageEndCompleted(); - } + try (TemporaryBuildDirectoryProviderImpl tempDirectoryProvider = new TemporaryBuildDirectoryProviderImpl()) { + ImageSingletons.add(TemporaryBuildDirectoryProvider.class, tempDirectoryProvider); + doRun(entryPoints, javaMainSupport, imageName, k, harnessSubstitutions); } finally { - analysisExecutor.shutdownNow(); - compilationExecutor.shutdownNow(); + reporter.ensureCreationStageEndCompleted(); } } @@ -583,8 +576,7 @@ public static void clearSystemPropertiesForImage() { @SuppressWarnings("try") protected void doRun(Map entryPoints, JavaMainSupport javaMainSupport, String imageName, NativeImageKind k, - SubstitutionProcessor harnessSubstitutions, - ForkJoinPool compilationExecutor, ForkJoinPool analysisExecutor) { + SubstitutionProcessor harnessSubstitutions) { List hostedEntryPoints = new ArrayList<>(); OptionValues options = HostedOptionValues.singleton(); @@ -592,7 +584,7 @@ protected void doRun(Map entryPoints, try (DebugContext debug = new Builder(options, new GraalDebugHandlersFactory(originalSnippetReflection)).build(); DebugCloseable featureCleanup = () -> featureHandler.forEachFeature(Feature::cleanup)) { - setupNativeImage(options, entryPoints, javaMainSupport, harnessSubstitutions, analysisExecutor, originalSnippetReflection, debug); + setupNativeImage(options, entryPoints, javaMainSupport, harnessSubstitutions, originalSnippetReflection, debug); boolean returnAfterAnalysis = runPointsToAnalysis(imageName, options, debug); if (returnAfterAnalysis) { @@ -670,8 +662,7 @@ protected void doRun(Map entryPoints, NativeImageCodeCache codeCache; CompileQueue compileQueue; try (StopTimer t = TimerCollection.createTimerAndStart(TimerCollection.Registry.COMPILE_TOTAL)) { - compileQueue = HostedConfiguration.instance().createCompileQueue(debug, featureHandler, hUniverse, runtimeConfiguration, DeoptTester.enabled(), - bb.getSnippetReflectionProvider(), compilationExecutor); + compileQueue = HostedConfiguration.instance().createCompileQueue(debug, featureHandler, hUniverse, runtimeConfiguration, DeoptTester.enabled(), bb.getSnippetReflectionProvider()); if (ImageSingletons.contains(RuntimeCompilationSupport.class)) { ImageSingletons.lookup(RuntimeCompilationSupport.class).onCompileQueueCreation(bb, hUniverse, compileQueue); } @@ -684,8 +675,8 @@ protected void doRun(Map entryPoints, codeCache = NativeImageCodeCacheFactory.get().newCodeCache(compileQueue, heap, loader.platform, ImageSingletons.lookup(TemporaryBuildDirectoryProvider.class).getTemporaryBuildDirectory()); codeCache.layoutConstants(); - codeCache.layoutMethods(debug, bb, compilationExecutor); - codeCache.buildRuntimeMetadata(bb.getSnippetReflectionProvider(), compilationExecutor); + codeCache.layoutMethods(debug, bb); + codeCache.buildRuntimeMetadata(bb.getSnippetReflectionProvider()); } AfterCompilationAccessImpl config = new AfterCompilationAccessImpl(featureHandler, loader, aUniverse, hUniverse, compileQueue.getCompilations(), codeCache, heap, debug, @@ -695,7 +686,7 @@ protected void doRun(Map entryPoints, } /* Re-run shadow heap verification after compilation. */ - aUniverse.getHeapVerifier().checkHeapSnapshot(hMetaAccess, compilationExecutor, "after compilation"); + aUniverse.getHeapVerifier().checkHeapSnapshot(hMetaAccess, "after compilation"); CodeCacheProvider codeCacheProvider = runtimeConfiguration.getBackendForNormalMethod().getProviders().getCodeCache(); reporter.printCreationStart(); @@ -714,7 +705,7 @@ protected void doRun(Map entryPoints, featureHandler.forEachFeature(feature -> feature.afterHeapLayout(config)); /* Re-run shadow heap verification after heap layout. */ - aUniverse.getHeapVerifier().checkHeapSnapshot(hMetaAccess, compilationExecutor, "after heap layout"); + aUniverse.getHeapVerifier().checkHeapSnapshot(hMetaAccess, "after heap layout"); createAbstractImage(k, hostedEntryPoints, heap, hMetaAccess, codeCache); @@ -755,7 +746,7 @@ protected void doRun(Map entryPoints, * not is an implementation detail of the image. */ Path tmpDir = ImageSingletons.lookup(TemporaryBuildDirectoryProvider.class).getTemporaryBuildDirectory(); - LinkerInvocation inv = image.write(debug, generatedFiles(HostedOptionValues.singleton()), tmpDir, imageName, beforeConfig, compilationExecutor); + LinkerInvocation inv = image.write(debug, generatedFiles(HostedOptionValues.singleton()), tmpDir, imageName, beforeConfig); if (NativeImageOptions.ExitAfterRelocatableImageWrite.getValue()) { return; } @@ -869,7 +860,7 @@ protected boolean verifyAssignableTypes() { @SuppressWarnings("try") protected void setupNativeImage(OptionValues options, Map entryPoints, JavaMainSupport javaMainSupport, - SubstitutionProcessor harnessSubstitutions, ForkJoinPool analysisExecutor, SnippetReflectionProvider originalSnippetReflection, DebugContext debug) { + SubstitutionProcessor harnessSubstitutions, SnippetReflectionProvider originalSnippetReflection, DebugContext debug) { try (Indent ignored = debug.logAndIndent("setup native-image builder")) { try (StopTimer ignored1 = TimerCollection.createTimerAndStart(TimerCollection.Registry.SETUP)) { SubstrateTargetDescription target = createTarget(); @@ -934,7 +925,7 @@ protected void setupNativeImage(OptionValues options, Map NumberOfThreads = new HostedOptionKey<>(Math.max(2, Math.min(Runtime.getRuntime().availableProcessors(), 32)), key -> { - int numberOfThreads = key.getValue(); - VMError.guarantee(numberOfThreads >= 2, "Number of threads must be at least 2. Validation should have happened in driver."); - }); - - /* - * Analysis scales well up to 12 cores and gives slight improvements until 18 cores. We set the - * default value to 16 to minimize wasted resources in large machines. - */ - @Option(help = "The number of threads to use for analysis during native image generation. The number must be smaller than the NumberOfThreads.", deprecated = true, deprecationMessage = "Please use '--parallelism' instead.")// + @Option(help = "Deprecated, option no longer has any effect", deprecated = true, deprecationMessage = "Please use '--parallelism' instead.")// public static final HostedOptionKey NumberOfAnalysisThreads = new HostedOptionKey<>(-1); @Option(help = "Return after analysis")// @@ -214,13 +198,13 @@ public static CStandards getCStandard() { @Option(help = "Exit after writing relocatable file")// public static final HostedOptionKey ExitAfterRelocatableImageWrite = new HostedOptionKey<>(false); - @Option(help = "Throw unsafe operation offset errors.)")// + @Option(help = "Throw unsafe operation offset errors.")// public static final HostedOptionKey ThrowUnsafeOffsetErrors = new HostedOptionKey<>(true); - @Option(help = "Print unsafe operation offset warnings.)")// + @Option(help = "Print unsafe operation offset warnings.")// public static final HostedOptionKey ReportUnsafeOffsetWarnings = new HostedOptionKey<>(false); - @Option(help = "Print unsafe operation offset warnings.)")// + @Option(help = "Print unsafe operation offset warnings.")// public static final HostedOptionKey UnsafeOffsetWarningsAreFatal = new HostedOptionKey<>(false); /** @@ -272,22 +256,4 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } } }; - - public static int getNumberOfAnalysisThreads(int maxNumberOfThreads, OptionValues optionValues) { - int analysisThreads; - if (NumberOfAnalysisThreads.hasBeenSet(optionValues)) { - analysisThreads = NumberOfAnalysisThreads.getValue(optionValues); - } else { - analysisThreads = Math.min(maxNumberOfThreads, DEFAULT_MAX_ANALYSIS_SCALING); - } - if (analysisThreads < 2) { - throw UserError.abort("Number of analysis threads was set to " + analysisThreads + ". Please set the '-H:NumberOfAnalysisThreads' option to at least 2."); - } - if (analysisThreads > maxNumberOfThreads) { - throw UserError.abort( - "NumberOfAnalysisThreads is not allowed to be larger than the number of threads set with the '--parallelism' option. Please set the '-H:NumberOfAnalysisThreads' option to a value between 1 and " + - (maxNumberOfThreads + 1) + "."); - } - return analysisThreads; - } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 64dceb12a375..309031cd0a71 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -392,13 +392,13 @@ private void printResourceInfo() { maxHeapSuffix = "set via '%s'".formatted(xmxValueOrNull); } - int maxNumberOfThreads = NativeImageOptions.NumberOfThreads.getValue(); + int maxNumberOfThreads = SubstrateOptions.NumberOfThreads.getValue(); recordJsonMetric(ResourceUsageKey.PARALLELISM, maxNumberOfThreads); int availableProcessors = runtime.availableProcessors(); recordJsonMetric(ResourceUsageKey.CPU_CORES_TOTAL, availableProcessors); String maxNumberOfThreadsSuffix = "determined at start"; - if (NativeImageOptions.NumberOfThreads.hasBeenSet()) { - maxNumberOfThreadsSuffix = "set via '%s'".formatted(SubstrateOptionsParser.commandArgument(NativeImageOptions.NumberOfThreads, Integer.toString(maxNumberOfThreads))); + if (SubstrateOptions.NumberOfThreads.hasBeenSet()) { + maxNumberOfThreadsSuffix = "set via '%s'".formatted(SubstrateOptionsParser.commandArgument(SubstrateOptions.NumberOfThreads, Integer.toString(maxNumberOfThreads))); } l().printLineSeparator(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImagePointsToAnalysis.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImagePointsToAnalysis.java index f096d25329d0..6411ed167001 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImagePointsToAnalysis.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImagePointsToAnalysis.java @@ -27,7 +27,6 @@ import java.lang.reflect.Executable; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.util.concurrent.ForkJoinPool; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.options.OptionValues; @@ -66,10 +65,9 @@ public class NativeImagePointsToAnalysis extends PointsToAnalysis implements Inf public NativeImagePointsToAnalysis(OptionValues options, AnalysisUniverse universe, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, - AnnotationSubstitutionProcessor annotationSubstitutionProcessor, ForkJoinPool executor, UnsupportedFeatures unsupportedFeatures, + AnnotationSubstitutionProcessor annotationSubstitutionProcessor, UnsupportedFeatures unsupportedFeatures, TimerCollection timerCollection) { - super(options, universe, universe.hostVM(), metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, executor, unsupportedFeatures, timerCollection, - SubstrateOptions.parseOnce()); + super(options, universe, universe.hostVM(), metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, timerCollection, SubstrateOptions.parseOnce()); this.annotationSubstitutionProcessor = annotationSubstitutionProcessor; dynamicHubInitializer = new DynamicHubInitializer(this); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImageReachabilityAnalysisEngine.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImageReachabilityAnalysisEngine.java index 0b129a9cf1c8..0330bd762c8e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImageReachabilityAnalysisEngine.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImageReachabilityAnalysisEngine.java @@ -24,12 +24,6 @@ */ package com.oracle.svm.hosted.analysis; -import java.util.concurrent.ForkJoinPool; - -import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; -import jdk.graal.compiler.options.OptionValues; -import jdk.graal.compiler.word.WordTypes; - import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMetaAccess; import com.oracle.graal.pointsto.meta.AnalysisType; @@ -41,6 +35,9 @@ import com.oracle.svm.hosted.SVMHost; import com.oracle.svm.hosted.substitute.AnnotationSubstitutionProcessor; +import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; +import jdk.graal.compiler.options.OptionValues; +import jdk.graal.compiler.word.WordTypes; import jdk.vm.ci.meta.ConstantReflectionProvider; public class NativeImageReachabilityAnalysisEngine extends ReachabilityAnalysisEngine implements Inflation { @@ -51,12 +48,10 @@ public class NativeImageReachabilityAnalysisEngine extends ReachabilityAnalysisE private final CustomTypeFieldHandler unknownFieldHandler; @SuppressWarnings("this-escape") - public NativeImageReachabilityAnalysisEngine(OptionValues options, AnalysisUniverse universe, - AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, - AnnotationSubstitutionProcessor annotationSubstitutionProcessor, - ForkJoinPool executor, TimerCollection timerCollection, ReachabilityMethodProcessingHandler reachabilityMethodProcessingHandler) { - super(options, universe, universe.hostVM(), metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, executor, new SubstrateUnsupportedFeatures(), timerCollection, + public NativeImageReachabilityAnalysisEngine(OptionValues options, AnalysisUniverse universe, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, AnnotationSubstitutionProcessor annotationSubstitutionProcessor, TimerCollection timerCollection, + ReachabilityMethodProcessingHandler reachabilityMethodProcessingHandler) { + super(options, universe, universe.hostVM(), metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, new SubstrateUnsupportedFeatures(), timerCollection, reachabilityMethodProcessingHandler); this.annotationSubstitutionProcessor = annotationSubstitutionProcessor; this.strengthenGraalGraphs = SubstrateOptions.parseOnce(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java index b7592f7e9287..71a3028aca69 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java @@ -37,7 +37,6 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ForkJoinPool; import org.graalvm.collections.EconomicMap; import jdk.graal.compiler.api.replacements.Fold; @@ -361,14 +360,14 @@ public Description getDescription() { @SuppressWarnings("this-escape") public CompileQueue(DebugContext debug, FeatureHandler featureHandler, HostedUniverse universe, RuntimeConfiguration runtimeConfiguration, Boolean deoptimizeAll, - SnippetReflectionProvider snippetReflection, ForkJoinPool executorService) { + SnippetReflectionProvider snippetReflection) { this.universe = universe; this.compilations = new ConcurrentHashMap<>(); this.runtimeConfig = runtimeConfiguration; this.metaAccess = runtimeConfiguration.getProviders().getMetaAccess(); this.deoptimizeAll = deoptimizeAll; this.dataCache = new ConcurrentHashMap<>(); - this.executor = new CompletionExecutor(universe.getBigBang(), executorService); + this.executor = new CompletionExecutor(universe.getBigBang()); this.featureHandler = featureHandler; this.snippetReflection = snippetReflection; this.graphTransplanter = createGraphTransplanter(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java index 8efb24238a98..1937ff5c7081 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java @@ -26,7 +26,6 @@ import java.nio.file.Path; import java.util.List; -import java.util.concurrent.ForkJoinPool; import jdk.graal.compiler.debug.DebugContext; @@ -54,16 +53,12 @@ public enum NativeImageKind { SHARED_LIBRARY(false) { @Override public String getFilenameSuffix() { - switch (ObjectFile.getNativeFormat()) { - case ELF: - return ".so"; - case MACH_O: - return ".dylib"; - case PECOFF: - return ".dll"; - default: - throw new AssertionError("Unreachable"); - } + return switch (ObjectFile.getNativeFormat()) { + case ELF -> ".so"; + case MACH_O -> ".dylib"; + case PECOFF -> ".dll"; + default -> throw new AssertionError("Unreachable"); + }; } }, EXECUTABLE(true), @@ -123,17 +118,17 @@ public NativeLibraries getNativeLibs() { /** * Write the image to the named file. */ - public abstract LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tempDirectory, String imageName, BeforeImageWriteAccessImpl config, ForkJoinPool forkJoinPool); + public abstract LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tempDirectory, String imageName, BeforeImageWriteAccessImpl config); // factory method public static AbstractImage create(NativeImageKind k, HostedUniverse universe, HostedMetaAccess metaAccess, NativeLibraries nativeLibs, NativeImageHeap heap, NativeImageCodeCache codeCache, List entryPoints, ClassLoader classLoader) { - switch (k) { - case SHARED_LIBRARY: - return new SharedLibraryImageViaCC(universe, metaAccess, nativeLibs, heap, codeCache, entryPoints, classLoader); - default: - return new ExecutableImageViaCC(k, universe, metaAccess, nativeLibs, heap, codeCache, entryPoints, classLoader); - } + return switch (k) { + case SHARED_LIBRARY -> + new SharedLibraryImageViaCC(universe, metaAccess, nativeLibs, heap, codeCache, entryPoints, classLoader); + case EXECUTABLE, STATIC_EXECUTABLE -> + new ExecutableImageViaCC(k, universe, metaAccess, nativeLibs, heap, codeCache, entryPoints, classLoader); + }; } public abstract String[] makeLaunchCommand(AbstractImage.NativeImageKind k, String imageName, Path binPath, Path workPath, java.lang.reflect.Method method); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java index efa5b82136b4..c7cf1c37bec5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java @@ -30,7 +30,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -139,7 +138,7 @@ private boolean verifyMethodLayout() { @SuppressWarnings("try") @Override - public void layoutMethods(DebugContext debug, BigBang bb, ForkJoinPool threadPool) { + public void layoutMethods(DebugContext debug, BigBang bb) { try (Indent indent = debug.logAndIndent("layout methods")) { // Assign initial location to all methods. diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java index 68bb406bb11a..90d4f5b6b094 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java @@ -46,7 +46,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import org.graalvm.collections.Pair; @@ -157,13 +156,13 @@ public NativeImage(NativeImageKind k, HostedUniverse universe, HostedMetaAccess @Override public abstract String[] makeLaunchCommand(NativeImageKind k, String imageName, Path binPath, Path workPath, java.lang.reflect.Method method); - protected final void write(DebugContext context, Path outputFile, ForkJoinPool forkJoinPool) { + protected final void write(DebugContext context, Path outputFile) { try { Path outFileParent = outputFile.normalize().getParent(); if (outFileParent != null) { Files.createDirectories(outFileParent); } - objectFile.write(context, outputFile, forkJoinPool); + objectFile.write(context, outputFile); } catch (Exception ex) { throw shouldNotReachHere(ex); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java index 195f666b8952..ece30a9b5148 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java @@ -46,7 +46,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import com.oracle.svm.hosted.DeadlockWatchdog; @@ -191,7 +190,7 @@ protected CompilationResult compilationResultFor(HostedMethod method) { return compilations.get(method); } - public abstract void layoutMethods(DebugContext debug, BigBang bb, ForkJoinPool threadPool); + public abstract void layoutMethods(DebugContext debug, BigBang bb); public void layoutConstants() { for (Pair pair : getOrderedCompilations()) { @@ -263,11 +262,11 @@ public int getAlignedConstantsSize() { return ConfigurationValues.getObjectLayout().alignUp(getConstantsSize()); } - public void buildRuntimeMetadata(SnippetReflectionProvider snippetReflectionProvider, ForkJoinPool threadPool) { - buildRuntimeMetadata(snippetReflectionProvider, threadPool, new MethodPointer(getFirstCompilation().getLeft(), true), WordFactory.signed(getCodeAreaSize())); + public void buildRuntimeMetadata(SnippetReflectionProvider snippetReflectionProvider) { + buildRuntimeMetadata(snippetReflectionProvider, new MethodPointer(getFirstCompilation().getLeft(), true), WordFactory.signed(getCodeAreaSize())); } - protected void buildRuntimeMetadata(SnippetReflectionProvider snippetReflection, ForkJoinPool threadPool, CFunctionPointer firstMethod, UnsignedWord codeSize) { + protected void buildRuntimeMetadata(SnippetReflectionProvider snippetReflection, CFunctionPointer firstMethod, UnsignedWord codeSize) { // Build run-time metadata. HostedFrameInfoCustomization frameInfoCustomization = new HostedFrameInfoCustomization(); CodeInfoEncoder.Encoders encoders = new CodeInfoEncoder.Encoders(); @@ -425,7 +424,7 @@ protected void buildRuntimeMetadata(SnippetReflectionProvider snippetReflection, verifyDeoptEntries(imageCodeInfo); } - assert verifyMethods(hUniverse, threadPool, codeInfoEncoder, imageCodeInfo); + assert verifyMethods(hUniverse, codeInfoEncoder, imageCodeInfo); } protected HostedImageCodeInfo installCodeInfo(SnippetReflectionProvider snippetReflection, CFunctionPointer firstMethod, UnsignedWord codeSize, CodeInfoEncoder codeInfoEncoder, @@ -568,12 +567,12 @@ private static boolean error(HostedMethod method, long encodedBci, String msg) { return true; } - protected boolean verifyMethods(HostedUniverse hUniverse, ForkJoinPool threadPool, CodeInfoEncoder codeInfoEncoder, CodeInfo codeInfo) { + protected boolean verifyMethods(HostedUniverse hUniverse, CodeInfoEncoder codeInfoEncoder, CodeInfo codeInfo) { /* * Run method verification in parallel to reduce computation time. */ BigBang bb = hUniverse.getBigBang(); - CompletionExecutor executor = new CompletionExecutor(bb, threadPool); + CompletionExecutor executor = new CompletionExecutor(bb); try { executor.init(); executor.start(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageViaCC.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageViaCC.java index 7870870bc43f..cce7224184ac 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageViaCC.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageViaCC.java @@ -34,7 +34,6 @@ import java.util.Collections; import java.util.Formatter; import java.util.List; -import java.util.concurrent.ForkJoinPool; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -89,13 +88,13 @@ private static List diagnoseLinkerFailure(String linkerOutput) { @Override @SuppressWarnings("try") - public LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tempDirectory, String imageName, BeforeImageWriteAccessImpl config, ForkJoinPool forkJoinPool) { + public LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tempDirectory, String imageName, BeforeImageWriteAccessImpl config) { try (Indent indent = debug.logAndIndent("Writing native image")) { // 0. Free codecache to make space for writing the objectFile codeCache.purge(); // 1. write the relocatable file - write(debug, tempDirectory.resolve(imageName + ObjectFile.getFilenameSuffix()), forkJoinPool); + write(debug, tempDirectory.resolve(imageName + ObjectFile.getFilenameSuffix())); if (NativeImageOptions.ExitAfterRelocatableImageWrite.getValue()) { return null; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/SharedLibraryImageViaCC.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/SharedLibraryImageViaCC.java index 44e18ba9c9c0..2f4d73adc14f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/SharedLibraryImageViaCC.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/SharedLibraryImageViaCC.java @@ -28,7 +28,6 @@ import java.lang.reflect.Method; import java.nio.file.Path; import java.util.List; -import java.util.concurrent.ForkJoinPool; import jdk.graal.compiler.debug.DebugContext; @@ -53,8 +52,8 @@ public String[] makeLaunchCommand(NativeImageKind k, String imageName, Path binP } @Override - public LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tempDirectory, String imageName, BeforeImageWriteAccessImpl config, ForkJoinPool forkJoinPool) { - LinkerInvocation inv = super.write(debug, outputDirectory, tempDirectory, imageName, config, forkJoinPool); + public LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tempDirectory, String imageName, BeforeImageWriteAccessImpl config) { + LinkerInvocation inv = super.write(debug, outputDirectory, tempDirectory, imageName, config); writeHeaderFiles(outputDirectory, imageName, false); writeHeaderFiles(outputDirectory, imageName, true); return inv; From e9a1f2da51c3471b637f4616cb88fa875bfd384c Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Wed, 18 Oct 2023 08:54:29 +0200 Subject: [PATCH 12/78] Load classes using the common pool. --- .../com/oracle/svm/hosted/ImageClassLoader.java | 14 ++------------ .../svm/hosted/NativeImageClassLoaderSupport.java | 1 + 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java index d861bcca5010..70783a3d6761 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java @@ -57,7 +57,6 @@ public final class ImageClassLoader { * This cannot be a HostedOption because the option parsing already relies on the list of loaded * classes. */ - private static final int CLASS_LOADING_MAX_SCALING = 8; private static final int CLASS_LOADING_TIMEOUT_IN_MINUTES = 10; static { @@ -87,20 +86,11 @@ public final class ImageClassLoader { } public void loadAllClasses() throws InterruptedException { - ForkJoinPool executor = new ForkJoinPool(Math.min(Runtime.getRuntime().availableProcessors(), CLASS_LOADING_MAX_SCALING)) { - @Override - public void execute(Runnable task) { - super.execute(() -> { - task.run(); - watchdog.recordActivity(); - }); - } - }; + ForkJoinPool executor = ForkJoinPool.commonPool(); try { classLoaderSupport.loadAllClasses(executor, this); } finally { - executor.shutdown(); - executor.awaitTermination(CLASS_LOADING_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES); + executor.awaitQuiescence(CLASS_LOADING_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES); } classLoaderSupport.reportBuilderClassesInApplication(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index be6d69b18ad1..9822516daf8c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -879,6 +879,7 @@ private void handleClassFileName(URI container, Module module, String className, } imageClassLoader.handleClass(clazz); } + imageClassLoader.watchdog.recordActivity(); } } From c58619b8c0338b00a062fa8b6795e178a6752553 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 19 Oct 2023 09:16:00 +0200 Subject: [PATCH 13/78] Await completion of class loading tasks. --- .../com/oracle/svm/hosted/ImageClassLoader.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java index 70783a3d6761..624b129bf195 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java @@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.oracle.svm.util.LogUtils; import org.graalvm.collections.EconomicSet; import jdk.graal.compiler.debug.GraalError; import jdk.graal.compiler.word.Word; @@ -53,12 +54,6 @@ public final class ImageClassLoader { - /* - * This cannot be a HostedOption because the option parsing already relies on the list of loaded - * classes. - */ - private static final int CLASS_LOADING_TIMEOUT_IN_MINUTES = 10; - static { /* * ImageClassLoader is one of the first classes used during image generation, so early @@ -90,7 +85,14 @@ public void loadAllClasses() throws InterruptedException { try { classLoaderSupport.loadAllClasses(executor, this); } finally { - executor.awaitQuiescence(CLASS_LOADING_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES); + boolean isQuiescence = false; + while (!isQuiescence) { + isQuiescence = executor.awaitQuiescence(10, TimeUnit.MINUTES); + if (!isQuiescence) { + LogUtils.warning("Class loading is slow. Waiting for tasks to complete..."); + /* DeadlockWatchdog should fail the build eventually. */ + } + } } classLoaderSupport.reportBuilderClassesInApplication(); } From a0ada6eb4c69c43ac2d9918fa3dfb784b1662977 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 19 Oct 2023 12:54:18 +0200 Subject: [PATCH 14/78] Extract reportFailureState logic in watchdog. --- .../oracle/svm/hosted/DeadlockWatchdog.java | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/DeadlockWatchdog.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/DeadlockWatchdog.java index 040eaa642c99..f098d86b8c0e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/DeadlockWatchdog.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/DeadlockWatchdog.java @@ -82,28 +82,8 @@ void watchdogThread() { while (!stopped) { long now = System.currentTimeMillis(); if (enabled && now >= nextDeadline) { - System.err.println(); - System.err.println("=== Image generator watchdog detected no activity. This can be a sign of a deadlock during image building. Dumping all stack traces. Current time: " + new Date()); - threadDump(); - Runtime runtime = Runtime.getRuntime(); - final long heapSizeUnit = 1024 * 1024; - long usedHeapSize = runtime.totalMemory() / heapSizeUnit; - long freeHeapSize = runtime.freeMemory() / heapSizeUnit; - long maximumHeapSize = runtime.maxMemory() / heapSizeUnit; - System.err.printf("=== Memory statistics (in MB):%n=== Used heap size: %d%n=== Free heap size: %d%n=== Maximum heap size: %d%n", usedHeapSize, freeHeapSize, maximumHeapSize); - System.err.flush(); - - if (watchdogExitOnTimeout) { - System.err.println("=== Image generator watchdog is aborting image generation. To configure the watchdog, use the options " + - SubstrateOptionsParser.commandArgument(SubstrateOptions.DeadlockWatchdogInterval, Integer.toString(watchdogInterval), null) + " and " + - SubstrateOptionsParser.commandArgument(SubstrateOptions.DeadlockWatchdogExitOnTimeout, "+", null)); - /* - * Since there is a likely deadlock somewhere, there is no less intrusive way to - * abort other than a hard exit of the image builder VM. - */ - System.exit(ExitStatus.WATCHDOG_EXIT.getValue()); - - } else { + reportFailureState(); + if (!watchdogExitOnTimeout) { recordActivity(); } } @@ -116,6 +96,30 @@ void watchdogThread() { } } + public void reportFailureState() { + System.err.println(); + System.err.println("=== Image generator watchdog detected no activity. This can be a sign of a deadlock during image building. Dumping all stack traces. Current time: " + new Date()); + threadDump(); + Runtime runtime = Runtime.getRuntime(); + final long heapSizeUnit = 1024 * 1024; + long usedHeapSize = runtime.totalMemory() / heapSizeUnit; + long freeHeapSize = runtime.freeMemory() / heapSizeUnit; + long maximumHeapSize = runtime.maxMemory() / heapSizeUnit; + System.err.printf("=== Memory statistics (in MB):%n=== Used heap size: %d%n=== Free heap size: %d%n=== Maximum heap size: %d%n", usedHeapSize, freeHeapSize, maximumHeapSize); + System.err.flush(); + + if (watchdogExitOnTimeout) { + System.err.println("=== Image generator watchdog is aborting image generation. To configure the watchdog, use the options " + + SubstrateOptionsParser.commandArgument(SubstrateOptions.DeadlockWatchdogInterval, Integer.toString(watchdogInterval), null) + " and " + + SubstrateOptionsParser.commandArgument(SubstrateOptions.DeadlockWatchdogExitOnTimeout, "+", null)); + /* + * Since there is a likely deadlock somewhere, there is no less intrusive way to abort + * other than a hard exit of the image builder VM. + */ + System.exit(ExitStatus.WATCHDOG_EXIT.getValue()); + } + } + public void setEnabled(boolean enable) { if (enable == this.enabled) { return; From cff94e6fe6cc82e52e7984765061187fde683564 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 19 Oct 2023 12:57:37 +0200 Subject: [PATCH 15/78] Let CompletionExecutor call `closeDumpHandlers()`. --- .../pointsto/standalone/PointsToAnalyzer.java | 3 +- .../StandalonePointsToAnalysis.java | 12 ++--- .../pointsto/AbstractAnalysisEngine.java | 4 +- .../graal/pointsto/PointsToAnalysis.java | 4 +- .../pointsto/heap/HeapSnapshotVerifier.java | 12 ++--- .../pointsto/util/CompletionExecutor.java | 52 +++++++++++-------- .../ReachabilityAnalysisEngine.java | 16 +++--- .../graal/llvm/LLVMNativeImageCodeCache.java | 2 +- .../core/graal/llvm/LLVMToolchainUtils.java | 4 +- .../graal/llvm/objectfile/LLVMObjectFile.java | 2 +- .../svm/hosted/HostedConfiguration.java | 4 +- .../svm/hosted/NativeImageGenerator.java | 18 +++---- .../hosted/NativeImageGeneratorRunner.java | 9 +--- .../analysis/NativeImagePointsToAnalysis.java | 13 ++--- ...NativeImageReachabilityAnalysisEngine.java | 7 +-- .../oracle/svm/hosted/code/CompileQueue.java | 2 +- .../hosted/image/NativeImageCodeCache.java | 12 ++--- 17 files changed, 90 insertions(+), 86 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/PointsToAnalyzer.java b/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/PointsToAnalyzer.java index 8785e202b5cf..2bbbca0263a0 100644 --- a/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/PointsToAnalyzer.java +++ b/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/PointsToAnalyzer.java @@ -151,7 +151,8 @@ private PointsToAnalyzer(String mainEntryClass, OptionValues options) { originalProviders.getPlatformConfigurationProvider(), aMetaAccessExtensionProvider, originalProviders.getLoopsDataProvider()); standaloneHost.initializeProviders(aProviders); analysisName = getAnalysisName(mainEntryClass); - bigbang = new StandalonePointsToAnalysis(options, aUniverse, standaloneHost, aMetaAccess, snippetReflection, aConstantReflection, aProviders.getWordTypes(), new TimerCollection()); + bigbang = new StandalonePointsToAnalysis(options, aUniverse, standaloneHost, aMetaAccess, snippetReflection, aConstantReflection, aProviders.getWordTypes(), debugContext, + new TimerCollection()); standaloneHost.setImageName(analysisName); aUniverse.setBigBang(bigbang); ImageHeap heap = new ImageHeap(); diff --git a/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/StandalonePointsToAnalysis.java b/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/StandalonePointsToAnalysis.java index 8e769b0bde7d..e48d66606b73 100644 --- a/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/StandalonePointsToAnalysis.java +++ b/substratevm/src/com.oracle.graal.pointsto.standalone/src/com/oracle/graal/pointsto/standalone/StandalonePointsToAnalysis.java @@ -29,10 +29,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; -import jdk.graal.compiler.options.OptionValues; -import jdk.graal.compiler.word.WordTypes; - import com.oracle.graal.pointsto.PointsToAnalysis; import com.oracle.graal.pointsto.api.HostVM; import com.oracle.graal.pointsto.constraints.UnsupportedFeatures; @@ -42,14 +38,18 @@ import com.oracle.graal.pointsto.meta.AnalysisUniverse; import com.oracle.graal.pointsto.util.TimerCollection; +import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; +import jdk.graal.compiler.debug.DebugContext; +import jdk.graal.compiler.options.OptionValues; +import jdk.graal.compiler.word.WordTypes; import jdk.vm.ci.meta.ConstantReflectionProvider; public class StandalonePointsToAnalysis extends PointsToAnalysis { private final Set addedClinits = ConcurrentHashMap.newKeySet(); public StandalonePointsToAnalysis(OptionValues options, AnalysisUniverse universe, HostVM hostVM, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, TimerCollection timerCollection) { - super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, new UnsupportedFeatures(), timerCollection, true); + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, DebugContext debugContext, TimerCollection timerCollection) { + super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, new UnsupportedFeatures(), debugContext, timerCollection, true); } @Override diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java index 687995ec9550..be32d2919e5a 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/AbstractAnalysisEngine.java @@ -100,7 +100,7 @@ public abstract class AbstractAnalysisEngine implements BigBang { @SuppressWarnings("this-escape") public AbstractAnalysisEngine(OptionValues options, AnalysisUniverse universe, HostVM hostVM, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, DebugContext debugContext, TimerCollection timerCollection) { this.options = options; this.universe = universe; @@ -109,7 +109,7 @@ public AbstractAnalysisEngine(OptionValues options, AnalysisUniverse universe, H this.metaAccess = metaAccess; this.analysisPolicy = universe.analysisPolicy(); this.hostVM = hostVM; - this.executor = new CompletionExecutor(this); + this.executor = new CompletionExecutor(debugContext, this); this.unsupportedFeatures = unsupportedFeatures; this.processFeaturesTimer = timerCollection.get(TimerCollection.Registry.FEATURES); diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java index 5c6af4e84765..23d0ec04653f 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java @@ -109,9 +109,9 @@ public abstract class PointsToAnalysis extends AbstractAnalysisEngine { @SuppressWarnings("this-escape") public PointsToAnalysis(OptionValues options, AnalysisUniverse universe, HostVM hostVM, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, TimerCollection timerCollection, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, DebugContext debugContext, TimerCollection timerCollection, boolean strengthenGraalGraphs) { - super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, timerCollection); + super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, debugContext, timerCollection); this.typeFlowTimer = timerCollection.createTimer("(typeflow)"); this.strengthenGraalGraphs = strengthenGraalGraphs; diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java index e79f60ca0a6b..d72d31940e9c 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/HeapSnapshotVerifier.java @@ -29,10 +29,6 @@ import java.util.Objects; import java.util.function.Consumer; -import jdk.graal.compiler.options.Option; -import jdk.graal.compiler.options.OptionKey; -import jdk.graal.compiler.options.OptionType; - import com.oracle.graal.pointsto.BigBang; import com.oracle.graal.pointsto.ObjectScanner; import com.oracle.graal.pointsto.ObjectScanner.ReusableSet; @@ -45,6 +41,10 @@ import com.oracle.graal.pointsto.util.CompletionExecutor; import com.oracle.svm.util.LogUtils; +import jdk.graal.compiler.debug.DebugContext; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionKey; +import jdk.graal.compiler.options.OptionType; import jdk.vm.ci.meta.JavaConstant; public class HeapSnapshotVerifier { @@ -73,8 +73,8 @@ public HeapSnapshotVerifier(BigBang bb, ImageHeap imageHeap, ImageHeapScanner sc verbosity = Options.HeapVerifierVerbosity.getValue(bb.getOptions()); } - public boolean checkHeapSnapshot(UniverseMetaAccess metaAccess, String stage) { - CompletionExecutor executor = new CompletionExecutor(bb); + public boolean checkHeapSnapshot(DebugContext debug, UniverseMetaAccess metaAccess, String stage) { + CompletionExecutor executor = new CompletionExecutor(debug, bb); executor.init(); return checkHeapSnapshot(metaAccess, executor, stage, false); } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java index 6435a701c420..7e87277bc7a7 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java @@ -65,6 +65,7 @@ private enum State { private final ForkJoinPool executorService; + private final DebugContext debug; private final BigBang bb; private Timing timing; private Object vmConfig; @@ -81,7 +82,8 @@ public interface Timing { void print(); } - public CompletionExecutor(BigBang bb) { + public CompletionExecutor(DebugContext debugContext, BigBang bb) { + this.debug = debugContext.areScopesEnabled() || debugContext.areMetricsEnabled() ? debugContext : null; this.bb = bb; executorService = ForkJoinPool.commonPool(); state = new AtomicReference<>(State.UNUSED); @@ -205,31 +207,37 @@ public long complete() throws InterruptedException { lastPrint = System.nanoTime(); } - while (true) { - assert state.get() == State.STARTED : state.get(); - - boolean quiescent = executorService.awaitQuiescence(100, TimeUnit.MILLISECONDS); - if (timing != null && !quiescent) { - long curTime = System.nanoTime(); - if (curTime - lastPrint > timing.getPrintIntervalNanos()) { - timing.print(); - lastPrint = curTime; + try { + while (true) { + assert state.get() == State.STARTED : state.get(); + + boolean quiescent = executorService.awaitQuiescence(100, TimeUnit.MILLISECONDS); + if (timing != null && !quiescent) { + long curTime = System.nanoTime(); + if (curTime - lastPrint > timing.getPrintIntervalNanos()) { + timing.print(); + lastPrint = curTime; + } } - } - long completed = completedOperations.sum(); - long posted = postedOperations.sum(); - assert completed <= posted : completed + ", " + posted; - if (completed == posted && exceptions.isEmpty()) { - if (timing != null) { - timing.print(); - } + long completed = completedOperations.sum(); + long posted = postedOperations.sum(); + assert completed <= posted : completed + ", " + posted; + if (completed == posted && exceptions.isEmpty()) { + if (timing != null) { + timing.print(); + } - return posted; + return posted; + } + if (!exceptions.isEmpty()) { + setState(State.UNUSED); + throw new ParallelExecutionException(exceptions); + } } - if (!exceptions.isEmpty()) { - setState(State.UNUSED); - throw new ParallelExecutionException(exceptions); + } finally { + if (debug != null) { + debug.closeDumpHandlers(true); } } } diff --git a/substratevm/src/com.oracle.graal.reachability/src/com/oracle/graal/reachability/ReachabilityAnalysisEngine.java b/substratevm/src/com.oracle.graal.reachability/src/com/oracle/graal/reachability/ReachabilityAnalysisEngine.java index f1eab31b64cc..2e0e854b4834 100644 --- a/substratevm/src/com.oracle.graal.reachability/src/com/oracle/graal/reachability/ReachabilityAnalysisEngine.java +++ b/substratevm/src/com.oracle.graal.reachability/src/com/oracle/graal/reachability/ReachabilityAnalysisEngine.java @@ -31,12 +31,6 @@ import java.util.HashSet; import java.util.Set; -import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; -import jdk.graal.compiler.core.common.type.TypedConstant; -import jdk.graal.compiler.nodes.StructuredGraph; -import jdk.graal.compiler.options.OptionValues; -import jdk.graal.compiler.word.WordTypes; - import com.oracle.graal.pointsto.AbstractAnalysisEngine; import com.oracle.graal.pointsto.api.HostVM; import com.oracle.graal.pointsto.constraints.UnsupportedFeatures; @@ -52,6 +46,12 @@ import com.oracle.graal.pointsto.util.TimerCollection; import com.oracle.svm.common.meta.MultiMethod; +import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; +import jdk.graal.compiler.core.common.type.TypedConstant; +import jdk.graal.compiler.debug.DebugContext; +import jdk.graal.compiler.nodes.StructuredGraph; +import jdk.graal.compiler.options.OptionValues; +import jdk.graal.compiler.word.WordTypes; import jdk.vm.ci.code.BytecodePosition; import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.JavaConstant; @@ -79,9 +79,9 @@ public abstract class ReachabilityAnalysisEngine extends AbstractAnalysisEngine @SuppressWarnings("this-escape") public ReachabilityAnalysisEngine(OptionValues options, AnalysisUniverse universe, HostVM hostVM, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, TimerCollection timerCollection, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, UnsupportedFeatures unsupportedFeatures, DebugContext debugContext, TimerCollection timerCollection, ReachabilityMethodProcessingHandler reachabilityMethodProcessingHandler) { - super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, timerCollection); + super(options, universe, hostVM, metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, debugContext, timerCollection); this.executor.init(getTiming()); this.reachabilityTimer = timerCollection.createTimer("(reachability)"); diff --git a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java index 6677d3c29752..d93999e889dc 100644 --- a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java @@ -119,7 +119,7 @@ public int codeSizeFor(HostedMethod method) { @SuppressWarnings({"unused", "try"}) public void layoutMethods(DebugContext debug, BigBang bb) { try (Indent indent = debug.logAndIndent("layout methods")) { - BatchExecutor executor = new BatchExecutor(bb); + BatchExecutor executor = new BatchExecutor(debug, bb); try (StopTimer t = TimerCollection.createTimerAndStart("(bitcode)")) { writeBitcode(executor); } diff --git a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMToolchainUtils.java b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMToolchainUtils.java index d7b746d668fe..b10092fff482 100644 --- a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMToolchainUtils.java +++ b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMToolchainUtils.java @@ -169,8 +169,8 @@ public static void llvmCleanupStackMaps(DebugContext debug, String inputPath, Pa public static final class BatchExecutor { private CompletionExecutor executor; - public BatchExecutor(BigBang bb) { - this.executor = new CompletionExecutor(bb); + public BatchExecutor(DebugContext debug, BigBang bb) { + this.executor = new CompletionExecutor(debug, bb); executor.init(); } diff --git a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/objectfile/LLVMObjectFile.java b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/objectfile/LLVMObjectFile.java index bcce8a587fea..bc3801c339a8 100644 --- a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/objectfile/LLVMObjectFile.java +++ b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/objectfile/LLVMObjectFile.java @@ -191,7 +191,7 @@ public final void write(DebugContext context, Path outputFile) throws IOExceptio writeParts(); - BatchExecutor batchExecutor = new BatchExecutor(bb); + BatchExecutor batchExecutor = new BatchExecutor(context, bb); compileBitcodeBatches(batchExecutor, context, numBatches); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java index e7255096bfbf..769a01f4c27a 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java @@ -172,8 +172,8 @@ public SVMHost createHostVM(OptionValues options, ClassLoader classLoader, Class return new SVMHost(options, classLoader, classInitializationSupport, automaticSubstitutions, platform); } - public CompileQueue createCompileQueue(DebugContext debug, FeatureHandler featureHandler, HostedUniverse hostedUniverse, - RuntimeConfiguration runtimeConfiguration, boolean deoptimizeAll, SnippetReflectionProvider aSnippetReflection) { + public CompileQueue createCompileQueue(DebugContext debug, FeatureHandler featureHandler, HostedUniverse hostedUniverse, RuntimeConfiguration runtimeConfiguration, boolean deoptimizeAll, + SnippetReflectionProvider aSnippetReflection) { return new CompileQueue(debug, featureHandler, hostedUniverse, runtimeConfiguration, deoptimizeAll, aSnippetReflection); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index 1714fd92b78b..135503c4fbab 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -574,9 +574,7 @@ public static void clearSystemPropertiesForImage() { } @SuppressWarnings("try") - protected void doRun(Map entryPoints, - JavaMainSupport javaMainSupport, String imageName, NativeImageKind k, - SubstitutionProcessor harnessSubstitutions) { + protected void doRun(Map entryPoints, JavaMainSupport javaMainSupport, String imageName, NativeImageKind k, SubstitutionProcessor harnessSubstitutions) { List hostedEntryPoints = new ArrayList<>(); OptionValues options = HostedOptionValues.singleton(); @@ -676,7 +674,7 @@ protected void doRun(Map entryPoints, ImageSingletons.lookup(TemporaryBuildDirectoryProvider.class).getTemporaryBuildDirectory()); codeCache.layoutConstants(); codeCache.layoutMethods(debug, bb); - codeCache.buildRuntimeMetadata(bb.getSnippetReflectionProvider()); + codeCache.buildRuntimeMetadata(debug, bb.getSnippetReflectionProvider()); } AfterCompilationAccessImpl config = new AfterCompilationAccessImpl(featureHandler, loader, aUniverse, hUniverse, compileQueue.getCompilations(), codeCache, heap, debug, @@ -686,7 +684,7 @@ protected void doRun(Map entryPoints, } /* Re-run shadow heap verification after compilation. */ - aUniverse.getHeapVerifier().checkHeapSnapshot(hMetaAccess, "after compilation"); + aUniverse.getHeapVerifier().checkHeapSnapshot(debug, hMetaAccess, "after compilation"); CodeCacheProvider codeCacheProvider = runtimeConfiguration.getBackendForNormalMethod().getProviders().getCodeCache(); reporter.printCreationStart(); @@ -705,7 +703,7 @@ protected void doRun(Map entryPoints, featureHandler.forEachFeature(feature -> feature.afterHeapLayout(config)); /* Re-run shadow heap verification after heap layout. */ - aUniverse.getHeapVerifier().checkHeapSnapshot(hMetaAccess, "after heap layout"); + aUniverse.getHeapVerifier().checkHeapSnapshot(debug, hMetaAccess, "after heap layout"); createAbstractImage(k, hostedEntryPoints, heap, hMetaAccess, codeCache); @@ -925,7 +923,7 @@ protected void setupNativeImage(OptionValues options, Map")); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImagePointsToAnalysis.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImagePointsToAnalysis.java index 6411ed167001..063263b7ef68 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImagePointsToAnalysis.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImagePointsToAnalysis.java @@ -28,10 +28,6 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; -import jdk.graal.compiler.options.OptionValues; -import jdk.graal.compiler.word.WordTypes; - import com.oracle.graal.pointsto.PointsToAnalysis; import com.oracle.graal.pointsto.constraints.UnsupportedFeatures; import com.oracle.graal.pointsto.flow.MethodFlowsGraph; @@ -50,6 +46,10 @@ import com.oracle.svm.hosted.meta.HostedType; import com.oracle.svm.hosted.substitute.AnnotationSubstitutionProcessor; +import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; +import jdk.graal.compiler.debug.DebugContext; +import jdk.graal.compiler.options.OptionValues; +import jdk.graal.compiler.word.WordTypes; import jdk.vm.ci.code.BytecodePosition; import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.ResolvedJavaType; @@ -66,8 +66,9 @@ public NativeImagePointsToAnalysis(OptionValues options, AnalysisUniverse univer AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, AnnotationSubstitutionProcessor annotationSubstitutionProcessor, UnsupportedFeatures unsupportedFeatures, - TimerCollection timerCollection) { - super(options, universe, universe.hostVM(), metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, timerCollection, SubstrateOptions.parseOnce()); + DebugContext debugContext, TimerCollection timerCollection) { + super(options, universe, universe.hostVM(), metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, unsupportedFeatures, debugContext, timerCollection, + SubstrateOptions.parseOnce()); this.annotationSubstitutionProcessor = annotationSubstitutionProcessor; dynamicHubInitializer = new DynamicHubInitializer(this); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImageReachabilityAnalysisEngine.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImageReachabilityAnalysisEngine.java index 0330bd762c8e..7d013ac2f524 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImageReachabilityAnalysisEngine.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/analysis/NativeImageReachabilityAnalysisEngine.java @@ -36,6 +36,7 @@ import com.oracle.svm.hosted.substitute.AnnotationSubstitutionProcessor; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; +import jdk.graal.compiler.debug.DebugContext; import jdk.graal.compiler.options.OptionValues; import jdk.graal.compiler.word.WordTypes; import jdk.vm.ci.meta.ConstantReflectionProvider; @@ -49,9 +50,9 @@ public class NativeImageReachabilityAnalysisEngine extends ReachabilityAnalysisE @SuppressWarnings("this-escape") public NativeImageReachabilityAnalysisEngine(OptionValues options, AnalysisUniverse universe, AnalysisMetaAccess metaAccess, SnippetReflectionProvider snippetReflectionProvider, - ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, AnnotationSubstitutionProcessor annotationSubstitutionProcessor, TimerCollection timerCollection, - ReachabilityMethodProcessingHandler reachabilityMethodProcessingHandler) { - super(options, universe, universe.hostVM(), metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, new SubstrateUnsupportedFeatures(), timerCollection, + ConstantReflectionProvider constantReflectionProvider, WordTypes wordTypes, AnnotationSubstitutionProcessor annotationSubstitutionProcessor, DebugContext debugContext, + TimerCollection timerCollection, ReachabilityMethodProcessingHandler reachabilityMethodProcessingHandler) { + super(options, universe, universe.hostVM(), metaAccess, snippetReflectionProvider, constantReflectionProvider, wordTypes, new SubstrateUnsupportedFeatures(), debugContext, timerCollection, reachabilityMethodProcessingHandler); this.annotationSubstitutionProcessor = annotationSubstitutionProcessor; this.strengthenGraalGraphs = SubstrateOptions.parseOnce(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java index 71a3028aca69..52fac102b699 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java @@ -367,7 +367,7 @@ public CompileQueue(DebugContext debug, FeatureHandler featureHandler, HostedUni this.metaAccess = runtimeConfiguration.getProviders().getMetaAccess(); this.deoptimizeAll = deoptimizeAll; this.dataCache = new ConcurrentHashMap<>(); - this.executor = new CompletionExecutor(universe.getBigBang()); + this.executor = new CompletionExecutor(debug, universe.getBigBang()); this.featureHandler = featureHandler; this.snippetReflection = snippetReflection; this.graphTransplanter = createGraphTransplanter(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java index ece30a9b5148..768433377bc8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java @@ -262,11 +262,11 @@ public int getAlignedConstantsSize() { return ConfigurationValues.getObjectLayout().alignUp(getConstantsSize()); } - public void buildRuntimeMetadata(SnippetReflectionProvider snippetReflectionProvider) { - buildRuntimeMetadata(snippetReflectionProvider, new MethodPointer(getFirstCompilation().getLeft(), true), WordFactory.signed(getCodeAreaSize())); + public void buildRuntimeMetadata(DebugContext debug, SnippetReflectionProvider snippetReflectionProvider) { + buildRuntimeMetadata(debug, snippetReflectionProvider, new MethodPointer(getFirstCompilation().getLeft(), true), WordFactory.signed(getCodeAreaSize())); } - protected void buildRuntimeMetadata(SnippetReflectionProvider snippetReflection, CFunctionPointer firstMethod, UnsignedWord codeSize) { + protected void buildRuntimeMetadata(DebugContext debug, SnippetReflectionProvider snippetReflection, CFunctionPointer firstMethod, UnsignedWord codeSize) { // Build run-time metadata. HostedFrameInfoCustomization frameInfoCustomization = new HostedFrameInfoCustomization(); CodeInfoEncoder.Encoders encoders = new CodeInfoEncoder.Encoders(); @@ -424,7 +424,7 @@ protected void buildRuntimeMetadata(SnippetReflectionProvider snippetReflection, verifyDeoptEntries(imageCodeInfo); } - assert verifyMethods(hUniverse, codeInfoEncoder, imageCodeInfo); + assert verifyMethods(debug, hUniverse, codeInfoEncoder, imageCodeInfo); } protected HostedImageCodeInfo installCodeInfo(SnippetReflectionProvider snippetReflection, CFunctionPointer firstMethod, UnsignedWord codeSize, CodeInfoEncoder codeInfoEncoder, @@ -567,12 +567,12 @@ private static boolean error(HostedMethod method, long encodedBci, String msg) { return true; } - protected boolean verifyMethods(HostedUniverse hUniverse, CodeInfoEncoder codeInfoEncoder, CodeInfo codeInfo) { + protected boolean verifyMethods(DebugContext debug, HostedUniverse hUniverse, CodeInfoEncoder codeInfoEncoder, CodeInfo codeInfo) { /* * Run method verification in parallel to reduce computation time. */ BigBang bb = hUniverse.getBigBang(); - CompletionExecutor executor = new CompletionExecutor(bb); + CompletionExecutor executor = new CompletionExecutor(debug, bb); try { executor.init(); executor.start(); From c575470014c993ea837dd24765dc23d5c16c9182 Mon Sep 17 00:00:00 2001 From: Raphael Mosaner Date: Wed, 20 Sep 2023 10:49:35 +0200 Subject: [PATCH 16/78] [GR-48430] Make UnreachedCode deopts non-floatable. --- .../src/jdk/graal/compiler/nodes/DeoptimizeNode.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/DeoptimizeNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/DeoptimizeNode.java index 130e67616f95..99c3a86dcdac 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/DeoptimizeNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/DeoptimizeNode.java @@ -138,7 +138,8 @@ public boolean canFloat() { * high and would be executed under the wrong conditions. */ public static boolean canFloat(DeoptimizationReason reason, DeoptimizationAction action) { - return action != DeoptimizationAction.None && reason != DeoptimizationReason.Unresolved && reason != DeoptimizationReason.NotCompiledExceptionHandler; + return action != DeoptimizationAction.None && reason != DeoptimizationReason.Unresolved && reason != DeoptimizationReason.NotCompiledExceptionHandler && + reason != DeoptimizationReason.UnreachedCode; } @NodeIntrinsic From 1f588e552cc6732447987932c5afd958d18674fb Mon Sep 17 00:00:00 2001 From: Raphael Mosaner Date: Tue, 19 Sep 2023 14:17:18 +0200 Subject: [PATCH 17/78] Check if a speculation log is present for SpeculativeGuardMovement. --- .../compiler/core/test/GraalCompilerTest.java | 3 ++ .../compiler/core/test/HashCodeTest.java | 9 +++--- .../graal/compiler/hotspot/stubs/Stub.java | 4 +++ .../phases/SpeculativeGuardMovementPhase.java | 29 ++++++++++++++----- .../hotspot/HotSpotTruffleCompilerImpl.java | 4 +++ .../com/oracle/svm/graal/GraalSupport.java | 6 ++++ .../svm/hosted/NativeImageGenerator.java | 3 ++ 7 files changed, 46 insertions(+), 12 deletions(-) diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/GraalCompilerTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/GraalCompilerTest.java index b28fb68a3d20..446e2aa13d9e 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/GraalCompilerTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/GraalCompilerTest.java @@ -136,6 +136,7 @@ import jdk.graal.compiler.printer.GraalDebugHandlersFactory; import jdk.graal.compiler.runtime.RuntimeProvider; import jdk.graal.compiler.test.GraalTest; +import jdk.graal.compiler.loop.phases.SpeculativeGuardMovementPhase; import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; @@ -283,6 +284,8 @@ protected Suites createSuites(OptionValues opts) { ret.getHighTier().removeSubTypePhases(Speculative.class); ret.getMidTier().removeSubTypePhases(Speculative.class); ret.getLowTier().removeSubTypePhases(Speculative.class); + // remove after GR-49600 is resolved: + ret.getMidTier().replaceAllPhases(SpeculativeGuardMovementPhase.class, () -> new SpeculativeGuardMovementPhase(CanonicalizerPhase.create(), false, false)); } ListIterator> iter = ret.getHighTier().findPhase(ConvertDeoptimizeToGuardPhase.class, true); diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/HashCodeTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/HashCodeTest.java index cc823e482e95..fe55951e4176 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/HashCodeTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/HashCodeTest.java @@ -24,13 +24,13 @@ */ package jdk.graal.compiler.core.test; -import jdk.graal.compiler.core.phases.HighTier; -import jdk.graal.compiler.core.phases.MidTier; import jdk.graal.compiler.nodes.InvokeNode; import jdk.graal.compiler.nodes.StructuredGraph; import jdk.graal.compiler.options.OptionValues; import jdk.graal.compiler.phases.OptimisticOptimizations; import jdk.graal.compiler.phases.tiers.MidTierContext; +import jdk.graal.compiler.phases.tiers.Suites; + import org.junit.Assert; import org.junit.Test; @@ -120,8 +120,9 @@ public void test06() { private StructuredGraph buildGraphAfterMidTier(String name) { StructuredGraph g = parseForCompile(getResolvedJavaMethod(name)); OptionValues options = getInitialOptions(); - new HighTier(options).apply(g, getDefaultHighTierContext()); - new MidTier(options).apply(g, new MidTierContext(getProviders(), getTargetProvider(), OptimisticOptimizations.ALL, g.getProfilingInfo())); + Suites suites = createSuites(options); + suites.getHighTier().apply(g, getDefaultHighTierContext()); + suites.getMidTier().apply(g, new MidTierContext(getProviders(), getTargetProvider(), OptimisticOptimizations.ALL, g.getProfilingInfo())); return g; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/Stub.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/Stub.java index 698563930261..a8c93e302528 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/Stub.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/stubs/Stub.java @@ -61,6 +61,8 @@ import jdk.graal.compiler.phases.OptimisticOptimizations; import jdk.graal.compiler.phases.PhaseSuite; import jdk.graal.compiler.phases.Speculative; +import jdk.graal.compiler.loop.phases.SpeculativeGuardMovementPhase; +import jdk.graal.compiler.phases.common.CanonicalizerPhase; import jdk.graal.compiler.phases.tiers.HighTierContext; import jdk.graal.compiler.phases.tiers.Suites; import jdk.graal.compiler.printer.GraalDebugHandlersFactory; @@ -321,6 +323,8 @@ protected Suites createSuites() { defaultSuites.getMidTier().removeSubTypePhases(Speculative.class); defaultSuites.getLowTier().removeSubTypePhases(Speculative.class); + // remove after GR-49600 is resolved: + defaultSuites.getMidTier().replaceAllPhases(SpeculativeGuardMovementPhase.class, () -> new SpeculativeGuardMovementPhase(CanonicalizerPhase.create(), false, false)); return new Suites(emptyHighTier, defaultSuites.getMidTier(), defaultSuites.getLowTier()); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/SpeculativeGuardMovementPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/SpeculativeGuardMovementPhase.java index 14a1e889fc68..6052577de895 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/SpeculativeGuardMovementPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/SpeculativeGuardMovementPhase.java @@ -122,8 +122,19 @@ */ public class SpeculativeGuardMovementPhase extends PostRunCanonicalizationPhase implements FloatingGuardPhase { + private final boolean ignoreFrequency; + private final boolean requireSpeculationLog; + public SpeculativeGuardMovementPhase(CanonicalizerPhase canonicalizer) { super(canonicalizer); + this.ignoreFrequency = false; + this.requireSpeculationLog = true; + } + + public SpeculativeGuardMovementPhase(CanonicalizerPhase canonicalizer, boolean ignoreFrequency, boolean requireSpeculationLog) { + super(canonicalizer); + this.ignoreFrequency = ignoreFrequency; + this.requireSpeculationLog = requireSpeculationLog; } @Override @@ -158,7 +169,7 @@ protected void run(StructuredGraph graph, MidTierContext context) { } LoopsData loops = context.getLoopsDataProvider().getLoopsData(graph); loops.detectCountedLoops(); - iterate = performSpeculativeGuardMovement(context, graph, loops); + iterate = performSpeculativeGuardMovement(context, graph, loops, ignoreFrequency, requireSpeculationLog); } if (change.getNodes().isEmpty() || !iterate) { break; @@ -174,20 +185,21 @@ public void updateGraphState(GraphState graphState) { } public static boolean performSpeculativeGuardMovement(MidTierContext context, StructuredGraph graph, LoopsData loops) { - return performSpeculativeGuardMovement(context, graph, loops, null, false); + return performSpeculativeGuardMovement(context, graph, loops, null, false, true); } - public static boolean performSpeculativeGuardMovement(MidTierContext context, StructuredGraph graph, LoopsData loops, boolean ignoreFrequency) { - return performSpeculativeGuardMovement(context, graph, loops, null, ignoreFrequency); + public static boolean performSpeculativeGuardMovement(MidTierContext context, StructuredGraph graph, LoopsData loops, boolean ignoreFrequency, boolean requireSpeculationLog) { + return performSpeculativeGuardMovement(context, graph, loops, null, ignoreFrequency, requireSpeculationLog); } public static boolean performSpeculativeGuardMovement(MidTierContext context, StructuredGraph graph, LoopsData loops, NodeBitMap toProcess) { - return performSpeculativeGuardMovement(context, graph, loops, toProcess, false); + return performSpeculativeGuardMovement(context, graph, loops, toProcess, false, true); } - public static boolean performSpeculativeGuardMovement(MidTierContext context, StructuredGraph graph, LoopsData loops, NodeBitMap toProcess, boolean ignoreFrequency) { + public static boolean performSpeculativeGuardMovement(MidTierContext context, StructuredGraph graph, LoopsData loops, NodeBitMap toProcess, boolean ignoreFrequency, + boolean requireSpeculationLog) { SpeculativeGuardMovement spec = new SpeculativeGuardMovement(loops, graph.createNodeMap(), graph, context.getProfilingInfo(), graph.getSpeculationLog(), toProcess, - ignoreFrequency); + ignoreFrequency, requireSpeculationLog); spec.run(); return spec.iterate; } @@ -204,11 +216,12 @@ private static class SpeculativeGuardMovement implements Runnable { private final NodeBitMap toProcess; SpeculativeGuardMovement(LoopsData loops, NodeMap earliestCache, StructuredGraph graph, ProfilingInfo profilingInfo, SpeculationLog speculationLog, NodeBitMap toProcess, - boolean ignoreFrequency) { + boolean ignoreFrequency, boolean requireSpeculationLog) { this.loops = loops; this.earliestCache = earliestCache; this.graph = graph; this.profilingInfo = profilingInfo; + GraalError.guarantee(requireSpeculationLog ? speculationLog != null : true, "Graph has no speculation log attached: %s", graph); this.speculationLog = speculationLog; this.toProcess = toProcess; this.ignoreFrequency = ignoreFrequency; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/HotSpotTruffleCompilerImpl.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/HotSpotTruffleCompilerImpl.java index e74ae619b2c6..d4b0a6cdf27e 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/HotSpotTruffleCompilerImpl.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/HotSpotTruffleCompilerImpl.java @@ -61,6 +61,7 @@ import jdk.graal.compiler.lir.asm.CompilationResultBuilderFactory; import jdk.graal.compiler.lir.asm.EntryPointDecorator; import jdk.graal.compiler.lir.phases.LIRSuites; +import jdk.graal.compiler.loop.phases.SpeculativeGuardMovementPhase; import jdk.graal.compiler.nodes.StructuredGraph; import jdk.graal.compiler.nodes.StructuredGraph.AllowAssumptions; import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; @@ -74,6 +75,7 @@ import jdk.graal.compiler.phases.PhaseSuite; import jdk.graal.compiler.phases.Speculative; import jdk.graal.compiler.phases.common.AbstractInliningPhase; +import jdk.graal.compiler.phases.common.CanonicalizerPhase; import jdk.graal.compiler.phases.tiers.CompilerConfiguration; import jdk.graal.compiler.phases.tiers.HighTierContext; import jdk.graal.compiler.phases.tiers.Suites; @@ -414,6 +416,8 @@ private static void removeSpeculativePhases(Suites suites) { suites.getHighTier().removeSubTypePhases(Speculative.class); suites.getMidTier().removeSubTypePhases(Speculative.class); suites.getLowTier().removeSubTypePhases(Speculative.class); + // remove after GR-49600 is resolved: + suites.getMidTier().replaceAllPhases(SpeculativeGuardMovementPhase.class, () -> new SpeculativeGuardMovementPhase(CanonicalizerPhase.create(), false, false)); } @Override diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/GraalSupport.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/GraalSupport.java index 51fdbcb121fd..32019015dce9 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/GraalSupport.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/GraalSupport.java @@ -53,6 +53,7 @@ import jdk.graal.compiler.lir.LIRInstructionClass; import jdk.graal.compiler.lir.phases.LIRPhase; import jdk.graal.compiler.lir.phases.LIRSuites; +import jdk.graal.compiler.loop.phases.SpeculativeGuardMovementPhase; import jdk.graal.compiler.nodes.EncodedGraph; import jdk.graal.compiler.nodes.GraphDecoder; import jdk.graal.compiler.nodes.StructuredGraph; @@ -60,6 +61,7 @@ import jdk.graal.compiler.phases.BasePhase; import jdk.graal.compiler.phases.FloatingGuardPhase; import jdk.graal.compiler.phases.Speculative; +import jdk.graal.compiler.phases.common.CanonicalizerPhase; import jdk.graal.compiler.phases.tiers.Suites; import jdk.graal.compiler.phases.util.Providers; import jdk.graal.compiler.serviceprovider.GraalServices; @@ -208,6 +210,8 @@ private static Suites getWithoutSpeculative(Suites s) { effectiveSuites.getHighTier().removeSubTypePhases(Speculative.class); effectiveSuites.getMidTier().removeSubTypePhases(Speculative.class); effectiveSuites.getLowTier().removeSubTypePhases(Speculative.class); + // remove after GR-49600 is resolved: + effectiveSuites.getMidTier().replaceAllPhases(SpeculativeGuardMovementPhase.class, () -> new SpeculativeGuardMovementPhase(CanonicalizerPhase.create(), false, false)); return effectiveSuites; } @@ -219,6 +223,8 @@ private static Suites getWithExplicitExceptions(Suites s) { effectiveSuites.getHighTier().removeSubTypePhases(Speculative.class); effectiveSuites.getMidTier().removeSubTypePhases(Speculative.class); effectiveSuites.getLowTier().removeSubTypePhases(Speculative.class); + // remove after GR-49600 is resolved: + effectiveSuites.getMidTier().replaceAllPhases(SpeculativeGuardMovementPhase.class, () -> new SpeculativeGuardMovementPhase(CanonicalizerPhase.create(), false, false)); return effectiveSuites; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index c159659fdfc2..f5f5607b4464 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -83,6 +83,7 @@ import jdk.graal.compiler.java.BciBlockMapping; import jdk.graal.compiler.lir.phases.LIRSuites; import jdk.graal.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; +import jdk.graal.compiler.loop.phases.SpeculativeGuardMovementPhase; import jdk.graal.compiler.nodes.StructuredGraph; import jdk.graal.compiler.nodes.gc.BarrierSet; import jdk.graal.compiler.nodes.graphbuilderconf.ClassInitializationPlugin; @@ -1639,6 +1640,8 @@ private static void removePhases(Suites suites, Class c) { suites.getHighTier().removeSubTypePhases(c); suites.getMidTier().removeSubTypePhases(c); suites.getLowTier().removeSubTypePhases(c); + // remove after GR-49600 is resolved: + suites.getMidTier().replaceAllPhases(SpeculativeGuardMovementPhase.class, () -> new SpeculativeGuardMovementPhase(CanonicalizerPhase.create(), false, false)); } @SuppressWarnings("unused") From 768bff35717eb301ef2e63c3e223b810ec858102 Mon Sep 17 00:00:00 2001 From: Lesia Chaban Date: Tue, 24 Oct 2023 15:00:54 +0300 Subject: [PATCH 18/78] Update the download links for Java-on-Truffle, WASM, and LLVM --- .../java-on-truffle/README.md | 13 ++++++----- docs/reference-manual/llvm/README.md | 23 ++++++++++--------- docs/reference-manual/wasm/README.md | 22 +++++++++--------- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/docs/reference-manual/java-on-truffle/README.md b/docs/reference-manual/java-on-truffle/README.md index 8419c55d1df8..892154fecbb8 100644 --- a/docs/reference-manual/java-on-truffle/README.md +++ b/docs/reference-manual/java-on-truffle/README.md @@ -31,12 +31,13 @@ Java on Truffle passes the Java Compatibility Kit (JCK or TCK for Java SE). As of GraalVM for JDK 21, Java on Truffle (known as Espresso) is available as a standalone distribution. You can download a standalone based on Oracle GraalVM or GraalVM Community Edition. -1. Download the Java on Truffle (Espresso) standalone for your operating system: - * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/04F488A062424081E0631818000A781E/content) - * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/04F556B0056D3A58E0631818000A322D/content) - * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/04F556B005743A58E0631818000A322D/content) - * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/04F556B0057A3A58E0631818000A322D/content) - * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/04F488A062484081E0631818000A781E/content) +1. Download the Java on Truffle (Espresso) 23.1 standalone for your operating system: + + * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/069B12298B3349EDE0631718000A11BC/content) + * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01B97519AE0631718000AA34D/content) + * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/07867F4EBBAF81ADE0631718000AA7AB/content) + * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/069B12298B4949EDE0631718000A11BC/content) + * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/07867F4EBBBD81ADE0631718000AA7AB/content) 2. Unzip the archive: diff --git a/docs/reference-manual/llvm/README.md b/docs/reference-manual/llvm/README.md index c02f5c6bde2f..b2efcae5ea70 100644 --- a/docs/reference-manual/llvm/README.md +++ b/docs/reference-manual/llvm/README.md @@ -17,19 +17,20 @@ This allows seamless interoperability with the dynamic languages supported by Gr As of GraalVM for JDK 21, the GraalVM LLVM runtime is available as a standalone distribution. You can download a standalone based on Oracle GraalVM or GraalVM Community Edition. -1. Download the LLVM standalone for your operating system: +1. Download the LLVM 23.1 standalone for your operating system: + - Native standalone - * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB6993C6E0631818000A2314/content) - * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/04F488A0621A4081E0631818000A781E/content) - * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/04F488A0621F4081E0631818000A781E/content) - * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/04F488A062244081E0631818000A781E/content) - * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/04F488A062294081E0631818000A781E/content) + * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/07867F4EBC7C81ADE0631718000AA7AB/content) + * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/07867F4EBC8181ADE0631718000AA7AB/content) + * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01CBE519AE0631718000AA34D/content) + * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/069B12298BE249EDE0631718000A11BC/content) + * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/069B12298BE749EDE0631718000A11BC/content) - JVM standalone - * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/04F488A0622E4081E0631818000A781E/content) - * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB5D93C6E0631818000A2314/content) - * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/04F488A062384081E0631818000A781E/content) - * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/04F488A0623D4081E0631818000A781E/content) - * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/04F556B005683A58E0631818000A322D/content) + * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/069B12298BEC49EDE0631718000A11BC/content) + * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01CC3519AE0631718000AA34D/content) + * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/07867F4EBC8881ADE0631718000AA7AB/content) + * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01CC8519AE0631718000AA34D/content) + * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/07867F4EBC8D81ADE0631718000AA7AB/content) 2. Unzip the archive: diff --git a/docs/reference-manual/wasm/README.md b/docs/reference-manual/wasm/README.md index 16108df1b4b6..a5866d149534 100644 --- a/docs/reference-manual/wasm/README.md +++ b/docs/reference-manual/wasm/README.md @@ -16,19 +16,19 @@ The support for WebAssembly is in the early stages of its development. As of GraalVM for JDK 21, the GraalVM WebAssembly runtime (known as Wasm) is available as a standalone distribution. You can download a standalone based on Oracle GraalVM or GraalVM Community Edition. -1. Download the Wasm standalone for your operating system: +1. Download the Wasm 23.1 standalone for your operating system: - Native standalone - * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB3A93C6E0631818000A2314/content) - * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB3F93C6E0631818000A2314/content) - * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/04F488A062154081E0631818000A781E/content) - * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB4493C6E0631818000A2314/content) - * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB4993C6E0631818000A2314/content) + * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01C4D519AE0631718000AA34D/content) + * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01C5E519AE0631718000AA34D/content) + * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/069B12298B9149EDE0631718000A11BC/content) + * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01C7B519AE0631718000AA34D/content) + * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/069B12298BB049EDE0631718000A11BC/content) - JVM standalone - * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB4E93C6E0631818000A2314/content) - * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB5393C6E0631818000A2314/content) - * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB5893C6E0631818000A2314/content) - * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB5D93C6E0631818000A2314/content) - * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/05013E46CB6293C6E0631818000A2314/content) + * [Linux x64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01C98519AE0631718000AA34D/content) + * [Linux AArch64](https://gds.oracle.com/api/20220101/artifacts/069B12298BDD49EDE0631718000A11BC/content) + * [macOS x64](https://gds.oracle.com/api/20220101/artifacts/07867F4EBC6481ADE0631718000AA7AB/content) + * [macOS AArch64](https://gds.oracle.com/api/20220101/artifacts/069B4EC01CB9519AE0631718000AA34D/content) + * [Windows x64](https://gds.oracle.com/api/20220101/artifacts/07867F4EBC7781ADE0631718000AA7AB/content) 2. Unzip the archive: From c8542a9dc3c4b6a30d8e5eb307d6ed5f4951fb74 Mon Sep 17 00:00:00 2001 From: Peter Hofer Date: Tue, 24 Oct 2023 14:20:07 +0200 Subject: [PATCH 19/78] Correctly print JSON values in agent trace files. --- .../svm/agent/tracing/TraceFileWriter.java | 6 ++-- .../oracle/svm/core/util/json/JsonWriter.java | 30 +++++++++++++------ .../com/oracle/svm/driver/BundleSupport.java | 4 +-- .../ImageHeapConnectedComponentsPrinter.java | 2 +- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/tracing/TraceFileWriter.java b/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/tracing/TraceFileWriter.java index 206d05a7b024..4fe924e444a8 100644 --- a/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/tracing/TraceFileWriter.java +++ b/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/tracing/TraceFileWriter.java @@ -99,13 +99,13 @@ private static void printArray(JsonWriter json, Object[] array) throws IOExcepti } private static void printValue(JsonWriter json, Object value) throws IOException { - String s = null; + Object s = null; if (value instanceof byte[]) { s = Base64.getEncoder().encodeToString((byte[]) value); } else if (value != null) { - s = value.toString(); + s = value; } - json.quote(s); + json.printValue(s); } private void traceEntry(String s) throws IOException { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/json/JsonWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/json/JsonWriter.java index b59d6f0143da..10b803b8a17b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/json/JsonWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/json/JsonWriter.java @@ -82,11 +82,11 @@ public JsonWriter appendFieldSeparator() throws IOException { } public JsonWriter appendKeyValue(String key, Object value) throws IOException { - return quote(key).appendFieldSeparator().quote(value); + return quote(key).appendFieldSeparator().printValue(value); } @SuppressWarnings({"unchecked", "rawtypes"}) - public void print(Object value) throws IOException { + public JsonWriter print(Object value) throws IOException { if (value instanceof Map map) { printMap(map); // Must always be } else if (value instanceof Iterator it) { @@ -94,8 +94,9 @@ public void print(Object value) throws IOException { } else if (value instanceof List list) { printIterator(list.iterator()); } else { - quote(value); + printValue(value); } + return this; } @SuppressWarnings("unchecked") @@ -130,15 +131,26 @@ private void printIterator(Iterator iter) throws IOException { append(']'); } - public JsonWriter quote(Object o) throws IOException { + public JsonWriter printValue(Object o) throws IOException { if (o == null) { return append("null"); - } else if (Boolean.TRUE.equals(o)) { - return append("true"); - } else if (Boolean.FALSE.equals(o)) { - return append("false"); - } else if (o instanceof Number) { + } else if (o instanceof Boolean || o instanceof Byte || o instanceof Short || o instanceof Integer || o instanceof Long) { + /* + * Note that sub-integer values here most likely become Integer objects when parsing, + * and comparisons such as equals() or compareTo() on boxed values only work on the + * exact same type. (Boolean values, however, should be deserialized as Boolean). + */ return append(o.toString()); + } else if (o instanceof Float f) { + if (f.isNaN() || f.isInfinite()) { + return quote(f.toString()); // cannot express, best we can do without failing + } + return append(f.toString()); + } else if (o instanceof Double d) { + if (d.isNaN() || d.isInfinite()) { + return quote(d.toString()); // cannot express, best we can do without failing + } + return append(d.toString()); } else { return quote(o.toString()); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 5eb2678dec0c..dba05ac7d47f 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -860,8 +860,8 @@ private static Manifest createManifest() { private static final String substitutionMapDstField = "dst"; private static void printPathMapping(Map.Entry entry, JsonWriter w) throws IOException { - w.append('{').quote(substitutionMapSrcField).append(':').quote(entry.getKey()); - w.append(',').quote(substitutionMapDstField).append(':').quote(entry.getValue()); + w.append('{').quote(substitutionMapSrcField).append(':').printValue(entry.getKey()); + w.append(',').quote(substitutionMapDstField).append(':').printValue(entry.getValue()); w.append('}'); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageHeapConnectedComponentsPrinter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageHeapConnectedComponentsPrinter.java index ad1db7eb3ffd..ff00eea47e36 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageHeapConnectedComponentsPrinter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageHeapConnectedComponentsPrinter.java @@ -216,7 +216,7 @@ public void printSummaryInfoForEveryObjectInConnectedComponents(PrintWriter out) for (Iterator iterator = connectedComponents.iterator(); iterator.hasNext();) { ConnectedComponent connectedComponent = iterator.next(); writer.append('{').newline(); - writer.quote("componentId").append(':').quote(connectedComponent.getId()).append(',').newline(); + writer.quote("componentId").append(':').printValue(connectedComponent.getId()).append(',').newline(); writer.quote("sizeInBytes").append(':').append(String.valueOf(connectedComponent.getSizeInBytes())).append(',').newline(); writer.quote("objects").append(":["); List objects = connectedComponent.getObjects(); From a180a4b70b97aa1b2ba1fa1b28b683e55173322a Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 11 May 2023 13:44:02 +0200 Subject: [PATCH 20/78] Move resource registration to beforeAnalysis phase --- .../oracle/svm/hosted/ResourcesFeature.java | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 99f6b23593fe..61b121e9ffb4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -83,7 +83,6 @@ import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; -import com.oracle.svm.hosted.FeatureImpl.DuringAnalysisAccessImpl; import com.oracle.svm.hosted.config.ConfigurationParserUtils; import com.oracle.svm.hosted.jdk.localization.LocalizationFeature; import com.oracle.svm.util.LogUtils; @@ -133,7 +132,7 @@ public static class Options { } private boolean sealed = false; - private final Set resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private Set resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set excludedResourcePatterns = Collections.newSetFromMap(new ConcurrentHashMap<>()); private int loadedConfigurations; private ImageClassLoader imageClassLoader; @@ -220,6 +219,31 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { resourcePatternWorkSet.addAll(Options.IncludeResources.getValue().values()); excludedResourcePatterns.addAll(Options.ExcludeResources.getValue().values()); + + if (!resourcePatternWorkSet.isEmpty()) { + FeatureImpl.BeforeAnalysisAccessImpl beforeAnalysisAccess = (FeatureImpl.BeforeAnalysisAccessImpl) access; + ResourcePattern[] includePatterns = compilePatterns(resourcePatternWorkSet); + if (MissingRegistrationUtils.throwMissingRegistrationErrors()) { + for (ResourcePattern resourcePattern : includePatterns) { + Resources.singleton().registerIncludePattern(resourcePattern.moduleName, resourcePattern.pattern.pattern()); + } + } + + ResourcePattern[] excludePatterns = compilePatterns(excludedResourcePatterns); + DebugContext debugContext = beforeAnalysisAccess.getDebugContext(); + ResourceCollectorImpl collector = new ResourceCollectorImpl(debugContext, includePatterns, excludePatterns); + try { + collector.prepareProgressReporter(); + ImageSingletons.lookup(ClassLoaderSupport.class).collectResources(collector); + } finally { + collector.shutDownProgressReporter(); + } + + // We set resourcePatternWorkSet to empty unmodifiable set, so we can be sure that it + // won't be populated in some later phase + resourcePatternWorkSet = Set.of(); + } + resourceRegistryImpl().flushConditionalConfiguration(access); } @@ -320,34 +344,6 @@ public void registerNegativeQuery(Module module, String resourceName) { } } - @Override - public void duringAnalysis(DuringAnalysisAccess access) { - resourceRegistryImpl().flushConditionalConfiguration(access); - if (resourcePatternWorkSet.isEmpty()) { - return; - } - - access.requireAnalysisIteration(); - - DuringAnalysisAccessImpl duringAnalysisAccess = ((DuringAnalysisAccessImpl) access); - ResourcePattern[] includePatterns = compilePatterns(resourcePatternWorkSet); - if (MissingRegistrationUtils.throwMissingRegistrationErrors()) { - for (ResourcePattern resourcePattern : includePatterns) { - Resources.singleton().registerIncludePattern(resourcePattern.moduleName, resourcePattern.pattern.pattern()); - } - } - ResourcePattern[] excludePatterns = compilePatterns(excludedResourcePatterns); - DebugContext debugContext = duringAnalysisAccess.getDebugContext(); - ResourceCollectorImpl collector = new ResourceCollectorImpl(debugContext, includePatterns, excludePatterns); - try { - collector.prepareProgressReporter(); - ImageSingletons.lookup(ClassLoaderSupport.class).collectResources(collector); - } finally { - collector.shutDownProgressReporter(); - } - resourcePatternWorkSet.clear(); - } - private ResourcePattern[] compilePatterns(Set patterns) { return patterns.stream() .filter(s -> s.length() > 0) From 252861c3db027c6eb4c4325588a824c9c27e8d01 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 11 May 2023 16:51:23 +0200 Subject: [PATCH 21/78] Try to speed resource traversing by introducing parallelism --- .../src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 37dfcb4134b8..1452c978ba52 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -106,6 +106,7 @@ private static Stream extractModuleLookupData(ModuleLayer la public void collectResources(ResourceCollector resourceCollector) { /* Collect resources from modules */ NativeImageClassLoaderSupport.allLayers(classLoaderSupport.moduleLayerForImageBuild).stream() + .parallel() .flatMap(ClassLoaderSupportImpl::extractModuleLookupData) .forEach(lookup -> collectResourceFromModule(resourceCollector, lookup)); From ee5f501558c3ea1ca0615d418f5491e02573de58 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 16 May 2023 12:43:27 +0200 Subject: [PATCH 22/78] Change function that public API uses to register resource --- .../hosted/RuntimeResourceAccess.java | 3 +-- .../impl/RuntimeResourceSupport.java | 2 ++ .../oracle/svm/hosted/ResourcesFeature.java | 24 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java index 355a4f06e4f7..d6dd9c83a71d 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java @@ -69,8 +69,7 @@ public final class RuntimeResourceAccess { public static void addResource(Module module, String resourcePath) { Objects.requireNonNull(module); Objects.requireNonNull(resourcePath); - ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), - withModuleName(module, Pattern.quote(resourcePath))); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(module, resourcePath); } /** diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java index 68242b311d69..3c18afe72df5 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java @@ -46,6 +46,8 @@ public interface RuntimeResourceSupport { void addResources(ConfigurationCondition condition, String pattern); + void addResources(Module module, String resourcePath); + void injectResource(Module module, String resourcePath, byte[] resourceContent); void ignoreResources(ConfigurationCondition condition, String pattern); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 61b121e9ffb4..91f7b1905f80 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -27,12 +27,14 @@ import static com.oracle.svm.core.jdk.Resources.RESOURCES_INTERNAL_PATH_SEPARATOR; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.net.URI; +import java.net.URL; import java.nio.file.FileSystem; import java.util.ArrayList; import java.util.Arrays; @@ -40,6 +42,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -155,6 +158,27 @@ public void addResources(ConfigurationCondition condition, String pattern) { }); } + @Override + public void addResources(Module module, String resourcePath) { + InputStream is; + boolean fromJar; + + if (module.isNamed()) { + try { + fromJar = new File(Objects.requireNonNull(module.getClassLoader().getResource(resourcePath)).getPath()).isDirectory(); + is = module.getResourceAsStream(resourcePath); + } catch (IOException e) { + // we ignore if user provided resource that doesn't exist + return; + } + } else { + fromJar = new File(Objects.requireNonNull(imageClassLoader.getClassLoader().getResource(resourcePath)).getPath()).isDirectory(); + is = imageClassLoader.getClassLoader().getResourceAsStream(resourcePath); + } + + Resources.registerResource(module, resourcePath, is, fromJar); + } + @Override public void injectResource(Module module, String resourcePath, byte[] resourceContent) { Resources.singleton().registerResource(module, resourcePath, resourceContent); From f3e59808baf5a81c365db52283868ecb77e6c634 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 19 May 2023 12:54:08 +0200 Subject: [PATCH 23/78] Add resource registration through API without patterns --- .../config/ResourceConfigurationTest.java | 4 +++ .../config/ResourceConfiguration.java | 25 +++++++++++-------- .../oracle/svm/hosted/ResourcesFeature.java | 4 +-- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java index 600b21440bb4..51f6e6cebf9b 100644 --- a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java +++ b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java @@ -93,6 +93,10 @@ public void addResources(ConfigurationCondition condition, String pattern) { addedResources.add(pattern); } + @Override + public void addResources(Module module, String resourcePath) { + } + @Override public void injectResource(Module module, String resourcePath, byte[] resourceContent) { } diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java index 803fbb86fa34..7824d1f51d2c 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java @@ -24,6 +24,16 @@ */ package com.oracle.svm.configure.config; +import com.oracle.svm.configure.ConfigurationBase; +import com.oracle.svm.core.configure.ConditionalElement; +import com.oracle.svm.core.configure.ConfigurationParser; +import com.oracle.svm.core.configure.ResourceConfigurationParser; +import com.oracle.svm.core.configure.ResourcesRegistry; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.core.util.json.JsonPrinter; +import com.oracle.svm.core.util.json.JsonWriter; +import org.graalvm.nativeimage.impl.ConfigurationCondition; + import java.io.IOException; import java.util.Collection; import java.util.Comparator; @@ -35,17 +45,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; -import org.graalvm.nativeimage.impl.ConfigurationCondition; - -import com.oracle.svm.configure.ConfigurationBase; -import com.oracle.svm.core.util.json.JsonPrinter; -import com.oracle.svm.core.util.json.JsonWriter; -import com.oracle.svm.core.configure.ConditionalElement; -import com.oracle.svm.core.configure.ConfigurationParser; -import com.oracle.svm.core.configure.ResourceConfigurationParser; -import com.oracle.svm.core.configure.ResourcesRegistry; -import com.oracle.svm.core.util.VMError; - public final class ResourceConfiguration extends ConfigurationBase { private static final String PROPERTY_BUNDLE = "java.util.PropertyResourceBundle"; @@ -63,6 +62,10 @@ public void addResources(ConfigurationCondition condition, String pattern) { configuration.addResourcePattern(condition, pattern); } + @Override + public void addResources(Module module, String resourcePath) { + } + @Override public void injectResource(Module module, String resourcePath, byte[] resourceContent) { VMError.shouldNotReachHere("Resource injection is only supported via Feature implementation"); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 91f7b1905f80..b4459a800068 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -165,14 +165,14 @@ public void addResources(Module module, String resourcePath) { if (module.isNamed()) { try { - fromJar = new File(Objects.requireNonNull(module.getClassLoader().getResource(resourcePath)).getPath()).isDirectory(); + fromJar = Objects.requireNonNull(module.getClassLoader().getResource(resourcePath)).getPath().contains(".jar"); is = module.getResourceAsStream(resourcePath); } catch (IOException e) { // we ignore if user provided resource that doesn't exist return; } } else { - fromJar = new File(Objects.requireNonNull(imageClassLoader.getClassLoader().getResource(resourcePath)).getPath()).isDirectory(); + fromJar = Objects.requireNonNull(imageClassLoader.getClassLoader().getResource(resourcePath)).getPath().contains(".jar"); is = imageClassLoader.getClassLoader().getResourceAsStream(resourcePath); } From 7865a6cb671f6584d566b0c4cb7385505c29af06 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 19 May 2023 15:51:11 +0200 Subject: [PATCH 24/78] Try to open modules and decide whether resource comes from jar --- .../hosted/RuntimeResourceAccess.java | 9 +++--- .../oracle/svm/hosted/ResourcesFeature.java | 30 +++++++++++++++---- .../com/oracle/svm/util/ModuleSupport.java | 2 +- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java index d6dd9c83a71d..885a7ab8564f 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java @@ -40,17 +40,16 @@ */ package org.graalvm.nativeimage.hosted; -import java.util.Arrays; -import java.util.Locale; -import java.util.Objects; -import java.util.regex.Pattern; - import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.graalvm.nativeimage.impl.RuntimeResourceSupport; +import java.util.Arrays; +import java.util.Locale; +import java.util.Objects; + /** * This class can be used to register Java resources and ResourceBundles that should be accessible * at run time. diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index b4459a800068..4f24e2c0044c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -27,7 +27,6 @@ import static com.oracle.svm.core.jdk.Resources.RESOURCES_INTERNAL_PATH_SEPARATOR; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; @@ -42,7 +41,6 @@ import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -89,6 +87,7 @@ import com.oracle.svm.hosted.config.ConfigurationParserUtils; import com.oracle.svm.hosted.jdk.localization.LocalizationFeature; import com.oracle.svm.util.LogUtils; +import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -165,15 +164,36 @@ public void addResources(Module module, String resourcePath) { if (module.isNamed()) { try { - fromJar = Objects.requireNonNull(module.getClassLoader().getResource(resourcePath)).getPath().contains(".jar"); + String resourcePackage = jdk.internal.module.Resources.toPackageName(resourcePath); + if (!resourcePackage.isEmpty()) { + if (!module.getPackages().contains(resourcePackage)) { + return; + } + ModuleSupport.accessModuleByClass(ModuleSupport.Access.OPEN, ResourcesFeature.class, module, resourcePackage); + } is = module.getResourceAsStream(resourcePath); + if (is == null) { + return; + } + + fromJar = false; } catch (IOException e) { // we ignore if user provided resource that doesn't exist return; } } else { - fromJar = Objects.requireNonNull(imageClassLoader.getClassLoader().getResource(resourcePath)).getPath().contains(".jar"); - is = imageClassLoader.getClassLoader().getResourceAsStream(resourcePath); + URL url = imageClassLoader.getClassLoader().getResource(resourcePath); + if (url != null) { + fromJar = url.getProtocol().equalsIgnoreCase("jar"); + try { + is = url.openStream(); + } catch (IOException e) { + // we ignore if user provided resource that doesn't exist + return; + } + } else { + return; + } } Resources.registerResource(module, resourcePath, is, fromJar); diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java index 03ec51bdfa18..4b5738a6116d 100644 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java +++ b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java @@ -131,7 +131,7 @@ public static void accessPackagesToClass(Access access, Class accessingClass, } @Platforms(Platform.HOSTED_ONLY.class) - private static void accessModuleByClass(Access access, Class accessingClass, Module declaringModule, String packageName) { + public static void accessModuleByClass(Access access, Class accessingClass, Module declaringModule, String packageName) { Module namedAccessingModule = null; if (accessingClass != null) { Module accessingModule = accessingClass.getModule(); From 705835d113619d35b7d75a2d76bc3522987f0a7b Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 25 May 2023 13:26:08 +0200 Subject: [PATCH 25/78] Replace CharsetSubstitution resource adding --- .../hosted/jdk/localization/CharsetSubstitutionsFeature.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java index 0d54073ac9ba..236d9d7c5d36 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java @@ -25,6 +25,7 @@ package com.oracle.svm.hosted.jdk.localization; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; import org.graalvm.nativeimage.impl.ConfigurationCondition; import com.oracle.svm.core.configure.ResourcesRegistry; @@ -35,6 +36,6 @@ class CharsetSubstitutionsFeature implements InternalFeature { @Override public void beforeAnalysis(BeforeAnalysisAccess access) { - ImageSingletons.lookup(ResourcesRegistry.class).addResources(ConfigurationCondition.create("java.lang.CharacterName"), "java/lang/uniName.dat"); + access.registerReachabilityHandler((e) -> RuntimeResourceAccess.addResource(ModuleLayer.boot().findModule("java.base").get(), "java/lang/uniName.dat"), access.findClassByName("java.lang.CharacterName")); } } From eaafc3d8ba61e7a49282854f980c3d5014185b60 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Mon, 29 May 2023 12:07:10 +0200 Subject: [PATCH 26/78] Change addResource function to check whether the entry is directory or not --- .../hosted/RuntimeResourceAccess.java | 2 +- .../impl/RuntimeResourceSupport.java | 2 +- .../config/ResourceConfigurationTest.java | 4 +- .../config/ResourceConfiguration.java | 3 +- .../com/oracle/svm/core/jdk/Resources.java | 4 ++ .../oracle/svm/hosted/ResourcesFeature.java | 64 +++++++++++++------ .../CharsetSubstitutionsFeature.java | 8 +-- 7 files changed, 59 insertions(+), 28 deletions(-) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java index 885a7ab8564f..76c43f86efd2 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java @@ -68,7 +68,7 @@ public final class RuntimeResourceAccess { public static void addResource(Module module, String resourcePath) { Objects.requireNonNull(module); Objects.requireNonNull(resourcePath); - ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(module, resourcePath); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourcePath); } /** diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java index 3c18afe72df5..2d370a511ffd 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java @@ -46,7 +46,7 @@ public interface RuntimeResourceSupport { void addResources(ConfigurationCondition condition, String pattern); - void addResources(Module module, String resourcePath); + void addResource(Module module, String resourcePath); void injectResource(Module module, String resourcePath, byte[] resourceContent); diff --git a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java index 51f6e6cebf9b..ef28c19bcd99 100644 --- a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java +++ b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Locale; +import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.junit.Assert; import org.junit.Test; @@ -94,7 +95,8 @@ public void addResources(ConfigurationCondition condition, String pattern) { } @Override - public void addResources(Module module, String resourcePath) { + public void addResource(Module module, String resourcePath) { + VMError.shouldNotReachHere("Unused function."); } @Override diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java index 7824d1f51d2c..c7dcb46b78e1 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java @@ -63,7 +63,8 @@ public void addResources(ConfigurationCondition condition, String pattern) { } @Override - public void addResources(Module module, String resourcePath) { + public void addResource(Module module, String resourcePath) { + VMError.shouldNotReachHere("Unused function."); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 00faeab2e2cb..1240eebf95d0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -214,6 +214,10 @@ public void registerResource(Module module, String resourceName, InputStream is, addEntry(module, resourceName, false, inputStreamToByteArray(is), fromJar); } + @Platforms(Platform.HOSTED_ONLY.class) + public static void registerResource(Module module, String resourceName, boolean isDir, InputStream is, boolean fromJar) { + addEntry(module, resourceName, isDir, inputStreamToByteArray(is), fromJar); + } @Platforms(Platform.HOSTED_ONLY.class) public void registerDirectoryResource(String resourceDirName, String content) { registerDirectoryResource(null, resourceDirName, content, true); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 4f24e2c0044c..871ef546062b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -33,8 +33,11 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -47,6 +50,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; +import java.util.jar.JarFile; import java.util.regex.Pattern; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; @@ -157,10 +161,25 @@ public void addResources(ConfigurationCondition condition, String pattern) { }); } + private String urlToJarPath(URL url) { + return String.valueOf(url).split("jar:file:")[1].split("!")[0]; + } + + private boolean resourceIsDirectory(URL url, boolean fromJar, String resourcePath) throws IOException, URISyntaxException { + if (fromJar) { + try (JarFile jf = new JarFile(urlToJarPath(url))) { + return jf.getEntry(resourcePath).isDirectory(); + } + } else { + return Files.isDirectory(Path.of(url.toURI())); + } + } + @Override - public void addResources(Module module, String resourcePath) { + public void addResource(Module module, String resourcePath) { InputStream is; - boolean fromJar; + boolean fromJar = false; + boolean isDirectory = false; if (module.isNamed()) { try { @@ -171,32 +190,41 @@ public void addResources(Module module, String resourcePath) { } ModuleSupport.accessModuleByClass(ModuleSupport.Access.OPEN, ResourcesFeature.class, module, resourcePackage); } - is = module.getResourceAsStream(resourcePath); - if (is == null) { - return; - } - fromJar = false; + is = module.getResourceAsStream(resourcePath); } catch (IOException e) { - // we ignore if user provided resource that doesn't exist + // we should ignore if user failed to provide resource return; } } else { URL url = imageClassLoader.getClassLoader().getResource(resourcePath); - if (url != null) { - fromJar = url.getProtocol().equalsIgnoreCase("jar"); - try { - is = url.openStream(); - } catch (IOException e) { - // we ignore if user provided resource that doesn't exist - return; - } - } else { + if (url == null) { + // we should ignore if user failed to provide resource return; } + + fromJar = url.getProtocol().equalsIgnoreCase("jar"); + try { + isDirectory = resourceIsDirectory(url, fromJar, resourcePath); + } catch (IOException e) { + // we should ignore if user failed to provide resource + return; + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + + try { + is = url.openStream(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + if (is == null) { + return; } - Resources.registerResource(module, resourcePath, is, fromJar); + Resources.registerResource(module, resourcePath, isDirectory, is, fromJar); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java index 236d9d7c5d36..c863b5c6261b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java @@ -24,13 +24,9 @@ */ package com.oracle.svm.hosted.jdk.localization; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; -import org.graalvm.nativeimage.impl.ConfigurationCondition; - -import com.oracle.svm.core.configure.ResourcesRegistry; -import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; +import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; @AutomaticallyRegisteredFeature class CharsetSubstitutionsFeature implements InternalFeature { From 82fdd40da4af4fb3a1d90ff98a0743a913934342 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Mon, 29 May 2023 16:31:18 +0200 Subject: [PATCH 27/78] Extract resource adding from beforeAnalysis to duringSetup in TruffleBaseFeature --- .../src/com/oracle/svm/truffle/TruffleBaseFeature.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java index 75f89555ad7b..750d6e4d5cac 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java @@ -312,7 +312,6 @@ public void afterRegistration(AfterRegistrationAccess a) { initializeHomeFinder(); needsAllEncodings = invokeStaticMethod("com.oracle.truffle.polyglot.LanguageCache", "getNeedsAllEncodings", Collections.emptyList()); - // reinitialize language cache invokeStaticMethod("com.oracle.truffle.api.library.LibraryFactory", "reinitializeNativeImageState", Collections.emptyList()); @@ -449,6 +448,10 @@ public void duringSetup(DuringSetupAccess access) { if (Options.TruffleCheckPreinitializedFiles.getValue()) { access.registerObjectReplacer(new TruffleFileCheck(((FeatureImpl.DuringSetupAccessImpl) access).getHostVM().getClassInitializationSupport())); } + + if (needsAllEncodings) { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), "org/graalvm/shadowed/org/jcodings/tables/.*bin$"); + } } void setGraalGraphObjectReplacer(GraalGraphObjectReplacer graalGraphObjectReplacer) { @@ -482,9 +485,6 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { config.registerSubtypeReachabilityHandler(TruffleBaseFeature::registerTruffleLibrariesAsInHeap, LibraryExport.class); - if (needsAllEncodings) { - ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), "org/graalvm/shadowed/org/jcodings/tables/.*bin$"); - } Class frameClass = config.findClassByName("com.oracle.truffle.api.impl.FrameWithoutBoxing"); config.registerFieldValueTransformer(config.findField(frameClass, "ASSERTIONS_ENABLED"), new AssertionStatusFieldTransformer(frameClass)); } From 5e717506506dc74a1109fb3120d13c36e0ce7f1f Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 30 May 2023 15:12:40 +0200 Subject: [PATCH 28/78] Extract bunndles registration from pattern based addResources --- .../oracle/svm/hosted/ResourcesFeature.java | 40 +++++++++++++++++++ .../hosted/xml/XMLParsersRegistration.java | 15 ++++++- .../svm/truffle/TruffleBaseFeature.java | 30 ++++++++------ 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 871ef546062b..dc06d85b6fe3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -68,6 +68,46 @@ import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.graalvm.nativeimage.impl.RuntimeResourceSupport; +import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; +import org.graalvm.compiler.options.Option; +import org.graalvm.compiler.options.OptionType; +import org.graalvm.compiler.phases.util.Providers; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; +import org.graalvm.nativeimage.impl.ConfigurationCondition; +import org.graalvm.nativeimage.impl.RuntimeResourceSupport; + +import com.oracle.svm.core.ClassLoaderSupport; +import com.oracle.svm.core.ClassLoaderSupport.ResourceCollector; +import com.oracle.svm.core.ParsingReason; +import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.configure.ConfigurationFile; +import com.oracle.svm.core.configure.ConfigurationFiles; +import com.oracle.svm.core.configure.ResourceConfigurationParser; +import com.oracle.svm.core.configure.ResourcesRegistry; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.jdk.Resources; +import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributes; +import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributesView; +import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystem; +import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystemProvider; +import com.oracle.svm.core.option.HostedOptionKey; +import com.oracle.svm.core.option.LocatableMultiOptionValue; +import com.oracle.svm.core.util.UserError; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.config.ConfigurationParserUtils; +import com.oracle.svm.hosted.jdk.localization.LocalizationFeature; +import com.oracle.svm.util.ModuleSupport; +import com.oracle.svm.util.ReflectionUtil; + +import jdk.vm.ci.meta.ResolvedJavaMethod; import com.oracle.svm.core.ClassLoaderSupport; import com.oracle.svm.core.ClassLoaderSupport.ResourceCollector; import com.oracle.svm.core.MissingRegistrationUtils; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/XMLParsersRegistration.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/XMLParsersRegistration.java index ed1cb565df8f..13940b2be407 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/XMLParsersRegistration.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/XMLParsersRegistration.java @@ -135,7 +135,20 @@ void registerResources() { ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xml.internal.serializer.utils.SerializerMessages"); ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xalan.internal.xsltc.compiler.util.ErrorMessages"); - ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), "com.sun.*.properties"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xml.internal.serializer.Encodings"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xml.internal.serializer.HTMLEntities"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xml.internal.serializer.XMLEntities"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.xpath.regex.message"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.DOMMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.DatatypeMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.JAXPValidationMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.SAXMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.XIncludeMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.XMLMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.XMLSchemaMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.XMLSerializerMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xerces.internal.impl.msg.XPointerMessages"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResourceBundles(ConfigurationCondition.alwaysTrue(), "com.sun.org.apache.xalan.internal.res.XSLTInfo"); classInitializationSupport.setConfigurationSealed(true); } diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java index 750d6e4d5cac..646f9bd974b3 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java @@ -229,8 +229,6 @@ public boolean getAsBoolean() { private final Set> registeredClasses = new HashSet<>(); private final Map, PossibleReplaceCandidatesSubtypeHandler> subtypeChecks = new HashMap<>(); private boolean profilingEnabled; - private boolean needsAllEncodings; - private Field uncachedDispatchField; private Field layoutInfoMapField; private Field layoutMapField; @@ -310,8 +308,12 @@ public void afterRegistration(AfterRegistrationAccess a) { initializeTruffleReflectively(imageClassLoader); initializeHomeFinder(); - needsAllEncodings = invokeStaticMethod("com.oracle.truffle.polyglot.LanguageCache", "getNeedsAllEncodings", + boolean needsAllEncodings = invokeStaticMethod("com.oracle.truffle.polyglot.LanguageCache", "getNeedsAllEncodings", Collections.emptyList()); + if (needsAllEncodings) { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), "org/graalvm/shadowed/org/jcodings/tables/.*bin$"); + } + // reinitialize language cache invokeStaticMethod("com.oracle.truffle.api.library.LibraryFactory", "reinitializeNativeImageState", Collections.emptyList()); @@ -448,10 +450,6 @@ public void duringSetup(DuringSetupAccess access) { if (Options.TruffleCheckPreinitializedFiles.getValue()) { access.registerObjectReplacer(new TruffleFileCheck(((FeatureImpl.DuringSetupAccessImpl) access).getHostVM().getClassInitializationSupport())); } - - if (needsAllEncodings) { - ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), "org/graalvm/shadowed/org/jcodings/tables/.*bin$"); - } } void setGraalGraphObjectReplacer(GraalGraphObjectReplacer graalGraphObjectReplacer) { @@ -1181,7 +1179,8 @@ final class Target_com_oracle_truffle_api_staticobject_PodBasedStaticShape { @TargetClass(className = "com.oracle.truffle.api.staticobject.ArrayBasedStaticShape", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_api_staticobject_ArrayBasedStaticShape { - @Alias @RecomputeFieldValue(kind = Kind.Custom, declClass = MapCleaner.class, isFinal = true) // + @Alias + @RecomputeFieldValue(kind = Kind.Custom, declClass = MapCleaner.class, isFinal = true) // static ConcurrentHashMap replacements; private static class MapCleaner implements FieldValueTransformerWithAvailability { @@ -1363,7 +1362,8 @@ final class Target_com_oracle_truffle_polyglot_LanguageCache { * verification in DisallowedImageHeapObjectFeature, so we also do the implicit reset using a * substitution. */ - @Alias @RecomputeFieldValue(kind = Kind.Reset) // + @Alias + @RecomputeFieldValue(kind = Kind.Reset) // private String languageHome; } @@ -1408,13 +1408,15 @@ final class Target_com_oracle_truffle_polyglot_InternalResourceCache_ResettableC @TargetClass(className = "com.oracle.truffle.object.CoreLocations$DynamicObjectFieldLocation", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_object_CoreLocations_DynamicObjectFieldLocation { - @Alias @RecomputeFieldValue(kind = Kind.AtomicFieldUpdaterOffset) // + @Alias + @RecomputeFieldValue(kind = Kind.AtomicFieldUpdaterOffset) // private long offset; } @TargetClass(className = "com.oracle.truffle.object.CoreLocations$DynamicLongFieldLocation", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_object_CoreLocations_DynamicLongFieldLocation { - @Alias @RecomputeFieldValue(kind = Kind.AtomicFieldUpdaterOffset) // + @Alias + @RecomputeFieldValue(kind = Kind.AtomicFieldUpdaterOffset) // private long offset; } @@ -1441,7 +1443,8 @@ final class Target_com_oracle_truffle_api_nodes_Node { @TargetClass(className = "com.oracle.truffle.api.nodes.NodeClassImpl", innerClass = "NodeFieldData", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_api_nodes_NodeClassImpl_NodeFieldData { - @Alias @RecomputeFieldValue(kind = Kind.Custom, declClass = OffsetComputer.class, isFinal = true) // + @Alias + @RecomputeFieldValue(kind = Kind.Custom, declClass = OffsetComputer.class, isFinal = true) // private long offset; private static class OffsetComputer implements FieldValueTransformerWithAvailability { @@ -1467,7 +1470,8 @@ public Object transform(Object receiver, Object originalValue) { @TargetClass(className = "com.oracle.truffle.api.dsl.InlineSupport$UnsafeField", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_api_dsl_InlineSupport_UnsafeField { - @Alias @RecomputeFieldValue(kind = Kind.Custom, declClass = OffsetComputer.class, isFinal = true) // + @Alias + @RecomputeFieldValue(kind = Kind.Custom, declClass = OffsetComputer.class, isFinal = true) // private long offset; /* From 1468ef1695809914b0184a4394b53be7bf8e11e2 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 31 May 2023 12:32:46 +0200 Subject: [PATCH 29/78] Replace resource Bundles registration with non-pattern-based implementation --- ...ContentSubstitutedLocalizationSupport.java | 5 ++-- .../jdk/localization/LocalizationSupport.java | 25 ++++++++++++++----- .../OptimizedLocalizationSupport.java | 4 +-- .../jdk/localization/LocalizationFeature.java | 10 ++++---- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java index 9f918232531e..22bd8772fe5f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java @@ -67,8 +67,9 @@ public class BundleContentSubstitutedLocalizationSupport extends LocalizationSup private final Set existingBundles = ConcurrentHashMap.newKeySet(); - public BundleContentSubstitutedLocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset, List requestedPatterns, ForkJoinPool pool) { - super(defaultLocale, locales, defaultCharset); + public BundleContentSubstitutedLocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset, List requestedPatterns, ForkJoinPool pool, + ClassLoader appClassLoader) { + super(defaultLocale, locales, defaultCharset, appClassLoader); this.pool = pool; this.compressBundlesPatterns = parseCompressBundlePatterns(requestedPatterns); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index d6ab244df836..60cb49e26eeb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -31,6 +31,7 @@ import java.util.IllformedLocaleException; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.PropertyResourceBundle; import java.util.ResourceBundle; import java.util.Set; @@ -69,11 +70,14 @@ public class LocalizationSupport { public final Charset defaultCharset; - public LocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset) { + private final ClassLoader appClassLoader; + + public LocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset, ClassLoader appClassLoader) { this.defaultLocale = defaultLocale; this.allLocales = locales.toArray(new Locale[0]); this.defaultCharset = defaultCharset; this.supportedLanguageTags = locales.stream().map(Locale::toString).collect(Collectors.toSet()); + this.appClassLoader = appClassLoader; } public boolean optimizedMode() { @@ -101,14 +105,23 @@ public Map getBundleContentOf(Object bundle) { public void prepareBundle(String bundleName, ResourceBundle bundle, Locale locale) { if (bundle instanceof PropertyResourceBundle) { String[] bundleNameWithModule = SubstrateUtil.split(bundleName, ":", 2); - String resultingPattern; + String resourceName; if (bundleNameWithModule.length < 2) { - resultingPattern = control.toBundleName(bundleName, locale).replace('.', '/'); + resourceName = control.toBundleName(bundleName, locale).replace('.', '/'); + if (appClassLoader.getResource(resourceName) != null) { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(appClassLoader.getUnnamedModule(), resourceName); + } else { + for (Module m : ModuleLayer.boot().modules()) { + if (m.getClassLoader().getResource(resourceName) != null) { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName); + } + } + } } else { - String patternWithLocale = control.toBundleName(bundleNameWithModule[1], locale).replace('.', '/'); - resultingPattern = bundleNameWithModule[0] + ':' + patternWithLocale; + resourceName = control.toBundleName(bundleNameWithModule[1], locale).replace('.', '/'); + Optional module = ModuleLayer.boot().findModule(bundleNameWithModule[0]); + module.ifPresent(m -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName)); } - ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), resultingPattern + "\\.properties"); } else { registerRequiredReflectionAndResourcesForBundle(bundleName, Set.of(locale)); RuntimeReflection.register(bundle.getClass()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java index 91187cdc9bd8..c00dabda73bd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java @@ -50,8 +50,8 @@ public class OptimizedLocalizationSupport extends LocalizationSupport { final Map, ResourceBundle> resourceBundles = new HashMap<>(); - public OptimizedLocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset) { - super(defaultLocale, locales, defaultCharset); + public OptimizedLocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset, ClassLoader appClassLoader) { + super(defaultLocale, locales, defaultCharset, appClassLoader); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java index 573ef4ddb6d9..3a8155d49986 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java @@ -264,7 +264,7 @@ public void afterRegistration(AfterRegistrationAccess access) { throw UserError.abort(ex, "Invalid default charset %s", defaultCharsetOptionValue); } allLocales.add(defaultLocale); - support = selectLocalizationSupport(); + support = selectLocalizationSupport(access.getApplicationClassLoader()); ImageSingletons.add(LocalizationSupport.class, support); addCharsets(); @@ -322,14 +322,14 @@ private Object eagerlyInitializeBundles(Object object) { } @Platforms(Platform.HOSTED_ONLY.class) - private LocalizationSupport selectLocalizationSupport() { + private LocalizationSupport selectLocalizationSupport(ClassLoader appClassLoader) { if (optimizedMode) { - return new OptimizedLocalizationSupport(defaultLocale, allLocales, defaultCharset); + return new OptimizedLocalizationSupport(defaultLocale, allLocales, defaultCharset, appClassLoader); } else if (substituteLoadLookup) { List requestedPatterns = Options.LocalizationCompressBundles.getValue().values(); - return new BundleContentSubstitutedLocalizationSupport(defaultLocale, allLocales, defaultCharset, requestedPatterns, compressionPool); + return new BundleContentSubstitutedLocalizationSupport(defaultLocale, allLocales, defaultCharset, requestedPatterns, compressionPool, appClassLoader); } - return new LocalizationSupport(defaultLocale, allLocales, defaultCharset); + return new LocalizationSupport(defaultLocale, allLocales, defaultCharset, appClassLoader); } @Override From 55c9af1521df817f9fb88ec37fb8cbaefffc7931 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 31 May 2023 14:37:34 +0200 Subject: [PATCH 30/78] Add null pointer check for module classLoader --- .../oracle/svm/core/jdk/localization/LocalizationSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index 60cb49e26eeb..6d20ccfaf51b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -112,7 +112,7 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Locale local ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(appClassLoader.getUnnamedModule(), resourceName); } else { for (Module m : ModuleLayer.boot().modules()) { - if (m.getClassLoader().getResource(resourceName) != null) { + if (m.getClassLoader() != null && m.getClassLoader().getResource(resourceName) != null) { ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName); } } From 5d536128c916c089a66e44aa2651274dca879815 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 1 Jun 2023 11:44:08 +0200 Subject: [PATCH 31/78] Add .properties extension to resourceBundle name --- .../oracle/svm/core/jdk/localization/LocalizationSupport.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index 6d20ccfaf51b..2c366c52f7be 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -107,7 +107,7 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Locale local String[] bundleNameWithModule = SubstrateUtil.split(bundleName, ":", 2); String resourceName; if (bundleNameWithModule.length < 2) { - resourceName = control.toBundleName(bundleName, locale).replace('.', '/'); + resourceName = control.toBundleName(bundleName, locale).replace('.', '/').concat(".properties"); if (appClassLoader.getResource(resourceName) != null) { ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(appClassLoader.getUnnamedModule(), resourceName); } else { @@ -118,7 +118,7 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Locale local } } } else { - resourceName = control.toBundleName(bundleNameWithModule[1], locale).replace('.', '/'); + resourceName = control.toBundleName(bundleNameWithModule[1], locale).replace('.', '/').concat(".properties"); Optional module = ModuleLayer.boot().findModule(bundleNameWithModule[0]); module.ifPresent(m -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName)); } From 5633755150af98043bef501e31ae77670b4f8465 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 8 Jun 2023 16:52:18 +0200 Subject: [PATCH 32/78] Add module lookup map --- .../oracle/svm/core/ClassLoaderSupport.java | 4 ++ ...ContentSubstitutedLocalizationSupport.java | 5 +-- .../jdk/localization/LocalizationSupport.java | 44 +++++++++++-------- .../OptimizedLocalizationSupport.java | 10 +++-- .../svm/hosted/ClassLoaderSupportImpl.java | 5 +++ .../oracle/svm/hosted/ResourcesFeature.java | 2 +- .../jdk/localization/LocalizationFeature.java | 22 +++++++--- 7 files changed, 59 insertions(+), 33 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java index 4c7e0aaca4ad..4b01c0d63c48 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java @@ -29,7 +29,9 @@ import java.net.URI; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.ResourceBundle; +import java.util.Set; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -66,4 +68,6 @@ public interface ResourceCollector { public abstract void collectResources(ResourceCollector resourceCollector); public abstract List getResourceBundle(String bundleName, Locale locale); + + public abstract Map> getPackageToModules(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java index 22bd8772fe5f..9f918232531e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java @@ -67,9 +67,8 @@ public class BundleContentSubstitutedLocalizationSupport extends LocalizationSup private final Set existingBundles = ConcurrentHashMap.newKeySet(); - public BundleContentSubstitutedLocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset, List requestedPatterns, ForkJoinPool pool, - ClassLoader appClassLoader) { - super(defaultLocale, locales, defaultCharset, appClassLoader); + public BundleContentSubstitutedLocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset, List requestedPatterns, ForkJoinPool pool) { + super(defaultLocale, locales, defaultCharset); this.pool = pool; this.compressBundlesPatterns = parseCompressBundlePatterns(requestedPatterns); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index 2c366c52f7be..5cfad84c4e4e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -35,6 +35,7 @@ import java.util.PropertyResourceBundle; import java.util.ResourceBundle; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import jdk.graal.compiler.debug.GraalError; @@ -46,6 +47,7 @@ import org.graalvm.nativeimage.impl.RuntimeReflectionSupport; import org.graalvm.nativeimage.impl.RuntimeResourceSupport; +import com.oracle.svm.core.ClassLoaderSupport; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.jdk.Resources; import com.oracle.svm.core.util.VMError; @@ -70,14 +72,11 @@ public class LocalizationSupport { public final Charset defaultCharset; - private final ClassLoader appClassLoader; - - public LocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset, ClassLoader appClassLoader) { + public LocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset) { this.defaultLocale = defaultLocale; this.allLocales = locales.toArray(new Locale[0]); this.defaultCharset = defaultCharset; this.supportedLanguageTags = locales.stream().map(Locale::toString).collect(Collectors.toSet()); - this.appClassLoader = appClassLoader; } public boolean optimizedMode() { @@ -102,25 +101,26 @@ public Map getBundleContentOf(Object bundle) { } @Platforms(Platform.HOSTED_ONLY.class) - public void prepareBundle(String bundleName, ResourceBundle bundle, Locale locale) { - if (bundle instanceof PropertyResourceBundle) { + public void prepareBundle(String bundleName, ResourceBundle bundle, Locale locale, Function> findModule) { + if (bundle instanceof PropertyResourceBundle prb) { String[] bundleNameWithModule = SubstrateUtil.split(bundleName, ":", 2); String resourceName; if (bundleNameWithModule.length < 2) { - resourceName = control.toBundleName(bundleName, locale).replace('.', '/').concat(".properties"); - if (appClassLoader.getResource(resourceName) != null) { - ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(appClassLoader.getUnnamedModule(), resourceName); - } else { - for (Module m : ModuleLayer.boot().modules()) { - if (m.getClassLoader() != null && m.getClassLoader().getResource(resourceName) != null) { - ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName); - } - } + // find module based on package name + Map> packageToModules = ImageSingletons.lookup(ClassLoaderSupport.class).getPackageToModules(); + Set modules = packageToModules.getOrDefault(packageName(bundleName), Collections.emptySet()); + + // there should be only one module but we will check all modules where given package + // is found + for (Module m : modules) { + resourceName = control.toBundleName(bundleName, prb.getLocale()).replace('.', '/').concat(".properties"); + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName); } } else { - resourceName = control.toBundleName(bundleNameWithModule[1], locale).replace('.', '/').concat(".properties"); - Optional module = ModuleLayer.boot().findModule(bundleNameWithModule[0]); - module.ifPresent(m -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName)); + resourceName = control.toBundleName(bundleNameWithModule[1], prb.getLocale()).replace('.', '/').concat(".properties"); + Optional module = findModule.apply(bundleNameWithModule[0]); + String finalResourceName = resourceName; + module.ifPresent(m -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, finalResourceName)); } } else { registerRequiredReflectionAndResourcesForBundle(bundleName, Set.of(locale)); @@ -130,6 +130,14 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Locale local } } + private static String packageName(String bundleName) { + int classSep = bundleName.lastIndexOf('.'); + if (classSep == -1) { + return ""; /* unnamed package */ + } + return bundleName.substring(0, classSep); + } + public String getResultingPattern(String bundleName, Locale locale) { String fixedBundleName = bundleName.replace("$", "\\$"); return getBundleName(fixedBundleName, locale); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java index c00dabda73bd..80c15a240b94 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java @@ -30,8 +30,10 @@ import java.util.Locale; import java.util.Map; import java.util.MissingResourceException; +import java.util.Optional; import java.util.ResourceBundle; import java.util.Set; +import java.util.function.Function; import java.util.spi.LocaleServiceProvider; import org.graalvm.collections.Pair; @@ -50,8 +52,8 @@ public class OptimizedLocalizationSupport extends LocalizationSupport { final Map, ResourceBundle> resourceBundles = new HashMap<>(); - public OptimizedLocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset, ClassLoader appClassLoader) { - super(defaultLocale, locales, defaultCharset, appClassLoader); + public OptimizedLocalizationSupport(Locale defaultLocale, Set locales, Charset defaultCharset) { + super(defaultLocale, locales, defaultCharset); } @Override @@ -89,7 +91,7 @@ public void prepareClassResourceBundle(String basename, Class bundleClass) { /*- Set the basename and locale to be consistent with JVM lookup process */ bundleNameField.set(bundle, basename); bundleLocaleField.set(bundle, locale); - prepareBundle(basename, bundle, locale); + prepareBundle(basename, bundle, locale, null); } catch (ReflectionUtil.ReflectionUtilError | ReflectiveOperationException e) { throw UserError.abort(e, "Failed to instantiated bundle from class %s, reason %s", bundleClass, e.getCause().getMessage()); } @@ -106,7 +108,7 @@ private static Locale extractLocale(Class bundleClass) { @Platforms(Platform.HOSTED_ONLY.class) @Override - public void prepareBundle(String bundleName, ResourceBundle bundle, Locale locale) { + public void prepareBundle(String bundleName, ResourceBundle bundle, Locale locale, Function> findModule) { bundle.keySet(); this.resourceBundles.put(Pair.create(bundleName, locale), bundle); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 1452c978ba52..a0adb54b7d19 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -283,6 +283,11 @@ public List getResourceBundle(String bundleSpec, Locale locale) return resourceBundles; } + @Override + public Map> getPackageToModules() { + return packageToModules; + } + private static String packageName(String bundleName) { int classSep = bundleName.lastIndexOf('.'); if (classSep == -1) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index dc06d85b6fe3..143219dd9c83 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -221,7 +221,7 @@ public void addResource(Module module, String resourcePath) { boolean fromJar = false; boolean isDirectory = false; - if (module.isNamed()) { + if (module != null && module.isNamed()) { try { String resourcePackage = jdk.internal.module.Resources.toPackageName(resourcePath); if (!resourcePackage.isEmpty()) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java index 3a8155d49986..eb6594c83866 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java @@ -89,6 +89,7 @@ import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.FeatureImpl.DuringAnalysisAccessImpl; import com.oracle.svm.hosted.FeatureImpl.DuringSetupAccessImpl; +import com.oracle.svm.hosted.ImageClassLoader; import jdk.internal.access.SharedSecrets; import jdk.vm.ci.meta.ResolvedJavaField; @@ -162,6 +163,10 @@ public class LocalizationFeature implements InternalFeature { private Field langAliasesCacheField; private Field parentLocalesMapField; + @Platforms(Platform.HOSTED_ONLY.class) private ClassLoader classLoader; + + @Platforms(Platform.HOSTED_ONLY.class) private ImageClassLoader imageClassLoader; + public static class Options { @Option(help = "Comma separated list of bundles to be included into the image.", type = OptionType.User)// public static final HostedOptionKey IncludeResourceBundles = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @@ -264,7 +269,7 @@ public void afterRegistration(AfterRegistrationAccess access) { throw UserError.abort(ex, "Invalid default charset %s", defaultCharsetOptionValue); } allLocales.add(defaultLocale); - support = selectLocalizationSupport(access.getApplicationClassLoader()); + support = selectLocalizationSupport(); ImageSingletons.add(LocalizationSupport.class, support); addCharsets(); @@ -292,6 +297,9 @@ public void duringSetup(DuringSetupAccess a) { String reason = "All ResourceBundleControlProvider that are registered as services end up as objects in the image heap, and are therefore registered to be initialized at image build time"; ServiceLoader.load(ResourceBundleControlProvider.class).stream() .forEach(provider -> ImageSingletons.lookup(RuntimeClassInitializationSupport.class).initializeAtBuildTime(provider.type(), reason)); + + this.classLoader = access.getApplicationClassLoader(); + this.imageClassLoader = access.getImageClassLoader(); } /** @@ -322,14 +330,14 @@ private Object eagerlyInitializeBundles(Object object) { } @Platforms(Platform.HOSTED_ONLY.class) - private LocalizationSupport selectLocalizationSupport(ClassLoader appClassLoader) { + private LocalizationSupport selectLocalizationSupport() { if (optimizedMode) { - return new OptimizedLocalizationSupport(defaultLocale, allLocales, defaultCharset, appClassLoader); + return new OptimizedLocalizationSupport(defaultLocale, allLocales, defaultCharset); } else if (substituteLoadLookup) { List requestedPatterns = Options.LocalizationCompressBundles.getValue().values(); - return new BundleContentSubstitutedLocalizationSupport(defaultLocale, allLocales, defaultCharset, requestedPatterns, compressionPool, appClassLoader); + return new BundleContentSubstitutedLocalizationSupport(defaultLocale, allLocales, defaultCharset, requestedPatterns, compressionPool); } - return new LocalizationSupport(defaultLocale, allLocales, defaultCharset, appClassLoader); + return new LocalizationSupport(defaultLocale, allLocales, defaultCharset); } @Override @@ -639,14 +647,14 @@ private void prepareBundle(String bundleName, ResourceBundle bundle, Locale loca */ for (ResourceBundle cur = bundle; cur != null; cur = SharedSecrets.getJavaUtilResourceBundleAccess().getParent(cur)) { /* Register all bundles with their corresponding locales */ - support.prepareBundle(bundleName, cur, cur.getLocale()); + support.prepareBundle(bundleName, cur, locale, this.imageClassLoader::findModule); } /* * Finally, register the requested bundle with requested locale (Requested might be more * specific than the actual bundle locale */ - support.prepareBundle(bundleName, bundle, locale); + support.prepareBundle(bundleName, bundle, locale, this.imageClassLoader::findModule); } @Platforms(Platform.HOSTED_ONLY.class) From fa166d80ba35a5c5379f8b6807a3c36308813769 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 9 Jun 2023 14:20:51 +0200 Subject: [PATCH 33/78] Add unnamed module case --- .../hosted/RuntimeResourceAccess.java | 2 +- .../impl/RuntimeResourceSupport.java | 2 +- .../jdk/localization/LocalizationSupport.java | 10 ++++- .../OptimizedLocalizationSupport.java | 6 +-- .../oracle/svm/hosted/ResourcesFeature.java | 40 ------------------- .../jdk/localization/LocalizationFeature.java | 4 +- 6 files changed, 15 insertions(+), 49 deletions(-) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java index 76c43f86efd2..1e26a651c290 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeResourceAccess.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java index 2d370a511ffd..6147690f249c 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeResourceSupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index 5cfad84c4e4e..d8c54dc823d5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -101,11 +101,13 @@ public Map getBundleContentOf(Object bundle) { } @Platforms(Platform.HOSTED_ONLY.class) - public void prepareBundle(String bundleName, ResourceBundle bundle, Locale locale, Function> findModule) { + public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule) { if (bundle instanceof PropertyResourceBundle prb) { String[] bundleNameWithModule = SubstrateUtil.split(bundleName, ":", 2); String resourceName; if (bundleNameWithModule.length < 2) { + resourceName = control.toBundleName(bundleName, prb.getLocale()).replace('.', '/').concat(".properties"); + // find module based on package name Map> packageToModules = ImageSingletons.lookup(ClassLoaderSupport.class).getPackageToModules(); Set modules = packageToModules.getOrDefault(packageName(bundleName), Collections.emptySet()); @@ -113,9 +115,13 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Locale local // there should be only one module but we will check all modules where given package // is found for (Module m : modules) { - resourceName = control.toBundleName(bundleName, prb.getLocale()).replace('.', '/').concat(".properties"); ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName); } + + // if didn't find resource in any module, we will try with unnamed module + if (modules.isEmpty()) { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(null, resourceName); + } } else { resourceName = control.toBundleName(bundleNameWithModule[1], prb.getLocale()).replace('.', '/').concat(".properties"); Optional module = findModule.apply(bundleNameWithModule[0]); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java index 80c15a240b94..30afc879b2af 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java @@ -91,7 +91,7 @@ public void prepareClassResourceBundle(String basename, Class bundleClass) { /*- Set the basename and locale to be consistent with JVM lookup process */ bundleNameField.set(bundle, basename); bundleLocaleField.set(bundle, locale); - prepareBundle(basename, bundle, locale, null); + prepareBundle(basename, bundle, null); } catch (ReflectionUtil.ReflectionUtilError | ReflectiveOperationException e) { throw UserError.abort(e, "Failed to instantiated bundle from class %s, reason %s", bundleClass, e.getCause().getMessage()); } @@ -108,9 +108,9 @@ private static Locale extractLocale(Class bundleClass) { @Platforms(Platform.HOSTED_ONLY.class) @Override - public void prepareBundle(String bundleName, ResourceBundle bundle, Locale locale, Function> findModule) { + public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule) { bundle.keySet(); - this.resourceBundles.put(Pair.create(bundleName, locale), bundle); + this.resourceBundles.put(Pair.create(bundleName, bundle.getLocale()), bundle); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 143219dd9c83..8007464845ba 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -68,46 +68,6 @@ import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.graalvm.nativeimage.impl.RuntimeResourceSupport; -import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; -import org.graalvm.compiler.debug.DebugContext; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; -import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; -import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; -import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; -import org.graalvm.compiler.options.Option; -import org.graalvm.compiler.options.OptionType; -import org.graalvm.compiler.phases.util.Providers; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; -import org.graalvm.nativeimage.impl.ConfigurationCondition; -import org.graalvm.nativeimage.impl.RuntimeResourceSupport; - -import com.oracle.svm.core.ClassLoaderSupport; -import com.oracle.svm.core.ClassLoaderSupport.ResourceCollector; -import com.oracle.svm.core.ParsingReason; -import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.configure.ConfigurationFile; -import com.oracle.svm.core.configure.ConfigurationFiles; -import com.oracle.svm.core.configure.ResourceConfigurationParser; -import com.oracle.svm.core.configure.ResourcesRegistry; -import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; -import com.oracle.svm.core.feature.InternalFeature; -import com.oracle.svm.core.jdk.Resources; -import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributes; -import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributesView; -import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystem; -import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystemProvider; -import com.oracle.svm.core.option.HostedOptionKey; -import com.oracle.svm.core.option.LocatableMultiOptionValue; -import com.oracle.svm.core.util.UserError; -import com.oracle.svm.core.util.VMError; -import com.oracle.svm.hosted.config.ConfigurationParserUtils; -import com.oracle.svm.hosted.jdk.localization.LocalizationFeature; -import com.oracle.svm.util.ModuleSupport; -import com.oracle.svm.util.ReflectionUtil; - -import jdk.vm.ci.meta.ResolvedJavaMethod; import com.oracle.svm.core.ClassLoaderSupport; import com.oracle.svm.core.ClassLoaderSupport.ResourceCollector; import com.oracle.svm.core.MissingRegistrationUtils; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java index eb6594c83866..156a2e4aabd7 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java @@ -647,14 +647,14 @@ private void prepareBundle(String bundleName, ResourceBundle bundle, Locale loca */ for (ResourceBundle cur = bundle; cur != null; cur = SharedSecrets.getJavaUtilResourceBundleAccess().getParent(cur)) { /* Register all bundles with their corresponding locales */ - support.prepareBundle(bundleName, cur, locale, this.imageClassLoader::findModule); + support.prepareBundle(bundleName, cur, this.imageClassLoader::findModule); } /* * Finally, register the requested bundle with requested locale (Requested might be more * specific than the actual bundle locale */ - support.prepareBundle(bundleName, bundle, locale, this.imageClassLoader::findModule); + support.prepareBundle(bundleName, bundle, this.imageClassLoader::findModule); } @Platforms(Platform.HOSTED_ONLY.class) From 2b762d9fb7eb189708c65b957f56715ae96cdae9 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 16 Jun 2023 10:32:23 +0200 Subject: [PATCH 34/78] Move resource adding to duringSetup when the RuntimeResourceSupport is added to singletones --- .../com/oracle/svm/truffle/TruffleBaseFeature.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java index 646f9bd974b3..8b6559f55062 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java @@ -239,6 +239,8 @@ public boolean getAsBoolean() { private Consumer markAsUnsafeAccessed; private final ConcurrentMap processedInlinedFields = new ConcurrentHashMap<>(); + private boolean needsAllEncodings; + private static void initializeTruffleReflectively(ClassLoader imageClassLoader) { invokeStaticMethod("com.oracle.truffle.api.impl.Accessor", "getTVMCI", Collections.emptyList()); invokeStaticMethod("com.oracle.truffle.polyglot.InternalResourceCache", "initializeNativeImageState", @@ -308,11 +310,8 @@ public void afterRegistration(AfterRegistrationAccess a) { initializeTruffleReflectively(imageClassLoader); initializeHomeFinder(); - boolean needsAllEncodings = invokeStaticMethod("com.oracle.truffle.polyglot.LanguageCache", "getNeedsAllEncodings", + needsAllEncodings = invokeStaticMethod("com.oracle.truffle.polyglot.LanguageCache", "getNeedsAllEncodings", Collections.emptyList()); - if (needsAllEncodings) { - ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), "org/graalvm/shadowed/org/jcodings/tables/.*bin$"); - } // reinitialize language cache invokeStaticMethod("com.oracle.truffle.api.library.LibraryFactory", "reinitializeNativeImageState", @@ -450,6 +449,10 @@ public void duringSetup(DuringSetupAccess access) { if (Options.TruffleCheckPreinitializedFiles.getValue()) { access.registerObjectReplacer(new TruffleFileCheck(((FeatureImpl.DuringSetupAccessImpl) access).getHostVM().getClassInitializationSupport())); } + + if (needsAllEncodings) { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResources(ConfigurationCondition.alwaysTrue(), "org/graalvm/shadowed/org/jcodings/tables/.*bin$"); + } } void setGraalGraphObjectReplacer(GraalGraphObjectReplacer graalGraphObjectReplacer) { From b4ca2f275c0d4fd4adbeaf5b933f8ec9f34a3d27 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 16 Jun 2023 16:08:21 +0200 Subject: [PATCH 35/78] Try to match resource pattern --- .../svm/hosted/ClassLoaderSupportImpl.java | 2 +- .../oracle/svm/hosted/ResourcesFeature.java | 60 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index a0adb54b7d19..4b1fd4dd5744 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -89,7 +89,7 @@ protected boolean isNativeImageClassLoaderImpl(ClassLoader loader) { return false; } - private record ResourceLookupInfo(ResolvedModule resolvedModule, Module module) { + public record ResourceLookupInfo(ResolvedModule resolvedModule, Module module) { } private static Stream extractModuleLookupData(ModuleLayer layer) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 8007464845ba..46ee4072a032 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -29,6 +29,9 @@ import java.io.IOException; import java.io.InputStream; +import java.lang.module.ModuleReader; +import java.lang.module.ModuleReference; +import java.lang.module.ResolvedModule; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -42,6 +45,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Set; @@ -52,6 +56,7 @@ import java.util.concurrent.atomic.LongAdder; import java.util.jar.JarFile; import java.util.regex.Pattern; +import java.util.stream.Stream; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.debug.DebugContext; @@ -143,6 +148,8 @@ public static class Options { private int loadedConfigurations; private ImageClassLoader imageClassLoader; + private HashMap> registerIfReachable = new HashMap<>(); + private class ResourcesRegistryImpl extends ConditionalConfigurationRegistry implements ResourcesRegistry { private final ConfigurationTypeResolver configurationTypeResolver; @@ -150,11 +157,64 @@ private class ResourcesRegistryImpl extends ConditionalConfigurationRegistry imp this.configurationTypeResolver = configurationTypeResolver; } + private static Stream extractModuleLookupData(ModuleLayer layer) { + List data = new ArrayList<>(layer.configuration().modules().size()); + for (ResolvedModule m : layer.configuration().modules()) { + Module module = layer.findModule(m.name()).orElse(null); + ClassLoaderSupportImpl.ResourceLookupInfo info = new ClassLoaderSupportImpl.ResourceLookupInfo(m, module); + data.add(info); + } + return data.stream(); + } + + private List findReachableForPattern(String pattern) { + List founded = new ArrayList<>(); + ResourcePattern compiledPattern = compilePatterns(Set.of(pattern))[0]; + NativeImageClassLoaderSupport.allLayers(imageClassLoader.classLoaderSupport.moduleLayerForImageBuild).stream() + .parallel() + .flatMap(ResourcesRegistryImpl::extractModuleLookupData) + .forEach(info -> { + ModuleReference moduleReference = info.resolvedModule().reference(); + try (ModuleReader moduleReader = moduleReference.open()) { + List foundResources = moduleReader.list() + .filter(resourceName -> { + System.out.println("Module files: " + resourceName); + return compiledPattern.pattern.matcher(resourceName).matches(); + }) + .toList(); + founded.addAll(foundResources); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + for (Path classpathFile : imageClassLoader.classLoaderSupport.classpath()) { + System.out.println("Classpath file: " + classpathFile); + if (compiledPattern.pattern.matcher(classpathFile.toString()).matches()) { + founded.add(classpathFile.toString()); + } + } + + System.out.println(founded); + return founded; + } + @Override public void addResources(ConfigurationCondition condition, String pattern) { if (configurationTypeResolver.resolveConditionType(condition.getTypeName()) == null) { return; } + + if (!ConfigurationCondition.alwaysTrue().equals(condition)) { + registerIfReachable.put(condition.getTypeName(), findReachableForPattern(pattern)); + } + + try { + throw new Exception("a"); + } catch (Exception e) { + e.printStackTrace(); + } + registerConditionalConfiguration(condition, () -> { UserError.guarantee(!sealed, "Resources added too late: %s", pattern); resourcePatternWorkSet.add(pattern); From 8a7f024cc24dd1b16df58a348160815476d05e48 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 20 Jun 2023 17:27:19 +0200 Subject: [PATCH 36/78] Cover the case when resource has condition --- .../impl/ConfigurationCondition.java | 6 +- .../config/ResourceConfiguration.java | 21 ++-- .../oracle/svm/core/ClassLoaderSupport.java | 9 +- .../com/oracle/svm/core/jdk/Resources.java | 1 + .../jdk/localization/LocalizationSupport.java | 5 +- .../svm/hosted/ClassLoaderSupportImpl.java | 104 ++++++++++++----- .../oracle/svm/hosted/ResourcesFeature.java | 108 ++++++------------ .../CharsetSubstitutionsFeature.java | 6 +- .../jdk/localization/LocalizationFeature.java | 4 - .../svm/truffle/TruffleBaseFeature.java | 18 +-- 10 files changed, 147 insertions(+), 135 deletions(-) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/ConfigurationCondition.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/ConfigurationCondition.java index 72a76b2d2718..e089491291de 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/ConfigurationCondition.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/ConfigurationCondition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -50,6 +50,10 @@ public static ConfigurationCondition alwaysTrue() { return OBJECT_REACHABLE; } + public static boolean isAlwaysTrue(ConfigurationCondition condition) { + return ConfigurationCondition.alwaysTrue().equals(condition); + } + public static ConfigurationCondition create(String typeReachability) { Objects.requireNonNull(typeReachability); if (OBJECT_REACHABLE.typeName.equals(typeReachability)) { diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java index c7dcb46b78e1..18cfeded7762 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java @@ -24,16 +24,6 @@ */ package com.oracle.svm.configure.config; -import com.oracle.svm.configure.ConfigurationBase; -import com.oracle.svm.core.configure.ConditionalElement; -import com.oracle.svm.core.configure.ConfigurationParser; -import com.oracle.svm.core.configure.ResourceConfigurationParser; -import com.oracle.svm.core.configure.ResourcesRegistry; -import com.oracle.svm.core.util.VMError; -import com.oracle.svm.core.util.json.JsonPrinter; -import com.oracle.svm.core.util.json.JsonWriter; -import org.graalvm.nativeimage.impl.ConfigurationCondition; - import java.io.IOException; import java.util.Collection; import java.util.Comparator; @@ -45,6 +35,17 @@ import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; +import org.graalvm.nativeimage.impl.ConfigurationCondition; + +import com.oracle.svm.configure.ConfigurationBase; +import com.oracle.svm.core.configure.ConditionalElement; +import com.oracle.svm.core.configure.ConfigurationParser; +import com.oracle.svm.core.configure.ResourceConfigurationParser; +import com.oracle.svm.core.configure.ResourcesRegistry; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.core.util.json.JsonPrinter; +import com.oracle.svm.core.util.json.JsonWriter; + public final class ResourceConfiguration extends ConfigurationBase { private static final String PROPERTY_BUNDLE = "java.util.PropertyResourceBundle"; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java index 4b01c0d63c48..2b5fd7d4f431 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java @@ -35,6 +35,7 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.impl.ConfigurationCondition; @Platforms(Platform.HOSTED_ONLY.class) public abstract class ClassLoaderSupport { @@ -53,16 +54,20 @@ public boolean isNativeImageClassLoader(ClassLoader classLoader) { protected abstract boolean isNativeImageClassLoaderImpl(ClassLoader classLoader); public interface ResourceCollector { - - boolean isIncluded(Module module, String resourceName, URI resourceURI); + ConfigurationCondition isIncluded(Module module, String resourceName, URI resourceURI); void addResource(Module module, String resourceName, InputStream resourceStream, boolean fromJar); + void addResourceConditionally(Module module, String resourceName, ConfigurationCondition condition); + void addDirectoryResource(Module module, String dir, String content, boolean fromJar); void registerNegativeQuery(Module module, String resourceName); void registerIOException(Module module, String resourceName, IOException e, boolean linkAtBuildTime); + + void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition); + } public abstract void collectResources(ResourceCollector resourceCollector); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 1240eebf95d0..85e2c519860b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -218,6 +218,7 @@ public void registerResource(Module module, String resourceName, InputStream is, public static void registerResource(Module module, String resourceName, boolean isDir, InputStream is, boolean fromJar) { addEntry(module, resourceName, isDir, inputStreamToByteArray(is), fromJar); } + @Platforms(Platform.HOSTED_ONLY.class) public void registerDirectoryResource(String resourceDirName, String content) { registerDirectoryResource(null, resourceDirName, content, true); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index d8c54dc823d5..e74248dc1ce1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -137,11 +137,12 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Function foundResources = moduleReader.list() - .filter(resourceName -> shouldIncludeEntry(info.module(), resourceCollector, resourceName, moduleReference.location().orElse(null), includeAll)) - .toList(); + List> resourcesFound = new ArrayList<>(); + moduleReader.list().forEach(resourceName -> { + ConfigurationCondition condition = shouldIncludeEntry(info.module(), resourceCollector, resourceName, moduleReference.location().orElse(null), includeAll); + if (condition != null) { + resourcesFound.add(Pair.create(condition, resourceName)); + } + }); - for (String resName : foundResources) { + for (Pair entry : resourcesFound) { + ConfigurationCondition condition = entry.getLeft(); + String resName = entry.getRight(); if (resName.endsWith("/")) { - resourceCollector.addDirectoryResource(info.module, resName, "", false); + if (ConfigurationCondition.isAlwaysTrue(condition)) { + resourceCollector.addDirectoryResource(info.module, resName, "", false); + } else { + resourceCollector.addResourceConditionally(info.module, resName, condition); + } continue; } Optional content = moduleReader.open(resName); @@ -145,19 +158,24 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso continue; } try (InputStream is = content.get()) { - resourceCollector.addResource(info.module, resName, is, false); + if (ConfigurationCondition.isAlwaysTrue(condition)) { + resourceCollector.addResource(info.module, resName, is, false); + } else { + resourceCollector.addResourceConditionally(info.module, resName, condition); + } } catch (IOException resourceException) { resourceCollector.registerIOException(info.module, resName, resourceException, LinkAtBuildTimeSupport.singleton().moduleLinkAtBuildTime(info.module.getName())); } } + } catch (IOException e) { throw VMError.shouldNotReachHere(e); } } - private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) { - Map> matchedDirectoryResources = new HashMap<>(); - Set allEntries = new HashSet<>(); + private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) throws IOException { + Map, List> matchedDirectoryResources = new HashMap<>(); + Set> allEntries = new HashSet<>(); ArrayDeque queue = new ArrayDeque<>(); queue.push(root); @@ -166,16 +184,19 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea /* Resources always use / as the separator, as do our resource inclusion patterns */ String relativeFilePath; + ConfigurationCondition condition; if (entry != root) { relativeFilePath = root.relativize(entry).toString().replace(File.separatorChar, RESOURCES_INTERNAL_PATH_SEPARATOR); - allEntries.add(relativeFilePath); + condition = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); + allEntries.add(Pair.create(condition, relativeFilePath)); } else { relativeFilePath = ""; + condition = collector.isIncluded(null, relativeFilePath, Path.of(relativeFilePath).toUri()); } if (Files.isDirectory(entry)) { - if (shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll)) { - matchedDirectoryResources.put(relativeFilePath, new ArrayList<>()); + if (condition != null) { + matchedDirectoryResources.put(Pair.create(condition, relativeFilePath), new ArrayList<>()); } try (Stream pathStream = Files.list(entry)) { Stream filtered = pathStream; @@ -187,8 +208,13 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea collector.registerIOException(null, relativeFilePath, resourceException, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(relativeFilePath)); } } else { - if (shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll)) { + if (condition != null) { try (InputStream is = Files.newInputStream(entry)) { + if (ConfigurationCondition.isAlwaysTrue(condition)) { + collector.addResource(null, relativeFilePath, is, false); + } else { + collector.addResourceConditionally(null, relativeFilePath, condition); + } collector.addResource(null, relativeFilePath, is, false); } catch (IOException resourceException) { collector.registerIOException(null, relativeFilePath, resourceException, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(relativeFilePath)); @@ -197,20 +223,28 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea } } - for (String entry : allEntries) { - int last = entry.lastIndexOf(RESOURCES_INTERNAL_PATH_SEPARATOR); - String key = last == -1 ? "" : entry.substring(0, last); - List dirContent = matchedDirectoryResources.get(key); - if (dirContent != null && !dirContent.contains(entry)) { - dirContent.add(entry.substring(last + 1)); + for (Pair entry : allEntries) { + ConfigurationCondition condition = entry.getLeft(); + String entryName = entry.getRight(); + int last = entryName.lastIndexOf(RESOURCES_INTERNAL_PATH_SEPARATOR); + String key = last == -1 ? "" : entryName.substring(0, last); + List dirContent = matchedDirectoryResources.get(Pair.create(condition, key)); + if (dirContent != null && !dirContent.contains(entryName)) { + dirContent.add(entryName.substring(last + 1)); } else if (dirContent == null) { collector.registerNegativeQuery(null, key); } } - matchedDirectoryResources.forEach((dir, content) -> { + matchedDirectoryResources.forEach((entry, content) -> { + ConfigurationCondition condition = entry.getLeft(); + String dir = entry.getRight(); content.sort(Comparator.naturalOrder()); - collector.addDirectoryResource(null, dir, String.join(System.lineSeparator(), content), false); + if (ConfigurationCondition.isAlwaysTrue(condition)) { + collector.addDirectoryResource(null, dir, String.join(System.lineSeparator(), content), false); + } else { + collector.addDirectoryResourceConditionally(null, dir, condition); + } }); } @@ -222,14 +256,24 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i URI uri = jarPath.toUri(); if (entry.isDirectory()) { String dirName = entry.getName().substring(0, entry.getName().length() - 1); - if (shouldIncludeEntry(null, collector, dirName, uri, includeAll)) { + ConfigurationCondition condition = shouldIncludeEntry(null, collector, dirName, jarPath.toUri(), includeAll); + if (condition != null) { // Register the directory with empty content to preserve Java behavior - collector.addDirectoryResource(null, dirName, "", true); + if (ConfigurationCondition.isAlwaysTrue(condition)) { + collector.addDirectoryResource(null, dirName, "", true); + } else { + collector.addDirectoryResourceConditionally(null, dirName, condition); + } } } else { - if (shouldIncludeEntry(null, collector, entry.getName(), uri, includeAll)) { + ConfigurationCondition condition = shouldIncludeEntry(null, collector, entry.getName(), jarPath.toUri(), includeAll); + if (condition != null) { try (InputStream is = jf.getInputStream(entry)) { - collector.addResource(null, entry.getName(), is, true); + if (ConfigurationCondition.isAlwaysTrue(condition)) { + collector.addResource(null, entry.getName(), is, true); + } else { + collector.addResourceConditionally(null, entry.getName(), condition); + } } catch (IOException resourceException) { collector.registerIOException(null, entry.getName(), resourceException, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(entry.getName())); } @@ -239,9 +283,13 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i } } - private static boolean shouldIncludeEntry(Module module, ResourceCollector collector, String fileName, URI uri, boolean includeAll) { - var isIncluded = collector.isIncluded(module, fileName, uri); - return isIncluded || (includeAll && !(fileName.endsWith(".class") || fileName.endsWith(".jar"))); + private static ConfigurationCondition shouldIncludeEntry(Module module, ResourceCollector collector, String fileName, URI uri, boolean includeAll) { + ConfigurationCondition conditions = collector.isIncluded(module, fileName, uri); + if (includeAll && !(fileName.endsWith(".class") || fileName.endsWith(".jar"))) { + return ConfigurationCondition.alwaysTrue(); + } + + return conditions; } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 46ee4072a032..5fb1e7f66b40 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -29,9 +29,6 @@ import java.io.IOException; import java.io.InputStream; -import java.lang.module.ModuleReader; -import java.lang.module.ModuleReference; -import java.lang.module.ResolvedModule; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -45,7 +42,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Set; @@ -56,7 +52,7 @@ import java.util.concurrent.atomic.LongAdder; import java.util.jar.JarFile; import java.util.regex.Pattern; -import java.util.stream.Stream; +import java.util.stream.Collectors; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.debug.DebugContext; @@ -143,13 +139,12 @@ public static class Options { } private boolean sealed = false; - private Set resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + private Set> resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set excludedResourcePatterns = Collections.newSetFromMap(new ConcurrentHashMap<>()); private int loadedConfigurations; private ImageClassLoader imageClassLoader; - private HashMap> registerIfReachable = new HashMap<>(); - private class ResourcesRegistryImpl extends ConditionalConfigurationRegistry implements ResourcesRegistry { private final ConfigurationTypeResolver configurationTypeResolver; @@ -157,68 +152,17 @@ private class ResourcesRegistryImpl extends ConditionalConfigurationRegistry imp this.configurationTypeResolver = configurationTypeResolver; } - private static Stream extractModuleLookupData(ModuleLayer layer) { - List data = new ArrayList<>(layer.configuration().modules().size()); - for (ResolvedModule m : layer.configuration().modules()) { - Module module = layer.findModule(m.name()).orElse(null); - ClassLoaderSupportImpl.ResourceLookupInfo info = new ClassLoaderSupportImpl.ResourceLookupInfo(m, module); - data.add(info); - } - return data.stream(); - } - - private List findReachableForPattern(String pattern) { - List founded = new ArrayList<>(); - ResourcePattern compiledPattern = compilePatterns(Set.of(pattern))[0]; - NativeImageClassLoaderSupport.allLayers(imageClassLoader.classLoaderSupport.moduleLayerForImageBuild).stream() - .parallel() - .flatMap(ResourcesRegistryImpl::extractModuleLookupData) - .forEach(info -> { - ModuleReference moduleReference = info.resolvedModule().reference(); - try (ModuleReader moduleReader = moduleReference.open()) { - List foundResources = moduleReader.list() - .filter(resourceName -> { - System.out.println("Module files: " + resourceName); - return compiledPattern.pattern.matcher(resourceName).matches(); - }) - .toList(); - founded.addAll(foundResources); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - for (Path classpathFile : imageClassLoader.classLoaderSupport.classpath()) { - System.out.println("Classpath file: " + classpathFile); - if (compiledPattern.pattern.matcher(classpathFile.toString()).matches()) { - founded.add(classpathFile.toString()); - } - } - - System.out.println(founded); - return founded; - } - @Override public void addResources(ConfigurationCondition condition, String pattern) { if (configurationTypeResolver.resolveConditionType(condition.getTypeName()) == null) { return; } - if (!ConfigurationCondition.alwaysTrue().equals(condition)) { - registerIfReachable.put(condition.getTypeName(), findReachableForPattern(pattern)); - } - try { - throw new Exception("a"); - } catch (Exception e) { - e.printStackTrace(); + resourcePatternWorkSet.add(Pair.create(condition, pattern)); + } catch (UnsupportedOperationException e) { + UserError.abort("Resource registration should be performed before beforeAnalysis phase."); } - - registerConditionalConfiguration(condition, () -> { - UserError.guarantee(!sealed, "Resources added too late: %s", pattern); - resourcePatternWorkSet.add(pattern); - }); } private String urlToJarPath(URL url) { @@ -348,19 +292,24 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { loadedConfigurations = ConfigurationParserUtils.parseAndRegisterConfigurations(parser, imageClassLoader, "resource", ConfigurationFiles.Options.ResourceConfigurationFiles, ConfigurationFiles.Options.ResourceConfigurationResources, ConfigurationFile.RESOURCES.getFileName()); - - resourcePatternWorkSet.addAll(Options.IncludeResources.getValue().values()); + resourcePatternWorkSet.addAll(Options.IncludeResources.getValue() + .values() + .stream() + .map(e -> Pair.create(ConfigurationCondition.alwaysTrue(), e)) + .toList()); excludedResourcePatterns.addAll(Options.ExcludeResources.getValue().values()); if (!resourcePatternWorkSet.isEmpty()) { FeatureImpl.BeforeAnalysisAccessImpl beforeAnalysisAccess = (FeatureImpl.BeforeAnalysisAccessImpl) access; - ResourcePattern[] includePatterns = compilePatterns(resourcePatternWorkSet); + Set> includePatterns = resourcePatternWorkSet + .stream() + .map(e -> Pair.create(e.getLeft(), makeResourcePattern(e.getRight()))) + .collect(Collectors.toSet()); if (MissingRegistrationUtils.throwMissingRegistrationErrors()) { for (ResourcePattern resourcePattern : includePatterns) { Resources.singleton().registerIncludePattern(resourcePattern.moduleName, resourcePattern.pattern.pattern()); } } - ResourcePattern[] excludePatterns = compilePatterns(excludedResourcePatterns); DebugContext debugContext = beforeAnalysisAccess.getDebugContext(); ResourceCollectorImpl collector = new ResourceCollectorImpl(debugContext, includePatterns, excludePatterns); @@ -381,7 +330,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { private static final class ResourceCollectorImpl implements ResourceCollector { private final DebugContext debugContext; - private final ResourcePattern[] includePatterns; + private final Set> includePatterns; private final ResourcePattern[] excludePatterns; private static final int WATCHDOG_RESET_AFTER_EVERY_N_RESOURCES = 1000; @@ -423,7 +372,7 @@ private void shutDownProgressReporter() { } @Override - public boolean isIncluded(Module module, String resourceName, URI resource) { + public ConfigurationCondition isIncluded(Module module, String resourceName, URI resource) { this.currentlyProcessedEntry = resource.getScheme().equals("jrt") ? (resource + "/" + resourceName) : resource.toString(); this.reachedResourceEntries.increment(); @@ -439,20 +388,20 @@ public boolean isIncluded(Module module, String resourceName, URI resource) { continue; } if (rp.pattern.matcher(resourceName).matches() || rp.pattern.matcher(relativePathWithTrailingSlash).matches()) { - return false; + return null; } } - for (ResourcePattern rp : includePatterns) { - if (!rp.moduleNameMatches(moduleName)) { + for (Pair rp : includePatterns) { + if (!rp.getRight().moduleNameMatches(moduleName)) { continue; } - if (rp.pattern.matcher(resourceName).matches() || rp.pattern.matcher(relativePathWithTrailingSlash).matches()) { - return true; + if (rp.getRight().pattern.matcher(resourceName).matches() || rp.getRight().pattern.matcher(relativePathWithTrailingSlash).matches()) { + return rp.getLeft(); } } - return false; + return null; } @Override @@ -460,6 +409,11 @@ public void addResource(Module module, String resourceName, InputStream resource registerResource(debugContext, module, resourceName, resourceStream, fromJar); } + @Override + public void addResourceConditionally(Module module, String resourceName, ConfigurationCondition condition) { + access.registerReachabilityHandler(e -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourceName), access.findClassByName(condition.getTypeName())); + } + @Override public void addDirectoryResource(Module module, String dir, String content, boolean fromJar) { registerDirectoryResource(debugContext, module, dir, content, fromJar); @@ -474,6 +428,12 @@ public void registerIOException(Module module, String resourceName, IOException public void registerNegativeQuery(Module module, String resourceName) { Resources.singleton().registerNegativeQuery(module, resourceName); } + + // TODO if pass all gates - remove + @Override + public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition) { + access.registerReachabilityHandler(e -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, dir), access.findClassByName(condition.getTypeName())); + } } private ResourcePattern[] compilePatterns(Set patterns) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java index c863b5c6261b..e7addb2ac95d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java @@ -24,14 +24,16 @@ */ package com.oracle.svm.hosted.jdk.localization; +import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; + import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; -import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; @AutomaticallyRegisteredFeature class CharsetSubstitutionsFeature implements InternalFeature { @Override public void beforeAnalysis(BeforeAnalysisAccess access) { - access.registerReachabilityHandler((e) -> RuntimeResourceAccess.addResource(ModuleLayer.boot().findModule("java.base").get(), "java/lang/uniName.dat"), access.findClassByName("java.lang.CharacterName")); + access.registerReachabilityHandler((e) -> RuntimeResourceAccess.addResource(ModuleLayer.boot().findModule("java.base").get(), + "java/lang/uniName.dat"), access.findClassByName("java.lang.CharacterName")); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java index 156a2e4aabd7..1c7dc43f37b8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java @@ -162,9 +162,6 @@ public class LocalizationFeature implements InternalFeature { private Field localeObjectCacheMapField; private Field langAliasesCacheField; private Field parentLocalesMapField; - - @Platforms(Platform.HOSTED_ONLY.class) private ClassLoader classLoader; - @Platforms(Platform.HOSTED_ONLY.class) private ImageClassLoader imageClassLoader; public static class Options { @@ -298,7 +295,6 @@ public void duringSetup(DuringSetupAccess a) { ServiceLoader.load(ResourceBundleControlProvider.class).stream() .forEach(provider -> ImageSingletons.lookup(RuntimeClassInitializationSupport.class).initializeAtBuildTime(provider.type(), reason)); - this.classLoader = access.getApplicationClassLoader(); this.imageClassLoader = access.getImageClassLoader(); } diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java index 8b6559f55062..5bc549275818 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java @@ -1182,8 +1182,7 @@ final class Target_com_oracle_truffle_api_staticobject_PodBasedStaticShape { @TargetClass(className = "com.oracle.truffle.api.staticobject.ArrayBasedStaticShape", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_api_staticobject_ArrayBasedStaticShape { - @Alias - @RecomputeFieldValue(kind = Kind.Custom, declClass = MapCleaner.class, isFinal = true) // + @Alias @RecomputeFieldValue(kind = Kind.Custom, declClass = MapCleaner.class, isFinal = true) // static ConcurrentHashMap replacements; private static class MapCleaner implements FieldValueTransformerWithAvailability { @@ -1365,8 +1364,7 @@ final class Target_com_oracle_truffle_polyglot_LanguageCache { * verification in DisallowedImageHeapObjectFeature, so we also do the implicit reset using a * substitution. */ - @Alias - @RecomputeFieldValue(kind = Kind.Reset) // + @Alias @RecomputeFieldValue(kind = Kind.Reset) // private String languageHome; } @@ -1411,15 +1409,13 @@ final class Target_com_oracle_truffle_polyglot_InternalResourceCache_ResettableC @TargetClass(className = "com.oracle.truffle.object.CoreLocations$DynamicObjectFieldLocation", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_object_CoreLocations_DynamicObjectFieldLocation { - @Alias - @RecomputeFieldValue(kind = Kind.AtomicFieldUpdaterOffset) // + @Alias @RecomputeFieldValue(kind = Kind.AtomicFieldUpdaterOffset) // private long offset; } @TargetClass(className = "com.oracle.truffle.object.CoreLocations$DynamicLongFieldLocation", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_object_CoreLocations_DynamicLongFieldLocation { - @Alias - @RecomputeFieldValue(kind = Kind.AtomicFieldUpdaterOffset) // + @Alias @RecomputeFieldValue(kind = Kind.AtomicFieldUpdaterOffset) // private long offset; } @@ -1446,8 +1442,7 @@ final class Target_com_oracle_truffle_api_nodes_Node { @TargetClass(className = "com.oracle.truffle.api.nodes.NodeClassImpl", innerClass = "NodeFieldData", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_api_nodes_NodeClassImpl_NodeFieldData { - @Alias - @RecomputeFieldValue(kind = Kind.Custom, declClass = OffsetComputer.class, isFinal = true) // + @Alias @RecomputeFieldValue(kind = Kind.Custom, declClass = OffsetComputer.class, isFinal = true) // private long offset; private static class OffsetComputer implements FieldValueTransformerWithAvailability { @@ -1473,8 +1468,7 @@ public Object transform(Object receiver, Object originalValue) { @TargetClass(className = "com.oracle.truffle.api.dsl.InlineSupport$UnsafeField", onlyWith = TruffleBaseFeature.IsEnabled.class) final class Target_com_oracle_truffle_api_dsl_InlineSupport_UnsafeField { - @Alias - @RecomputeFieldValue(kind = Kind.Custom, declClass = OffsetComputer.class, isFinal = true) // + @Alias @RecomputeFieldValue(kind = Kind.Custom, declClass = OffsetComputer.class, isFinal = true) // private long offset; /* From 7a34bb3b4cbea2e2af5ba17a49deeb8e69d626cc Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 5 Jul 2023 10:00:51 +0200 Subject: [PATCH 37/78] Add conditional resource directory registration test --- substratevm/mx.substratevm/mx_substratevm.py | 10 +++++ .../oracle/svm/core/ClassLoaderSupport.java | 2 +- .../svm/hosted/ClassLoaderSupportImpl.java | 44 ++++++++++++------- .../oracle/svm/hosted/ResourcesFeature.java | 4 +- .../com.oracle.svm.test/resource-config.json | 13 ++++++ .../svm/test/NativeImageResourceTest.java | 31 +++++++++++++ 6 files changed, 84 insertions(+), 20 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/resource-config.json diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index a2013afb94e5..ace077d69a27 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -28,6 +28,7 @@ import tempfile from glob import glob from contextlib import contextmanager +from distutils.dir_util import mkpath, remove_tree # pylint: disable=no-name-in-module from os.path import join, exists, dirname import pipes from argparse import ArgumentParser @@ -475,9 +476,18 @@ def native_unittests_task(extra_build_args=None): # GR-24075 mx_unittest.add_global_ignore_glob('com.oracle.svm.test.ProcessPropertiesTest') + # add resources that are not in jar but in the separate directory + cp_entry_name = join(svmbuild_dir(), 'cpEntryDir') + mkpath(cp_entry_name) + for i in range(4): + with open(join(cp_entry_name, "resourcesFromDir", f'cond-resource{i}.txt'), 'w') as out: + out.write(f"Conditional file{i}" + '\n') + + additional_build_args = svm_experimental_options([ '-H:AdditionalSecurityProviders=com.oracle.svm.test.SecurityServiceTest$NoOpProvider', '-H:AdditionalSecurityServiceTypes=com.oracle.svm.test.SecurityServiceTest$JCACompliantNoOpService', + '-cp', cp_entry_name ]) if extra_build_args is not None: additional_build_args += extra_build_args diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java index 2b5fd7d4f431..a9ec9d579594 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java @@ -66,7 +66,7 @@ public interface ResourceCollector { void registerIOException(Module module, String resourceName, IOException e, boolean linkAtBuildTime); - void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition); + void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index d1dd62a5e2c1..39165aee4aa3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -175,7 +175,8 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) throws IOException { Map, List> matchedDirectoryResources = new HashMap<>(); - Set> allEntries = new HashSet<>(); + Map> conditionsForDirectory = new HashMap<>(); + Set allEntries = new HashSet<>(); ArrayDeque queue = new ArrayDeque<>(); queue.push(root); @@ -188,15 +189,19 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea if (entry != root) { relativeFilePath = root.relativize(entry).toString().replace(File.separatorChar, RESOURCES_INTERNAL_PATH_SEPARATOR); condition = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); - allEntries.add(Pair.create(condition, relativeFilePath)); + allEntries.add(relativeFilePath); } else { - relativeFilePath = ""; + relativeFilePath = String.valueOf(RESOURCES_INTERNAL_PATH_SEPARATOR); condition = collector.isIncluded(null, relativeFilePath, Path.of(relativeFilePath).toUri()); } if (Files.isDirectory(entry)) { if (condition != null) { matchedDirectoryResources.put(Pair.create(condition, relativeFilePath), new ArrayList<>()); + // add new condition for the directory (if that is the first condition, create + // list) + conditionsForDirectory.computeIfAbsent(relativeFilePath, k -> new ArrayList<>()); + conditionsForDirectory.get(relativeFilePath).add(condition); } try (Stream pathStream = Files.list(entry)) { Stream filtered = pathStream; @@ -215,7 +220,6 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea } else { collector.addResourceConditionally(null, relativeFilePath, condition); } - collector.addResource(null, relativeFilePath, is, false); } catch (IOException resourceException) { collector.registerIOException(null, relativeFilePath, resourceException, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(relativeFilePath)); } @@ -223,16 +227,21 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea } } - for (Pair entry : allEntries) { - ConfigurationCondition condition = entry.getLeft(); - String entryName = entry.getRight(); - int last = entryName.lastIndexOf(RESOURCES_INTERNAL_PATH_SEPARATOR); - String key = last == -1 ? "" : entryName.substring(0, last); - List dirContent = matchedDirectoryResources.get(Pair.create(condition, key)); - if (dirContent != null && !dirContent.contains(entryName)) { - dirContent.add(entryName.substring(last + 1)); - } else if (dirContent == null) { - collector.registerNegativeQuery(null, key); + for (String entry : allEntries) { + int last = entry.lastIndexOf(RESOURCES_INTERNAL_PATH_SEPARATOR); + String parentDirectory = last == -1 ? "" : entry.substring(0, last); + // one parent can be added under various condition, so we have to add its content for + // all conditions + List conditions = conditionsForDirectory.get(parentDirectory); + if (conditions != null) { + for (ConfigurationCondition condition : conditions) { + List dirContent = matchedDirectoryResources.get(Pair.create(condition, parentDirectory)); + if (dirContent != null && !dirContent.contains(entry)) { + dirContent.add(entry.substring(last + 1)); + } else if (dirContent == null) { + collector.registerNegativeQuery(null, key); + } + } } } @@ -240,10 +249,11 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea ConfigurationCondition condition = entry.getLeft(); String dir = entry.getRight(); content.sort(Comparator.naturalOrder()); + String contentName = String.join(System.lineSeparator(), content); if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addDirectoryResource(null, dir, String.join(System.lineSeparator(), content), false); + collector.addDirectoryResource(null, dir, contentName, false); } else { - collector.addDirectoryResourceConditionally(null, dir, condition); + collector.addDirectoryResourceConditionally(null, dir, condition, contentName, false); } }); } @@ -262,7 +272,7 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i if (ConfigurationCondition.isAlwaysTrue(condition)) { collector.addDirectoryResource(null, dirName, "", true); } else { - collector.addDirectoryResourceConditionally(null, dirName, condition); + collector.addDirectoryResourceConditionally(null, dirName, condition, "", true); } } } else { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 5fb1e7f66b40..73e905f2e847 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -431,8 +431,8 @@ public void registerNegativeQuery(Module module, String resourceName) { // TODO if pass all gates - remove @Override - public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition) { - access.registerReachabilityHandler(e -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, dir), access.findClassByName(condition.getTypeName())); + public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar) { + access.registerReachabilityHandler(e -> addDirectoryResource(module, dir, content, fromJar), access.findClassByName(condition.getTypeName())); } } diff --git a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/resource-config.json b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/resource-config.json new file mode 100644 index 000000000000..bdba15d5f51b --- /dev/null +++ b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/resource-config.json @@ -0,0 +1,13 @@ +{ + "bundles": [], + "resources": { + "includes": [ + { + "condition": { + "typeReachable": "java.lang.String" + }, + "pattern": "resourcesFromDir/.*" + } + ] + } +} \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java index 2afbe70d49cb..a1373e673186 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java @@ -33,18 +33,22 @@ import static com.oracle.svm.test.NativeImageResourceUtils.compareTwoURLs; import static com.oracle.svm.test.NativeImageResourceUtils.resourceNameToURL; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Enumeration; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.StreamSupport; import org.junit.Assert; @@ -126,6 +130,33 @@ public void classGetDirectoryResource() { resourceNameToURL(nonCanonicalResourceDirectoryName, false); } + @Test + public void getConditionalDirectoryResource() throws IOException { + // check if resource is added conditionally + String directoryName = "/resourcesFromDir"; + URL directory = NativeImageResourceUtils.class.getResource(directoryName); + Assert.assertNotNull("Resource " + directory + " is not found!", directory); + + // check content of resource + List expected = IntStream.range(0, 4).mapToObj(i -> "cond-resource" + i + ".txt").toList(); + BufferedReader reader = new BufferedReader(new InputStreamReader(directory.openStream())); + List actual = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + actual.add(line); + } + + for (String resource : expected) { + // check if resource contains expected content + Assert.assertTrue(actual.contains(resource)); + + // check if we can get resource which directory contains + String resourceName = directoryName + "/" + resource; + URL resourceUrl = NativeImageResourceUtils.class.getResource(resourceName); + Assert.assertNotNull("Cannot find resource: " + resourceName, resourceUrl); + } + } + /** *

* Access a resource using {@link URLClassLoader}. From 8b02c19a2975979fb0bdfd64e6b32536b916a85b Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 12 Jul 2023 18:10:21 +0200 Subject: [PATCH 38/78] Refactor adding content to directory resource --- .../svm/hosted/ClassLoaderSupportImpl.java | 41 ++++++++----------- .../com.oracle.svm.test/resource-config.json | 2 +- .../svm/test/NativeImageResourceTest.java | 6 --- 3 files changed, 17 insertions(+), 32 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 39165aee4aa3..aa8c8da2351a 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -173,6 +173,20 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso } } + private static String makeDirContent(Set allEntries, String dir) { + List dirContent = new ArrayList<>(); + for (String entry : allEntries) { + int last = entry.lastIndexOf(RESOURCES_INTERNAL_PATH_SEPARATOR); + String parentDirectory = last == -1 ? "" : entry.substring(0, last); + if (parentDirectory.equals(dir)) { + dirContent.add(entry.substring(last + 1)); + } + } + + dirContent.sort(Comparator.naturalOrder()); + return String.join(System.lineSeparator(), dirContent); + } + private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) throws IOException { Map, List> matchedDirectoryResources = new HashMap<>(); Map> conditionsForDirectory = new HashMap<>(); @@ -198,10 +212,6 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea if (Files.isDirectory(entry)) { if (condition != null) { matchedDirectoryResources.put(Pair.create(condition, relativeFilePath), new ArrayList<>()); - // add new condition for the directory (if that is the first condition, create - // list) - conditionsForDirectory.computeIfAbsent(relativeFilePath, k -> new ArrayList<>()); - conditionsForDirectory.get(relativeFilePath).add(condition); } try (Stream pathStream = Files.list(entry)) { Stream filtered = pathStream; @@ -227,29 +237,10 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea } } - for (String entry : allEntries) { - int last = entry.lastIndexOf(RESOURCES_INTERNAL_PATH_SEPARATOR); - String parentDirectory = last == -1 ? "" : entry.substring(0, last); - // one parent can be added under various condition, so we have to add its content for - // all conditions - List conditions = conditionsForDirectory.get(parentDirectory); - if (conditions != null) { - for (ConfigurationCondition condition : conditions) { - List dirContent = matchedDirectoryResources.get(Pair.create(condition, parentDirectory)); - if (dirContent != null && !dirContent.contains(entry)) { - dirContent.add(entry.substring(last + 1)); - } else if (dirContent == null) { - collector.registerNegativeQuery(null, key); - } - } - } - } - - matchedDirectoryResources.forEach((entry, content) -> { + matchedDirectoryResources.forEach(entry -> { ConfigurationCondition condition = entry.getLeft(); String dir = entry.getRight(); - content.sort(Comparator.naturalOrder()); - String contentName = String.join(System.lineSeparator(), content); + String contentName = makeDirContent(allEntries, dir); if (ConfigurationCondition.isAlwaysTrue(condition)) { collector.addDirectoryResource(null, dir, contentName, false); } else { diff --git a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/resource-config.json b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/resource-config.json index bdba15d5f51b..16146de3e580 100644 --- a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/resource-config.json +++ b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/resource-config.json @@ -6,7 +6,7 @@ "condition": { "typeReachable": "java.lang.String" }, - "pattern": "resourcesFromDir/.*" + "pattern": "resourcesFromDir" } ] } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java index a1373e673186..4f1158d86dde 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java @@ -147,13 +147,7 @@ public void getConditionalDirectoryResource() throws IOException { } for (String resource : expected) { - // check if resource contains expected content Assert.assertTrue(actual.contains(resource)); - - // check if we can get resource which directory contains - String resourceName = directoryName + "/" + resource; - URL resourceUrl = NativeImageResourceUtils.class.getResource(resourceName); - Assert.assertNotNull("Cannot find resource: " + resourceName, resourceUrl); } } From f2a4fbcdf26b23393ee03b869e2ac9a7ba303a31 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 18 Jul 2023 13:52:48 +0200 Subject: [PATCH 39/78] Replace pairs with records to get more semantic --- substratevm/mx.substratevm/mx_substratevm.py | 20 +++-- .../com/oracle/svm/core/jdk/Resources.java | 6 +- .../svm/hosted/ClassLoaderSupportImpl.java | 26 ++++--- .../oracle/svm/hosted/ResourcesFeature.java | 76 +++++++++++++------ .../svm/test/NativeImageResourceTest.java | 12 ++- .../svm/test/NativeImageResourceUtils.java | 2 + 6 files changed, 96 insertions(+), 46 deletions(-) diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index ace077d69a27..8159e089890b 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -476,13 +476,21 @@ def native_unittests_task(extra_build_args=None): # GR-24075 mx_unittest.add_global_ignore_glob('com.oracle.svm.test.ProcessPropertiesTest') - # add resources that are not in jar but in the separate directory - cp_entry_name = join(svmbuild_dir(), 'cpEntryDir') - mkpath(cp_entry_name) - for i in range(4): - with open(join(cp_entry_name, "resourcesFromDir", f'cond-resource{i}.txt'), 'w') as out: - out.write(f"Conditional file{i}" + '\n') + # add resources that are not in jar but in the separate directory + cp_entry_name = join(svmbuild_dir(), 'cpEntryDir') + resources_from_dir = join(cp_entry_name, 'resourcesFromDir') + simple_dir = join(cp_entry_name, 'simpleDir') + mkpath(cp_entry_name) + mkpath(resources_from_dir) + mkpath(simple_dir) + + for i in range(4): + with open(join(cp_entry_name, "resourcesFromDir", f'cond-resource{i}.txt'), 'w') as out: + out.write(f"Conditional file{i}" + '\n') + + with open(join(cp_entry_name, "simpleDir", f'simple-resource{i}.txt'), 'w') as out: + out.write(f"Simple file{i}" + '\n') additional_build_args = svm_experimental_options([ '-H:AdditionalSecurityProviders=com.oracle.svm.test.SecurityServiceTest$NoOpProvider', diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 85e2c519860b..9e8d191b5b2b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -48,6 +48,7 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import com.oracle.svm.core.BuildPhaseProvider; import com.oracle.svm.core.BuildPhaseProvider; import com.oracle.svm.core.MissingRegistrationUtils; import com.oracle.svm.core.SubstrateOptions; @@ -214,11 +215,6 @@ public void registerResource(Module module, String resourceName, InputStream is, addEntry(module, resourceName, false, inputStreamToByteArray(is), fromJar); } - @Platforms(Platform.HOSTED_ONLY.class) - public static void registerResource(Module module, String resourceName, boolean isDir, InputStream is, boolean fromJar) { - addEntry(module, resourceName, isDir, inputStreamToByteArray(is), fromJar); - } - @Platforms(Platform.HOSTED_ONLY.class) public void registerDirectoryResource(String resourceDirName, String content) { registerDirectoryResource(null, resourceDirName, content, true); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index aa8c8da2351a..b81ee9b691d2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -55,7 +55,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.graalvm.collections.Pair; import org.graalvm.nativeimage.impl.ConfigurationCondition; import com.oracle.svm.core.ClassLoaderSupport; @@ -74,6 +73,9 @@ public class ClassLoaderSupportImpl extends ClassLoaderSupport { private final Map> packageToModules; + private record ConditionalResource(ConfigurationCondition condition, String resourceName) { + } + public ClassLoaderSupportImpl(NativeImageClassLoaderSupport classLoaderSupport) { this.classLoaderSupport = classLoaderSupport; imageClassLoader = classLoaderSupport.getClassLoader(); @@ -132,17 +134,17 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso ModuleReference moduleReference = info.resolvedModule.reference(); try (ModuleReader moduleReader = moduleReference.open()) { var includeAll = classLoaderSupport.getJavaModuleNamesToInclude().contains(info.resolvedModule().name()); - List> resourcesFound = new ArrayList<>(); + List resourcesFound = new ArrayList<>(); moduleReader.list().forEach(resourceName -> { ConfigurationCondition condition = shouldIncludeEntry(info.module(), resourceCollector, resourceName, moduleReference.location().orElse(null), includeAll); if (condition != null) { - resourcesFound.add(Pair.create(condition, resourceName)); + resourcesFound.add(new ConditionalResource(condition, resourceName)); } }); - for (Pair entry : resourcesFound) { - ConfigurationCondition condition = entry.getLeft(); - String resName = entry.getRight(); + for (ConditionalResource entry : resourcesFound) { + ConfigurationCondition condition = entry.condition(); + String resName = entry.resourceName(); if (resName.endsWith("/")) { if (ConfigurationCondition.isAlwaysTrue(condition)) { resourceCollector.addDirectoryResource(info.module, resName, "", false); @@ -151,6 +153,7 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso } continue; } + Optional content = moduleReader.open(resName); if (content.isEmpty()) { /* This is to be resilient, but the resources returned by list() should exist */ @@ -188,9 +191,8 @@ private static String makeDirContent(Set allEntries, String dir) { } private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) throws IOException { - Map, List> matchedDirectoryResources = new HashMap<>(); - Map> conditionsForDirectory = new HashMap<>(); - Set allEntries = new HashSet<>(); + List matchedDirectoryResources = new ArrayList<>(); + Set allEntries = new HashSet<>(); ArrayDeque queue = new ArrayDeque<>(); queue.push(root); @@ -211,7 +213,7 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea if (Files.isDirectory(entry)) { if (condition != null) { - matchedDirectoryResources.put(Pair.create(condition, relativeFilePath), new ArrayList<>()); + matchedDirectoryResources.add(new ConditionalResource(condition, relativeFilePath)); } try (Stream pathStream = Files.list(entry)) { Stream filtered = pathStream; @@ -238,8 +240,8 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea } matchedDirectoryResources.forEach(entry -> { - ConfigurationCondition condition = entry.getLeft(); - String dir = entry.getRight(); + ConfigurationCondition condition = entry.condition(); + String dir = entry.resourceName(); String contentName = makeDirContent(allEntries, dir); if (ConfigurationCondition.isAlwaysTrue(condition)) { collector.addDirectoryResource(null, dir, contentName, false); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 73e905f2e847..836994221bd1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -27,11 +27,13 @@ import static com.oracle.svm.core.jdk.Resources.RESOURCES_INTERNAL_PATH_SEPARATOR; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.net.JarURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -42,6 +44,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Set; @@ -53,6 +56,7 @@ import java.util.jar.JarFile; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.debug.DebugContext; @@ -140,7 +144,13 @@ public static class Options { private boolean sealed = false; - private Set> resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private record ConditionalPattern(ConfigurationCondition condition, String pattern) { + } + + private record CompiledConditionalPattern(ConfigurationCondition condition, ResourcePattern compiledPattern) { + } + + private Set resourcePatternWorkSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set excludedResourcePatterns = Collections.newSetFromMap(new ConcurrentHashMap<>()); private int loadedConfigurations; private ImageClassLoader imageClassLoader; @@ -159,14 +169,18 @@ public void addResources(ConfigurationCondition condition, String pattern) { } try { - resourcePatternWorkSet.add(Pair.create(condition, pattern)); + resourcePatternWorkSet.add(new ConditionalPattern(condition, pattern)); } catch (UnsupportedOperationException e) { UserError.abort("Resource registration should be performed before beforeAnalysis phase."); } } private String urlToJarPath(URL url) { - return String.valueOf(url).split("jar:file:")[1].split("!")[0]; + try { + return ((JarURLConnection) url.openConnection()).getJarFileURL().getFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } } private boolean resourceIsDirectory(URL url, boolean fromJar, String resourcePath) throws IOException, URISyntaxException { @@ -179,11 +193,20 @@ private boolean resourceIsDirectory(URL url, boolean fromJar, String resourcePat } } + private String getDirectoryContent(String dir) throws IOException { + try (Stream contentStream = Files.list(Path.of(dir))) { + List content = new ArrayList<>(contentStream.map(Path::toString).toList()); + content.sort(Comparator.naturalOrder()); + return String.join(System.lineSeparator(), content); + } + } + @Override public void addResource(Module module, String resourcePath) { InputStream is; - boolean fromJar = false; - boolean isDirectory = false; + boolean fromJar; + boolean isDirectory; + String content = ""; if (module != null && module.isNamed()) { try { @@ -196,6 +219,11 @@ public void addResource(Module module, String resourcePath) { } is = module.getResourceAsStream(resourcePath); + fromJar = false; + isDirectory = new File(resourcePath).isDirectory(); + if (isDirectory) { + content = getDirectoryContent(resourcePath); + } } catch (IOException e) { // we should ignore if user failed to provide resource return; @@ -207,28 +235,32 @@ public void addResource(Module module, String resourcePath) { return; } - fromJar = url.getProtocol().equalsIgnoreCase("jar"); try { + is = url.openStream(); + fromJar = url.getProtocol().equalsIgnoreCase("jar"); isDirectory = resourceIsDirectory(url, fromJar, resourcePath); + // if directory is from jar content should remain empty (same as in scanJar + // function from ClassLoaderSupportImpl) + if (isDirectory && !fromJar) { + content = getDirectoryContent(url.getPath()); + } } catch (IOException e) { // we should ignore if user failed to provide resource return; } catch (URISyntaxException e) { throw new RuntimeException(e); } - - try { - is = url.openStream(); - } catch (IOException e) { - throw new RuntimeException(e); - } } if (is == null) { return; } - Resources.registerResource(module, resourcePath, isDirectory, is, fromJar); + if (isDirectory) { + Resources.registerDirectoryResource(module, resourcePath, content); + } else { + Resources.registerResource(module, resourcePath, is, fromJar); + } } @Override @@ -295,15 +327,15 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { resourcePatternWorkSet.addAll(Options.IncludeResources.getValue() .values() .stream() - .map(e -> Pair.create(ConfigurationCondition.alwaysTrue(), e)) + .map(e -> new ConditionalPattern(ConfigurationCondition.alwaysTrue(), e)) .toList()); excludedResourcePatterns.addAll(Options.ExcludeResources.getValue().values()); if (!resourcePatternWorkSet.isEmpty()) { FeatureImpl.BeforeAnalysisAccessImpl beforeAnalysisAccess = (FeatureImpl.BeforeAnalysisAccessImpl) access; - Set> includePatterns = resourcePatternWorkSet + Set includePatterns = resourcePatternWorkSet .stream() - .map(e -> Pair.create(e.getLeft(), makeResourcePattern(e.getRight()))) + .map(e -> new CompiledConditionalPattern(e.condition(), makeResourcePattern(e.pattern()))) .collect(Collectors.toSet()); if (MissingRegistrationUtils.throwMissingRegistrationErrors()) { for (ResourcePattern resourcePattern : includePatterns) { @@ -330,7 +362,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { private static final class ResourceCollectorImpl implements ResourceCollector { private final DebugContext debugContext; - private final Set> includePatterns; + private final Set includePatterns; private final ResourcePattern[] excludePatterns; private static final int WATCHDOG_RESET_AFTER_EVERY_N_RESOURCES = 1000; @@ -341,7 +373,7 @@ private static final class ResourceCollectorImpl implements ResourceCollector { private volatile String currentlyProcessedEntry; ScheduledExecutorService scheduledExecutor; - private ResourceCollectorImpl(DebugContext debugContext, ResourcePattern[] includePatterns, ResourcePattern[] excludePatterns) { + private ResourceCollectorImpl(DebugContext debugContext, Set includePatterns, ResourcePattern[] excludePatterns) { this.debugContext = debugContext; this.includePatterns = includePatterns; this.excludePatterns = excludePatterns; @@ -392,12 +424,12 @@ public ConfigurationCondition isIncluded(Module module, String resourceName, URI } } - for (Pair rp : includePatterns) { - if (!rp.getRight().moduleNameMatches(moduleName)) { + for (CompiledConditionalPattern rp : includePatterns) { + if (!rp.compiledPattern().moduleNameMatches(moduleName)) { continue; } - if (rp.getRight().pattern.matcher(resourceName).matches() || rp.getRight().pattern.matcher(relativePathWithTrailingSlash).matches()) { - return rp.getLeft(); + if (rp.compiledPattern().pattern.matcher(resourceName).matches() || rp.compiledPattern().pattern.matcher(relativePathWithTrailingSlash).matches()) { + return rp.condition(); } } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java index 4f1158d86dde..a4fe9af3cfbf 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceTest.java @@ -30,6 +30,7 @@ import static com.oracle.svm.test.NativeImageResourceUtils.RESOURCE_FILE_2; import static com.oracle.svm.test.NativeImageResourceUtils.RESOURCE_FILE_3; import static com.oracle.svm.test.NativeImageResourceUtils.RESOURCE_FILE_4; +import static com.oracle.svm.test.NativeImageResourceUtils.SIMPLE_RESOURCE_DIR; import static com.oracle.svm.test.NativeImageResourceUtils.compareTwoURLs; import static com.oracle.svm.test.NativeImageResourceUtils.resourceNameToURL; @@ -130,12 +131,21 @@ public void classGetDirectoryResource() { resourceNameToURL(nonCanonicalResourceDirectoryName, false); } + @Test + public void registeredResourceDirectoryHasContent() throws IOException { + URL directory = NativeImageResourceUtils.class.getResource(SIMPLE_RESOURCE_DIR); + Assert.assertNotNull("Resource " + SIMPLE_RESOURCE_DIR + " is not found!", directory); + + BufferedReader reader = new BufferedReader(new InputStreamReader(directory.openStream())); + Assert.assertNotNull("Resource" + SIMPLE_RESOURCE_DIR + " should have content", reader.readLine()); + } + @Test public void getConditionalDirectoryResource() throws IOException { // check if resource is added conditionally String directoryName = "/resourcesFromDir"; URL directory = NativeImageResourceUtils.class.getResource(directoryName); - Assert.assertNotNull("Resource " + directory + " is not found!", directory); + Assert.assertNotNull("Resource " + directoryName + " is not found!", directory); // check content of resource List expected = IntStream.range(0, 4).mapToObj(i -> "cond-resource" + i + ".txt").toList(); diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceUtils.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceUtils.java index 8d07f56f7f91..7cd52b889c7a 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceUtils.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/NativeImageResourceUtils.java @@ -42,6 +42,7 @@ public class NativeImageResourceUtils { public static final String ROOT_DIRECTORY = "/"; public static final String RESOURCE_DIR = "/resources"; + public static final String SIMPLE_RESOURCE_DIR = "/simpleDir"; public static final String RESOURCE_EMPTY_DIR = RESOURCE_DIR + "/empty"; public static final String RESOURCE_FILE_1 = RESOURCE_DIR + "/resource-test1.txt"; public static final String RESOURCE_FILE_2 = RESOURCE_DIR + "/resource-test2.txt"; @@ -55,6 +56,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { // Remove leading / for the resource patterns Module resourceModule = TestFeature.class.getModule(); RuntimeResourceAccess.addResource(resourceModule, RESOURCE_DIR.substring(1)); + RuntimeResourceAccess.addResource(resourceModule, SIMPLE_RESOURCE_DIR.substring(1)); RuntimeResourceAccess.addResource(resourceModule, RESOURCE_EMPTY_DIR.substring(1)); RuntimeResourceAccess.addResource(resourceModule, RESOURCE_FILE_1.substring(1)); RuntimeResourceAccess.addResource(resourceModule, RESOURCE_FILE_2.substring(1)); From f49fe93daa6c6457fc9e6b68458a30e75095600f Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 19 Jul 2023 17:02:28 +0200 Subject: [PATCH 40/78] Get content for directories inside jar --- .../oracle/svm/hosted/ResourcesFeature.java | 68 ++++++++++++++++--- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 836994221bd1..aa25d248caf8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -38,13 +38,16 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.Enumeration; import java.util.List; import java.util.Locale; import java.util.Set; @@ -53,6 +56,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; +import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -193,12 +197,54 @@ private boolean resourceIsDirectory(URL url, boolean fromJar, String resourcePat } } - private String getDirectoryContent(String dir) throws IOException { - try (Stream contentStream = Files.list(Path.of(dir))) { - List content = new ArrayList<>(contentStream.map(Path::toString).toList()); - content.sort(Comparator.naturalOrder()); - return String.join(System.lineSeparator(), content); + private String getDirectoryContent(String path, boolean fromJar) throws IOException { + List content = new ArrayList<>(); + if (fromJar) { + try (JarFile jf = new JarFile(urlToJarPath(URI.create(path).toURL()))) { + String pathSeparator = FileSystems.getDefault().getSeparator(); + String directoryPath = path.split("!")[1]; + + // we are removing leading slash because jar entry names don't start with slash + if (directoryPath.startsWith(pathSeparator)) { + directoryPath = directoryPath.substring(1); + } + + Enumeration entries = jf.entries(); + while (entries.hasMoreElements()) { + String entry = entries.nextElement().getName(); + if (entry.startsWith(directoryPath)) { + String contentEntry = entry.substring(directoryPath.length()); + + // remove the leading slash + if (contentEntry.startsWith(pathSeparator)) { + contentEntry = contentEntry.substring(1); + } + + // prevent adding empty strings as a content + if (!contentEntry.isEmpty()) { + // get top level content only + int firstSlash = contentEntry.indexOf(pathSeparator); + if (firstSlash != -1) { + content.add(contentEntry.substring(0, firstSlash)); + } else { + content.add(contentEntry); + } + } + } + } + + } + } else { + try (Stream contentStream = Files.list(Path.of(path))) { + content = new ArrayList<>(contentStream + .map(Path::getFileName) + .map(Path::toString) + .toList()); + } } + + content.sort(Comparator.naturalOrder()); + return String.join(System.lineSeparator(), content); } @Override @@ -222,7 +268,7 @@ public void addResource(Module module, String resourcePath) { fromJar = false; isDirectory = new File(resourcePath).isDirectory(); if (isDirectory) { - content = getDirectoryContent(resourcePath); + content = getDirectoryContent(resourcePath, false); } } catch (IOException e) { // we should ignore if user failed to provide resource @@ -241,8 +287,8 @@ public void addResource(Module module, String resourcePath) { isDirectory = resourceIsDirectory(url, fromJar, resourcePath); // if directory is from jar content should remain empty (same as in scanJar // function from ClassLoaderSupportImpl) - if (isDirectory && !fromJar) { - content = getDirectoryContent(url.getPath()); + if (isDirectory) { + content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); } } catch (IOException e) { // we should ignore if user failed to provide resource @@ -261,6 +307,12 @@ public void addResource(Module module, String resourcePath) { } else { Resources.registerResource(module, resourcePath, is, fromJar); } + + try { + is.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override From a5ca553276c0ed00c8de1260499b54e717c51bab Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 28 Jul 2023 17:01:47 +0200 Subject: [PATCH 41/78] Add possiblity to register resource for various conditions --- .../oracle/svm/core/ClassLoaderSupport.java | 2 +- .../svm/hosted/ClassLoaderSupportImpl.java | 31 ++++++++++--------- .../oracle/svm/hosted/ResourcesFeature.java | 10 +++--- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java index a9ec9d579594..85851ab218b3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java @@ -54,7 +54,7 @@ public boolean isNativeImageClassLoader(ClassLoader classLoader) { protected abstract boolean isNativeImageClassLoaderImpl(ClassLoader classLoader); public interface ResourceCollector { - ConfigurationCondition isIncluded(Module module, String resourceName, URI resourceURI); + List isIncluded(Module module, String resourceName, URI resourceURI); void addResource(Module module, String resourceName, InputStream resourceStream, boolean fromJar); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index b81ee9b691d2..fc8c85f7a941 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -136,8 +136,8 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso var includeAll = classLoaderSupport.getJavaModuleNamesToInclude().contains(info.resolvedModule().name()); List resourcesFound = new ArrayList<>(); moduleReader.list().forEach(resourceName -> { - ConfigurationCondition condition = shouldIncludeEntry(info.module(), resourceCollector, resourceName, moduleReference.location().orElse(null), includeAll); - if (condition != null) { + List conditions = shouldIncludeEntry(info.module, resourceCollector, resourceName, moduleReference.location().orElse(null), includeAll); + for (ConfigurationCondition condition : conditions) { resourcesFound.add(new ConditionalResource(condition, resourceName)); } }); @@ -192,7 +192,7 @@ private static String makeDirContent(Set allEntries, String dir) { private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) throws IOException { List matchedDirectoryResources = new ArrayList<>(); - Set allEntries = new HashSet<>(); + Set allEntries = new HashSet<>(); ArrayDeque queue = new ArrayDeque<>(); queue.push(root); @@ -201,20 +201,21 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea /* Resources always use / as the separator, as do our resource inclusion patterns */ String relativeFilePath; - ConfigurationCondition condition; + List conditions; if (entry != root) { relativeFilePath = root.relativize(entry).toString().replace(File.separatorChar, RESOURCES_INTERNAL_PATH_SEPARATOR); - condition = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); + conditions = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); allEntries.add(relativeFilePath); } else { relativeFilePath = String.valueOf(RESOURCES_INTERNAL_PATH_SEPARATOR); - condition = collector.isIncluded(null, relativeFilePath, Path.of(relativeFilePath).toUri()); + conditions = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); } if (Files.isDirectory(entry)) { - if (condition != null) { + for (ConfigurationCondition condition : conditions) { matchedDirectoryResources.add(new ConditionalResource(condition, relativeFilePath)); } + try (Stream pathStream = Files.list(entry)) { Stream filtered = pathStream; if (ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT.equals(entry)) { @@ -225,7 +226,7 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea collector.registerIOException(null, relativeFilePath, resourceException, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(relativeFilePath)); } } else { - if (condition != null) { + for (ConfigurationCondition condition : conditions) { try (InputStream is = Files.newInputStream(entry)) { if (ConfigurationCondition.isAlwaysTrue(condition)) { collector.addResource(null, relativeFilePath, is, false); @@ -259,8 +260,8 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i URI uri = jarPath.toUri(); if (entry.isDirectory()) { String dirName = entry.getName().substring(0, entry.getName().length() - 1); - ConfigurationCondition condition = shouldIncludeEntry(null, collector, dirName, jarPath.toUri(), includeAll); - if (condition != null) { + List conditions = shouldIncludeEntry(null, collector, dirName, jarPath.toUri(), includeAll); + for (ConfigurationCondition condition : conditions) { // Register the directory with empty content to preserve Java behavior if (ConfigurationCondition.isAlwaysTrue(condition)) { collector.addDirectoryResource(null, dirName, "", true); @@ -269,8 +270,8 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i } } } else { - ConfigurationCondition condition = shouldIncludeEntry(null, collector, entry.getName(), jarPath.toUri(), includeAll); - if (condition != null) { + List conditions = shouldIncludeEntry(null, collector, entry.getName(), jarPath.toUri(), includeAll); + for (ConfigurationCondition condition : conditions) { try (InputStream is = jf.getInputStream(entry)) { if (ConfigurationCondition.isAlwaysTrue(condition)) { collector.addResource(null, entry.getName(), is, true); @@ -286,10 +287,10 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i } } - private static ConfigurationCondition shouldIncludeEntry(Module module, ResourceCollector collector, String fileName, URI uri, boolean includeAll) { - ConfigurationCondition conditions = collector.isIncluded(module, fileName, uri); + private static List shouldIncludeEntry(Module module, ResourceCollector collector, String fileName, URI uri, boolean includeAll) { + List conditions = collector.isIncluded(module, fileName, uri); if (includeAll && !(fileName.endsWith(".class") || fileName.endsWith(".jar"))) { - return ConfigurationCondition.alwaysTrue(); + return Collections.singletonList(ConfigurationCondition.alwaysTrue()); } return conditions; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index aa25d248caf8..2fb02cfeb051 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -456,7 +456,9 @@ private void shutDownProgressReporter() { } @Override - public ConfigurationCondition isIncluded(Module module, String resourceName, URI resource) { + public List isIncluded(Module module, String resourceName, URI resource) { + // Possibly we can have multiple conditions for one resource + List conditions = new ArrayList<>(); this.currentlyProcessedEntry = resource.getScheme().equals("jrt") ? (resource + "/" + resourceName) : resource.toString(); this.reachedResourceEntries.increment(); @@ -472,7 +474,7 @@ public ConfigurationCondition isIncluded(Module module, String resourceName, URI continue; } if (rp.pattern.matcher(resourceName).matches() || rp.pattern.matcher(relativePathWithTrailingSlash).matches()) { - return null; + return conditions; // returns empty list } } @@ -481,11 +483,11 @@ public ConfigurationCondition isIncluded(Module module, String resourceName, URI continue; } if (rp.compiledPattern().pattern.matcher(resourceName).matches() || rp.compiledPattern().pattern.matcher(relativePathWithTrailingSlash).matches()) { - return rp.condition(); + conditions.add(rp.condition()); } } - return null; + return conditions; } @Override From 1f28251c66df1502b23146bc329d31e1ac974319 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 1 Aug 2023 17:23:52 +0200 Subject: [PATCH 42/78] Prevent registration of same resources multiple times --- .../com/oracle/svm/core/jdk/Resources.java | 18 ++++++++------ .../oracle/svm/hosted/ResourcesFeature.java | 24 +++++++++++++++++-- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 9e8d191b5b2b..2165af201c39 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -167,12 +167,14 @@ private void updateTimeStamp() { } @Platforms(Platform.HOSTED_ONLY.class) - private ResourceStorageEntryBase addEntry(Module module, String resourceName, ResourceStorageEntryBase newEntry, boolean isDirectory, boolean fromJar) { + private static void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar) { + VMError.guarantee(!BuildPhaseProvider.isAnalysisFinished(), "Trying to add a resource entry after analysis."); Module m = module != null && module.isNamed() ? module : null; if (m != null) { m = RuntimeModuleSupport.instance().getRuntimeModuleForHostedModule(m); } + var resources = singleton().resources; synchronized (resources) { Pair key = createStorageKey(m, resourceName); ResourceStorageEntryBase entry = resources.get(key); @@ -180,14 +182,16 @@ private ResourceStorageEntryBase addEntry(Module module, String resourceName, Re entry = newEntry == null ? new ResourceStorageEntry(isDirectory, fromJar) : newEntry; updateTimeStamp(); resources.put(key, entry); + } else { + if (key.getLeft() != null) { + // if the entry already exists, and it comes from a module, it is the same entry + // that we registered at some point before + return; + } } - return entry; - } - } - private void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar) { - ResourceStorageEntryBase entry = addEntry(module, resourceName, null, isDirectory, fromJar); - entry.getData().add(data); + entry.getData().add(data); + } } @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 2fb02cfeb051..7ab2dffd6dc9 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -48,6 +48,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.Enumeration; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; @@ -490,6 +491,25 @@ public List isIncluded(Module module, String resourceNam return conditions; } + private final Set alreadyAddedResources = new HashSet<>(); + + private void registerResourceIfNeeded(boolean isDirectory, Object... arguments) { + Module module = (Module) arguments[0]; + String resourceName = (String) arguments[1]; + // we only do this if we are on the classPath + if ((module == null || !module.isNamed()) && !alreadyAddedResources.contains(resourceName)) { + if (isDirectory) { + String content = (String) arguments[2]; + boolean fromJar = (boolean) arguments[3]; + addDirectoryResource(module, resourceName, content, fromJar); + } else { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourceName); + } + + alreadyAddedResources.add(resourceName); + } + } + @Override public void addResource(Module module, String resourceName, InputStream resourceStream, boolean fromJar) { registerResource(debugContext, module, resourceName, resourceStream, fromJar); @@ -497,7 +517,7 @@ public void addResource(Module module, String resourceName, InputStream resource @Override public void addResourceConditionally(Module module, String resourceName, ConfigurationCondition condition) { - access.registerReachabilityHandler(e -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourceName), access.findClassByName(condition.getTypeName())); + access.registerReachabilityHandler(e -> registerResourceIfNeeded(false, module, resourceName), access.findClassByName(condition.getTypeName())); } @Override @@ -518,7 +538,7 @@ public void registerNegativeQuery(Module module, String resourceName) { // TODO if pass all gates - remove @Override public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar) { - access.registerReachabilityHandler(e -> addDirectoryResource(module, dir, content, fromJar), access.findClassByName(condition.getTypeName())); + access.registerReachabilityHandler(e -> registerResourceIfNeeded(true, module, dir, content, fromJar), access.findClassByName(condition.getTypeName())); } } From 130eb2ae0330daffd9d6f02d337b9cb88abc2488 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 11 Aug 2023 15:18:53 +0200 Subject: [PATCH 43/78] Resolve conflicts --- .../com/oracle/svm/core/jdk/Resources.java | 22 +++++++++++-------- ...ContentSubstitutedLocalizationSupport.java | 9 +++++--- .../jdk/localization/LocalizationSupport.java | 5 +++-- .../oracle/svm/hosted/ResourcesFeature.java | 13 ++++++----- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 2165af201c39..662acbcb47cb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -48,7 +48,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import com.oracle.svm.core.BuildPhaseProvider; import com.oracle.svm.core.BuildPhaseProvider; import com.oracle.svm.core.MissingRegistrationUtils; import com.oracle.svm.core.SubstrateOptions; @@ -167,20 +166,25 @@ private void updateTimeStamp() { } @Platforms(Platform.HOSTED_ONLY.class) - private static void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar) { - + private void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar, boolean isNegativeQuery) { VMError.guarantee(!BuildPhaseProvider.isAnalysisFinished(), "Trying to add a resource entry after analysis."); Module m = module != null && module.isNamed() ? module : null; if (m != null) { m = RuntimeModuleSupport.instance().getRuntimeModuleForHostedModule(m); } - var resources = singleton().resources; synchronized (resources) { Pair key = createStorageKey(m, resourceName); ResourceStorageEntryBase entry = resources.get(key); + if (isNegativeQuery) { + if (entry == null) { + resources.put(key, NEGATIVE_QUERY_MARKER); + } + return; + } + if (entry == null || entry == NEGATIVE_QUERY_MARKER) { - entry = newEntry == null ? new ResourceStorageEntry(isDirectory, fromJar) : newEntry; updateTimeStamp(); + entry = new ResourceStorageEntry(isDirectory, fromJar); resources.put(key, entry); } else { if (key.getLeft() != null) { @@ -211,12 +215,12 @@ public void registerResource(Module module, String resourceName, InputStream is) @Platforms(Platform.HOSTED_ONLY.class) public void registerResource(Module module, String resourceName, byte[] resourceContent) { - addEntry(module, resourceName, false, resourceContent, true); + addEntry(module, resourceName, false, resourceContent, true, false); } @Platforms(Platform.HOSTED_ONLY.class) public void registerResource(Module module, String resourceName, InputStream is, boolean fromJar) { - addEntry(module, resourceName, false, inputStreamToByteArray(is), fromJar); + addEntry(module, resourceName, false, inputStreamToByteArray(is), fromJar, false); } @Platforms(Platform.HOSTED_ONLY.class) @@ -241,7 +245,7 @@ public void registerDirectoryResource(Module module, String resourceDirName, Str * specified directory, separated with new line delimiter and joined into one string which * is later converted into a byte array and placed into the resources map. */ - addEntry(module, resourceDirName, true, content.getBytes(), fromJar); + addEntry(module, resourceDirName, true, content.getBytes(), fromJar, false); } @Platforms(Platform.HOSTED_ONLY.class) @@ -272,7 +276,7 @@ public void registerNegativeQuery(String resourceName) { @Platforms(Platform.HOSTED_ONLY.class) public void registerNegativeQuery(Module module, String resourceName) { - addEntry(module, resourceName, NEGATIVE_QUERY_MARKER, false, false); + addEntry(module, resourceName, false, null, false, true); } @Platforms(Platform.HOSTED_ONLY.class) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java index 9f918232531e..a43d7f15d982 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java @@ -30,10 +30,12 @@ import java.util.ListResourceBundle; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.ResourceBundle; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ForkJoinPool; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -172,11 +174,11 @@ public boolean isNotIncluded(String bundleName) { } @Override - public void prepareBundle(String bundleName, ResourceBundle bundle, Locale locale) { - super.prepareBundle(bundleName, bundle, locale); + public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule) { + super.prepareBundle(bundleName, bundle, null); /* Initialize ResourceBundle.keySet eagerly */ bundle.keySet(); - this.existingBundles.add(control.toBundleName(bundleName, locale)); + this.existingBundles.add(control.toBundleName(bundleName, bundle.getLocale())); } @Override @@ -184,4 +186,5 @@ public void prepareClassResourceBundle(String basename, Class bundleClass) { super.prepareClassResourceBundle(basename, bundleClass); this.existingBundles.add(bundleClass.getName()); } + } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index e74248dc1ce1..1b905050e403 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -27,6 +27,7 @@ import java.nio.charset.Charset; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.IllformedLocaleException; import java.util.Locale; @@ -112,8 +113,8 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Function> packageToModules = ImageSingletons.lookup(ClassLoaderSupport.class).getPackageToModules(); Set modules = packageToModules.getOrDefault(packageName(bundleName), Collections.emptySet()); - // there should be only one module but we will check all modules where given package - // is found + // there should be only one module, but we will check all modules where given + // package is found for (Module m : modules) { ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 7ab2dffd6dc9..3a690dae798b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -304,9 +304,9 @@ public void addResource(Module module, String resourcePath) { } if (isDirectory) { - Resources.registerDirectoryResource(module, resourcePath, content); + Resources.singleton().registerDirectoryResource(module, resourcePath, content); } else { - Resources.registerResource(module, resourcePath, is, fromJar); + Resources.singleton().registerResource(module, resourcePath, is, fromJar); } try { @@ -391,9 +391,11 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { .map(e -> new CompiledConditionalPattern(e.condition(), makeResourcePattern(e.pattern()))) .collect(Collectors.toSet()); if (MissingRegistrationUtils.throwMissingRegistrationErrors()) { - for (ResourcePattern resourcePattern : includePatterns) { - Resources.singleton().registerIncludePattern(resourcePattern.moduleName, resourcePattern.pattern.pattern()); - } + includePatterns.stream() + .map(pattern -> pattern.compiledPattern) + .forEach(resourcePattern -> { + Resources.singleton().registerIncludePattern(resourcePattern.moduleName, resourcePattern.pattern.pattern()); + }); } ResourcePattern[] excludePatterns = compilePatterns(excludedResourcePatterns); DebugContext debugContext = beforeAnalysisAccess.getDebugContext(); @@ -535,7 +537,6 @@ public void registerNegativeQuery(Module module, String resourceName) { Resources.singleton().registerNegativeQuery(module, resourceName); } - // TODO if pass all gates - remove @Override public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar) { access.registerReachabilityHandler(e -> registerResourceIfNeeded(true, module, dir, content, fromJar), access.findClassByName(condition.getTypeName())); From 18a179746b0b9631f41e9d0170cda4f543d6ba96 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 16 Aug 2023 11:04:59 +0200 Subject: [PATCH 44/78] Remove exceptions that could not be thrown from method signature --- .../BundleContentSubstitutedLocalizationSupport.java | 1 - .../src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java index a43d7f15d982..3543494941d0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java @@ -186,5 +186,4 @@ public void prepareClassResourceBundle(String basename, Class bundleClass) { super.prepareClassResourceBundle(basename, bundleClass); this.existingBundles.add(bundleClass.getName()); } - } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index fc8c85f7a941..543145f3c1c3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -190,7 +190,7 @@ private static String makeDirContent(Set allEntries, String dir) { return String.join(System.lineSeparator(), dirContent); } - private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) throws IOException { + private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) { List matchedDirectoryResources = new ArrayList<>(); Set allEntries = new HashSet<>(); From 08167efe4ce9ed37487f0ecd0c7eb0a78f05d81b Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 16 Aug 2023 11:20:33 +0200 Subject: [PATCH 45/78] Remove unused import from mx file --- substratevm/mx.substratevm/mx_substratevm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 8159e089890b..3c873193198e 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -28,7 +28,7 @@ import tempfile from glob import glob from contextlib import contextmanager -from distutils.dir_util import mkpath, remove_tree # pylint: disable=no-name-in-module +from distutils.dir_util import mkpath # pylint: disable=no-name-in-module from os.path import join, exists, dirname import pipes from argparse import ArgumentParser From 12aa181f033798b5b9c11455c11af0b65732bd5d Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 18 Aug 2023 17:12:48 +0200 Subject: [PATCH 46/78] Revert locale argument for prepareBundle function --- .../BundleContentSubstitutedLocalizationSupport.java | 6 +++--- .../svm/core/jdk/localization/LocalizationSupport.java | 8 ++++---- .../jdk/localization/OptimizedLocalizationSupport.java | 6 +++--- .../svm/hosted/jdk/localization/LocalizationFeature.java | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java index 3543494941d0..10014521a9ce 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java @@ -174,11 +174,11 @@ public boolean isNotIncluded(String bundleName) { } @Override - public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule) { - super.prepareBundle(bundleName, bundle, null); + public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule, Locale locale) { + super.prepareBundle(bundleName, bundle, null, locale); /* Initialize ResourceBundle.keySet eagerly */ bundle.keySet(); - this.existingBundles.add(control.toBundleName(bundleName, bundle.getLocale())); + this.existingBundles.add(control.toBundleName(bundleName, locale)); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index 1b905050e403..b7eda880c015 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -102,12 +102,12 @@ public Map getBundleContentOf(Object bundle) { } @Platforms(Platform.HOSTED_ONLY.class) - public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule) { - if (bundle instanceof PropertyResourceBundle prb) { + public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule, Locale locale) { + if (bundle instanceof PropertyResourceBundle) { String[] bundleNameWithModule = SubstrateUtil.split(bundleName, ":", 2); String resourceName; if (bundleNameWithModule.length < 2) { - resourceName = control.toBundleName(bundleName, prb.getLocale()).replace('.', '/').concat(".properties"); + resourceName = control.toBundleName(bundleName, locale).replace('.', '/').concat(".properties"); // find module based on package name Map> packageToModules = ImageSingletons.lookup(ClassLoaderSupport.class).getPackageToModules(); @@ -124,7 +124,7 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Function module = findModule.apply(bundleNameWithModule[0]); String finalResourceName = resourceName; module.ifPresent(m -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, finalResourceName)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java index 30afc879b2af..15d185dff739 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java @@ -91,7 +91,7 @@ public void prepareClassResourceBundle(String basename, Class bundleClass) { /*- Set the basename and locale to be consistent with JVM lookup process */ bundleNameField.set(bundle, basename); bundleLocaleField.set(bundle, locale); - prepareBundle(basename, bundle, null); + prepareBundle(basename, bundle, null, locale); } catch (ReflectionUtil.ReflectionUtilError | ReflectiveOperationException e) { throw UserError.abort(e, "Failed to instantiated bundle from class %s, reason %s", bundleClass, e.getCause().getMessage()); } @@ -108,9 +108,9 @@ private static Locale extractLocale(Class bundleClass) { @Platforms(Platform.HOSTED_ONLY.class) @Override - public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule) { + public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule, Locale locale) { bundle.keySet(); - this.resourceBundles.put(Pair.create(bundleName, bundle.getLocale()), bundle); + this.resourceBundles.put(Pair.create(bundleName, locale), bundle); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java index 1c7dc43f37b8..7aabef6e0565 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java @@ -643,14 +643,14 @@ private void prepareBundle(String bundleName, ResourceBundle bundle, Locale loca */ for (ResourceBundle cur = bundle; cur != null; cur = SharedSecrets.getJavaUtilResourceBundleAccess().getParent(cur)) { /* Register all bundles with their corresponding locales */ - support.prepareBundle(bundleName, cur, this.imageClassLoader::findModule); + support.prepareBundle(bundleName, cur, this.imageClassLoader::findModule, cur.getLocale()); } /* * Finally, register the requested bundle with requested locale (Requested might be more * specific than the actual bundle locale */ - support.prepareBundle(bundleName, bundle, this.imageClassLoader::findModule); + support.prepareBundle(bundleName, bundle, this.imageClassLoader::findModule, locale); } @Platforms(Platform.HOSTED_ONLY.class) From de5d3075ff723a50a684a4dbe40923c3500ce4f1 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 22 Aug 2023 11:58:36 +0200 Subject: [PATCH 47/78] Add missing throw when using VMError --- .../configure/test/config/ResourceConfigurationTest.java | 6 +++--- .../oracle/svm/configure/config/ResourceConfiguration.java | 2 +- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java index ef28c19bcd99..d4007bbee5e6 100644 --- a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java +++ b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/ResourceConfigurationTest.java @@ -33,15 +33,15 @@ import java.util.List; import java.util.Locale; -import com.oracle.svm.core.util.VMError; import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.junit.Assert; import org.junit.Test; import com.oracle.svm.configure.config.ResourceConfiguration; -import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.core.configure.ResourceConfigurationParser; import com.oracle.svm.core.configure.ResourcesRegistry; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.core.util.json.JsonWriter; public class ResourceConfigurationTest { @@ -96,7 +96,7 @@ public void addResources(ConfigurationCondition condition, String pattern) { @Override public void addResource(Module module, String resourcePath) { - VMError.shouldNotReachHere("Unused function."); + throw VMError.shouldNotReachHere("Unused function."); } @Override diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java index 18cfeded7762..6a561ee133e7 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ResourceConfiguration.java @@ -65,7 +65,7 @@ public void addResources(ConfigurationCondition condition, String pattern) { @Override public void addResource(Module module, String resourcePath) { - VMError.shouldNotReachHere("Unused function."); + throw VMError.shouldNotReachHere("Unused function."); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 3a690dae798b..5d4056630a8e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -176,7 +176,7 @@ public void addResources(ConfigurationCondition condition, String pattern) { try { resourcePatternWorkSet.add(new ConditionalPattern(condition, pattern)); } catch (UnsupportedOperationException e) { - UserError.abort("Resource registration should be performed before beforeAnalysis phase."); + throw UserError.abort("Resource registration should be performed before beforeAnalysis phase."); } } From fb8105dddf4b1c160fd79a100dff1fa25681ab94 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 22 Aug 2023 12:00:59 +0200 Subject: [PATCH 48/78] Replace findModule with Class.class.getModule for java.base scenario --- .../hosted/jdk/localization/CharsetSubstitutionsFeature.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java index e7addb2ac95d..8da31a190df0 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/CharsetSubstitutionsFeature.java @@ -33,7 +33,7 @@ class CharsetSubstitutionsFeature implements InternalFeature { @Override public void beforeAnalysis(BeforeAnalysisAccess access) { - access.registerReachabilityHandler((e) -> RuntimeResourceAccess.addResource(ModuleLayer.boot().findModule("java.base").get(), + access.registerReachabilityHandler((e) -> RuntimeResourceAccess.addResource(Class.class.getModule(), "java/lang/uniName.dat"), access.findClassByName("java.lang.CharacterName")); } } From 69646bce35ee5cea2b4423826349fbb581d5fdf9 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 22 Aug 2023 12:02:00 +0200 Subject: [PATCH 49/78] Remove comments from prepareBundle --- .../oracle/svm/core/jdk/localization/LocalizationSupport.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index b7eda880c015..52a11c0c1ba4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -109,17 +109,13 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Function> packageToModules = ImageSingletons.lookup(ClassLoaderSupport.class).getPackageToModules(); Set modules = packageToModules.getOrDefault(packageName(bundleName), Collections.emptySet()); - // there should be only one module, but we will check all modules where given - // package is found for (Module m : modules) { ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, resourceName); } - // if didn't find resource in any module, we will try with unnamed module if (modules.isEmpty()) { ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(null, resourceName); } From 9c5990191a93069ed8fad1ab7c0c7805b243adca Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 22 Aug 2023 13:40:25 +0200 Subject: [PATCH 50/78] Add null check guard for findModule --- .../svm/core/jdk/localization/LocalizationSupport.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index 52a11c0c1ba4..fa5cf7ae302a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -120,10 +120,12 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Function module = findModule.apply(bundleNameWithModule[0]); - String finalResourceName = resourceName; - module.ifPresent(m -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, finalResourceName)); + if (findModule != null) { + resourceName = control.toBundleName(bundleNameWithModule[1], locale).replace('.', '/').concat(".properties"); + Optional module = findModule.apply(bundleNameWithModule[0]); + String finalResourceName = resourceName; + module.ifPresent(m -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, finalResourceName)); + } } } else { registerRequiredReflectionAndResourcesForBundle(bundleName, Set.of(locale)); From 03854b2223d363e461674981c8d90a6a5e9045ec Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Mon, 2 Oct 2023 16:01:20 +0200 Subject: [PATCH 51/78] Remove redudant functions and improve duplicates checking --- .../com/oracle/svm/core/jdk/Resources.java | 43 +++------------ .../oracle/svm/hosted/ResourcesFeature.java | 55 ++++++------------- 2 files changed, 25 insertions(+), 73 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 662acbcb47cb..d48590c4fad8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.Date; import java.util.Enumeration; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -165,6 +166,8 @@ private void updateTimeStamp() { } } + private HashSet set = new HashSet(); + @Platforms(Platform.HOSTED_ONLY.class) private void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar, boolean isNegativeQuery) { VMError.guarantee(!BuildPhaseProvider.isAnalysisFinished(), "Trying to add a resource entry after analysis."); @@ -194,25 +197,15 @@ private void addEntry(Module module, String resourceName, boolean isDirectory, b } } + if (set.contains(resourceName)) { + System.out.println("RESOURCE: " + resourceName + " in module: " + module + " from jar: " + fromJar); + System.out.println("IS NEGATIVE QUERY: " + isNegativeQuery); + } entry.getData().add(data); + set.add(resourceName); } } - @Platforms(Platform.HOSTED_ONLY.class) - public static void registerResource(String resourceName, InputStream is) { - singleton().registerResource(null, resourceName, is, true); - } - - @Platforms(Platform.HOSTED_ONLY.class) - public void registerResource(String resourceName, InputStream is, boolean fromJar) { - registerResource(null, resourceName, is, fromJar); - } - - @Platforms(Platform.HOSTED_ONLY.class) - public void registerResource(Module module, String resourceName, InputStream is) { - registerResource(module, resourceName, is, true); - } - @Platforms(Platform.HOSTED_ONLY.class) public void registerResource(Module module, String resourceName, byte[] resourceContent) { addEntry(module, resourceName, false, resourceContent, true, false); @@ -223,21 +216,6 @@ public void registerResource(Module module, String resourceName, InputStream is, addEntry(module, resourceName, false, inputStreamToByteArray(is), fromJar, false); } - @Platforms(Platform.HOSTED_ONLY.class) - public void registerDirectoryResource(String resourceDirName, String content) { - registerDirectoryResource(null, resourceDirName, content, true); - } - - @Platforms(Platform.HOSTED_ONLY.class) - public void registerDirectoryResource(String resourceDirName, String content, boolean fromJar) { - registerDirectoryResource(null, resourceDirName, content, fromJar); - } - - @Platforms(Platform.HOSTED_ONLY.class) - public void registerDirectoryResource(Module module, String resourceDirName, String content) { - registerDirectoryResource(module, resourceDirName, content, true); - } - @Platforms(Platform.HOSTED_ONLY.class) public void registerDirectoryResource(Module module, String resourceDirName, String content, boolean fromJar) { /* @@ -248,11 +226,6 @@ public void registerDirectoryResource(Module module, String resourceDirName, Str addEntry(module, resourceDirName, true, content.getBytes(), fromJar, false); } - @Platforms(Platform.HOSTED_ONLY.class) - public void registerIOException(String resourceName, IOException e, boolean linkAtBuildTime) { - registerIOException(null, resourceName, e, linkAtBuildTime); - } - @Platforms(Platform.HOSTED_ONLY.class) public void registerIOException(Module module, String resourceName, IOException e, boolean linkAtBuildTime) { if (linkAtBuildTime) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 5d4056630a8e..e4311380b7a6 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -304,7 +304,7 @@ public void addResource(Module module, String resourcePath) { } if (isDirectory) { - Resources.singleton().registerDirectoryResource(module, resourcePath, content); + Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); } else { Resources.singleton().registerResource(module, resourcePath, is, fromJar); } @@ -495,36 +495,38 @@ public List isIncluded(Module module, String resourceNam private final Set alreadyAddedResources = new HashSet<>(); - private void registerResourceIfNeeded(boolean isDirectory, Object... arguments) { - Module module = (Module) arguments[0]; - String resourceName = (String) arguments[1]; + public void registerResourceIfNeeded(Module module, String resourceName) { // we only do this if we are on the classPath - if ((module == null || !module.isNamed()) && !alreadyAddedResources.contains(resourceName)) { - if (isDirectory) { - String content = (String) arguments[2]; - boolean fromJar = (boolean) arguments[3]; - addDirectoryResource(module, resourceName, content, fromJar); - } else { + if ((module == null || !module.isNamed())) { + if (!alreadyAddedResources.contains(resourceName)) { ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourceName); + alreadyAddedResources.add(resourceName); } - - alreadyAddedResources.add(resourceName); + } else { + // we should always try to register module entries (checked later in addEntries) + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourceName); } + } @Override public void addResource(Module module, String resourceName, InputStream resourceStream, boolean fromJar) { - registerResource(debugContext, module, resourceName, resourceStream, fromJar); + registerResourceIfNeeded(module, resourceName); } @Override public void addResourceConditionally(Module module, String resourceName, ConfigurationCondition condition) { - access.registerReachabilityHandler(e -> registerResourceIfNeeded(false, module, resourceName), access.findClassByName(condition.getTypeName())); + access.registerReachabilityHandler(e -> registerResourceIfNeeded(module, resourceName), access.findClassByName(condition.getTypeName())); } @Override public void addDirectoryResource(Module module, String dir, String content, boolean fromJar) { - registerDirectoryResource(debugContext, module, dir, content, fromJar); + registerResourceIfNeeded(module, dir); + } + + @Override + public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar) { + access.registerReachabilityHandler(e -> registerResourceIfNeeded(module, dir), access.findClassByName(condition.getTypeName())); } @Override @@ -536,11 +538,6 @@ public void registerIOException(Module module, String resourceName, IOException public void registerNegativeQuery(Module module, String resourceName) { Resources.singleton().registerNegativeQuery(module, resourceName); } - - @Override - public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar) { - access.registerReachabilityHandler(e -> registerResourceIfNeeded(true, module, dir, content, fromJar), access.findClassByName(condition.getTypeName())); - } } private ResourcePattern[] compilePatterns(Set patterns) { @@ -599,24 +596,6 @@ public void beforeCompilation(BeforeCompilationAccess access) { } } - @SuppressWarnings("try") - private static void registerResource(DebugContext debugContext, Module module, String resourceName, InputStream resourceStream, boolean fromJar) { - try (DebugContext.Scope s = debugContext.scope("registerResource")) { - String moduleNamePrefix = module == null ? "" : module.getName() + ":"; - debugContext.log(DebugContext.VERBOSE_LEVEL, "ResourcesFeature: registerResource: %s%s", moduleNamePrefix, resourceName); - Resources.singleton().registerResource(module, resourceName, resourceStream, fromJar); - } - } - - @SuppressWarnings("try") - private static void registerDirectoryResource(DebugContext debugContext, Module module, String dir, String content, boolean fromJar) { - try (DebugContext.Scope s = debugContext.scope("registerResource")) { - String moduleNamePrefix = module == null ? "" : module.getName() + ":"; - debugContext.log(DebugContext.VERBOSE_LEVEL, "ResourcesFeature: registerResource: %s%s", moduleNamePrefix, dir); - Resources.singleton().registerDirectoryResource(module, dir, content, fromJar); - } - } - @Override public void registerInvocationPlugins(Providers providers, SnippetReflectionProvider snippetReflection, GraphBuilderConfiguration.Plugins plugins, ParsingReason reason) { if (!reason.duringAnalysis() || reason == ParsingReason.JITCompilation) { From 9b9ed1feda8320fa78a883edee50d9e723516b79 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 6 Oct 2023 13:34:22 +0200 Subject: [PATCH 52/78] Register negative query directories --- .../src/com/oracle/svm/core/jdk/Resources.java | 10 +--------- .../svm/hosted/ClassLoaderSupportImpl.java | 4 ++++ .../com/oracle/svm/hosted/ResourcesFeature.java | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index d48590c4fad8..3b5d615f4273 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -35,7 +35,6 @@ import java.util.Collections; import java.util.Date; import java.util.Enumeration; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -107,7 +106,7 @@ public record ModuleResourcePair(String module, String resource) { * specified in the configuration, but we do not want to throw directly (for example when we try * to check all the modules for a resource). */ - private static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase(); + public static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase(); /** * Embedding a resource into an image is counted as a modification. Since all resources are @@ -166,8 +165,6 @@ private void updateTimeStamp() { } } - private HashSet set = new HashSet(); - @Platforms(Platform.HOSTED_ONLY.class) private void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar, boolean isNegativeQuery) { VMError.guarantee(!BuildPhaseProvider.isAnalysisFinished(), "Trying to add a resource entry after analysis."); @@ -197,12 +194,7 @@ private void addEntry(Module module, String resourceName, boolean isDirectory, b } } - if (set.contains(resourceName)) { - System.out.println("RESOURCE: " + resourceName + " in module: " + module + " from jar: " + fromJar); - System.out.println("IS NEGATIVE QUERY: " + isNegativeQuery); - } entry.getData().add(data); - set.add(resourceName); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 543145f3c1c3..d9629d0926e5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -216,6 +216,10 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea matchedDirectoryResources.add(new ConditionalResource(condition, relativeFilePath)); } + if (conditions.isEmpty()) { + collector.registerNegativeQuery(null, relativeFilePath); + } + try (Stream pathStream = Files.list(entry)) { Stream filtered = pathStream; if (ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT.equals(entry)) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index e4311380b7a6..9d8015b75cee 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -94,6 +94,7 @@ import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributesView; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystem; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystemProvider; +import com.oracle.svm.core.jdk.resources.ResourceStorageEntryBase; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.util.UserError; @@ -583,6 +584,22 @@ boolean moduleNameMatches(String resourceContainerModuleName) { @Override public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; + var entryIter = Resources.singleton().getResourceStorage().getEntries(); + while (entryIter.advance()) { + var key = entryIter.getKey(); + ResourceStorageEntryBase val = entryIter.getValue(); + String valStr; + if (val.isException()) { + valStr = val.getException().getClass().getName(); + } else if (val == Resources.NEGATIVE_QUERY_MARKER) { + valStr = "NEGATIVE_QUERY_MARKER"; + } else if (val == Resources.MISSING_METADATA_MARKER) { + valStr = "MISSING_METADATA_MARKER"; + } else { + valStr = "" + val.getData().stream().map(bytes -> bytes.length).reduce((a, b) -> a + b).get(); + } + System.out.println("RESOURCE: " + key.getRight() + ": " + valStr); + } } @Override From 167a72749cf3afa26fc5bb43d964fe740755c27a Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Mon, 9 Oct 2023 17:06:15 +0200 Subject: [PATCH 53/78] Fix package export from module --- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 9d8015b75cee..0096b24bdbe0 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -260,10 +260,9 @@ public void addResource(Module module, String resourcePath) { try { String resourcePackage = jdk.internal.module.Resources.toPackageName(resourcePath); if (!resourcePackage.isEmpty()) { - if (!module.getPackages().contains(resourcePackage)) { - return; + if (module.getPackages().contains(resourcePackage)) { + ModuleSupport.accessModuleByClass(ModuleSupport.Access.EXPORT, ResourcesFeature.class, module, resourcePackage); } - ModuleSupport.accessModuleByClass(ModuleSupport.Access.OPEN, ResourcesFeature.class, module, resourcePackage); } is = module.getResourceAsStream(resourcePath); From 620f8137d411510231c9b3ec8703d8ae7f74f077 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 10 Oct 2023 12:06:55 +0200 Subject: [PATCH 54/78] Remove redudant parameters from resource registration functions --- .../oracle/svm/core/ClassLoaderSupport.java | 5 +- .../svm/hosted/ClassLoaderSupportImpl.java | 43 ++-- .../oracle/svm/hosted/ResourcesFeature.java | 187 +++++++++--------- 3 files changed, 113 insertions(+), 122 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java index 85851ab218b3..937dd94f0773 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java @@ -25,7 +25,6 @@ package com.oracle.svm.core; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.util.List; import java.util.Locale; @@ -56,11 +55,11 @@ public boolean isNativeImageClassLoader(ClassLoader classLoader) { public interface ResourceCollector { List isIncluded(Module module, String resourceName, URI resourceURI); - void addResource(Module module, String resourceName, InputStream resourceStream, boolean fromJar); + void addResource(Module module, String resourceName); void addResourceConditionally(Module module, String resourceName, ConfigurationCondition condition); - void addDirectoryResource(Module module, String dir, String content, boolean fromJar); + void addDirectoryResource(Module module, String dir); void registerNegativeQuery(Module module, String resourceName); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index d9629d0926e5..7b23c73ebad5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -147,7 +147,7 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso String resName = entry.resourceName(); if (resName.endsWith("/")) { if (ConfigurationCondition.isAlwaysTrue(condition)) { - resourceCollector.addDirectoryResource(info.module, resName, "", false); + resourceCollector.addDirectoryResource(info.module, resName); } else { resourceCollector.addResourceConditionally(info.module, resName, condition); } @@ -160,14 +160,11 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso resourceCollector.registerNegativeQuery(info.module, resName); continue; } - try (InputStream is = content.get()) { - if (ConfigurationCondition.isAlwaysTrue(condition)) { - resourceCollector.addResource(info.module, resName, is, false); - } else { - resourceCollector.addResourceConditionally(info.module, resName, condition); - } - } catch (IOException resourceException) { - resourceCollector.registerIOException(info.module, resName, resourceException, LinkAtBuildTimeSupport.singleton().moduleLinkAtBuildTime(info.module.getName())); + + if (ConfigurationCondition.isAlwaysTrue(condition)) { + resourceCollector.addResource(info.module, resName); + } else { + resourceCollector.addResourceConditionally(info.module, resName, condition); } } @@ -231,14 +228,10 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea } } else { for (ConfigurationCondition condition : conditions) { - try (InputStream is = Files.newInputStream(entry)) { - if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addResource(null, relativeFilePath, is, false); - } else { - collector.addResourceConditionally(null, relativeFilePath, condition); - } - } catch (IOException resourceException) { - collector.registerIOException(null, relativeFilePath, resourceException, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(relativeFilePath)); + if (ConfigurationCondition.isAlwaysTrue(condition)) { + collector.addResource(null, relativeFilePath); + } else { + collector.addResourceConditionally(null, relativeFilePath, condition); } } } @@ -249,7 +242,7 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea String dir = entry.resourceName(); String contentName = makeDirContent(allEntries, dir); if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addDirectoryResource(null, dir, contentName, false); + collector.addDirectoryResource(null, dir); } else { collector.addDirectoryResourceConditionally(null, dir, condition, contentName, false); } @@ -268,7 +261,7 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i for (ConfigurationCondition condition : conditions) { // Register the directory with empty content to preserve Java behavior if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addDirectoryResource(null, dirName, "", true); + collector.addDirectoryResource(null, dirName); } else { collector.addDirectoryResourceConditionally(null, dirName, condition, "", true); } @@ -276,14 +269,10 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i } else { List conditions = shouldIncludeEntry(null, collector, entry.getName(), jarPath.toUri(), includeAll); for (ConfigurationCondition condition : conditions) { - try (InputStream is = jf.getInputStream(entry)) { - if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addResource(null, entry.getName(), is, true); - } else { - collector.addResourceConditionally(null, entry.getName(), condition); - } - } catch (IOException resourceException) { - collector.registerIOException(null, entry.getName(), resourceException, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(entry.getName())); + if (ConfigurationCondition.isAlwaysTrue(condition)) { + collector.addResource(null, entry.getName()); + } else { + collector.addResourceConditionally(null, entry.getName(), condition); } } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 0096b24bdbe0..d982b78dae0f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -46,12 +46,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.Enumeration; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -163,6 +163,7 @@ private record CompiledConditionalPattern(ConfigurationCondition condition, Reso private class ResourcesRegistryImpl extends ConditionalConfigurationRegistry implements ResourcesRegistry { private final ConfigurationTypeResolver configurationTypeResolver; + private final Set alreadyAddedResources = new HashSet<>(); ResourcesRegistryImpl(ConfigurationTypeResolver configurationTypeResolver) { this.configurationTypeResolver = configurationTypeResolver; @@ -181,81 +182,17 @@ public void addResources(ConfigurationCondition condition, String pattern) { } } - private String urlToJarPath(URL url) { - try { - return ((JarURLConnection) url.openConnection()).getJarFileURL().getFile(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private boolean resourceIsDirectory(URL url, boolean fromJar, String resourcePath) throws IOException, URISyntaxException { - if (fromJar) { - try (JarFile jf = new JarFile(urlToJarPath(url))) { - return jf.getEntry(resourcePath).isDirectory(); - } - } else { - return Files.isDirectory(Path.of(url.toURI())); - } - } - - private String getDirectoryContent(String path, boolean fromJar) throws IOException { - List content = new ArrayList<>(); - if (fromJar) { - try (JarFile jf = new JarFile(urlToJarPath(URI.create(path).toURL()))) { - String pathSeparator = FileSystems.getDefault().getSeparator(); - String directoryPath = path.split("!")[1]; - - // we are removing leading slash because jar entry names don't start with slash - if (directoryPath.startsWith(pathSeparator)) { - directoryPath = directoryPath.substring(1); - } - - Enumeration entries = jf.entries(); - while (entries.hasMoreElements()) { - String entry = entries.nextElement().getName(); - if (entry.startsWith(directoryPath)) { - String contentEntry = entry.substring(directoryPath.length()); - - // remove the leading slash - if (contentEntry.startsWith(pathSeparator)) { - contentEntry = contentEntry.substring(1); - } - - // prevent adding empty strings as a content - if (!contentEntry.isEmpty()) { - // get top level content only - int firstSlash = contentEntry.indexOf(pathSeparator); - if (firstSlash != -1) { - content.add(contentEntry.substring(0, firstSlash)); - } else { - content.add(contentEntry); - } - } - } - } - - } - } else { - try (Stream contentStream = Files.list(Path.of(path))) { - content = new ArrayList<>(contentStream - .map(Path::getFileName) - .map(Path::toString) - .toList()); - } - } - - content.sort(Comparator.naturalOrder()); - return String.join(System.lineSeparator(), content); - } - + /* Adds single resource defined with its module and name */ @Override public void addResource(Module module, String resourcePath) { + if (!shouldRegisterResource(module, resourcePath)) { + return; + } + InputStream is; boolean fromJar; boolean isDirectory; String content = ""; - if (module != null && module.isNamed()) { try { String resourcePackage = jdk.internal.module.Resources.toPackageName(resourcePath); @@ -356,6 +293,88 @@ public void addResourceBundles(ConfigurationCondition condition, String basename } registerConditionalConfiguration(condition, () -> ImageSingletons.lookup(LocalizationFeature.class).prepareBundle(basename, locales)); } + + public boolean shouldRegisterResource(Module module, String resourceName) { + // we only do this if we are on the classPath + if ((module == null || !module.isNamed())) { + if (!alreadyAddedResources.contains(resourceName)) { + alreadyAddedResources.add(resourceName); + return true; + } else { + return false; + } + } else { + // we should always try to register module entries (checked later in addEntries) + return true; + } + } + + private String urlToJarPath(URL url) { + try { + return ((JarURLConnection) url.openConnection()).getJarFileURL().getFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private boolean resourceIsDirectory(URL url, boolean fromJar, String resourcePath) throws IOException, URISyntaxException { + if (fromJar) { + try (JarFile jf = new JarFile(urlToJarPath(url))) { + return jf.getEntry(resourcePath).isDirectory(); + } + } else { + return Files.isDirectory(Path.of(url.toURI())); + } + } + + private String getDirectoryContent(String path, boolean fromJar) throws IOException { + Set content = new TreeSet<>(); + if (fromJar) { + try (JarFile jf = new JarFile(urlToJarPath(URI.create(path).toURL()))) { + String pathSeparator = FileSystems.getDefault().getSeparator(); + String directoryPath = path.split("!")[1]; + + // we are removing leading slash because jar entry names don't start with slash + if (directoryPath.startsWith(pathSeparator)) { + directoryPath = directoryPath.substring(1); + } + + Enumeration entries = jf.entries(); + while (entries.hasMoreElements()) { + String entry = entries.nextElement().getName(); + if (entry.startsWith(directoryPath)) { + String contentEntry = entry.substring(directoryPath.length()); + + // remove the leading slash + if (contentEntry.startsWith(pathSeparator)) { + contentEntry = contentEntry.substring(1); + } + + // prevent adding empty strings as a content + if (!contentEntry.isEmpty()) { + // get top level content only + int firstSlash = contentEntry.indexOf(pathSeparator); + if (firstSlash != -1) { + content.add(contentEntry.substring(0, firstSlash)); + } else { + content.add(contentEntry); + } + } + } + } + + } + } else { + try (Stream contentStream = Files.list(Path.of(path))) { + content = new TreeSet<>(contentStream + .map(Path::getFileName) + .map(Path::toString) + .toList()); + } + } + + return String.join(System.lineSeparator(), content); + } } @Override @@ -493,40 +512,24 @@ public List isIncluded(Module module, String resourceNam return conditions; } - private final Set alreadyAddedResources = new HashSet<>(); - - public void registerResourceIfNeeded(Module module, String resourceName) { - // we only do this if we are on the classPath - if ((module == null || !module.isNamed())) { - if (!alreadyAddedResources.contains(resourceName)) { - ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourceName); - alreadyAddedResources.add(resourceName); - } - } else { - // we should always try to register module entries (checked later in addEntries) - ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourceName); - } - - } - @Override - public void addResource(Module module, String resourceName, InputStream resourceStream, boolean fromJar) { - registerResourceIfNeeded(module, resourceName); + public void addResource(Module module, String resourceName) { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, resourceName); } @Override public void addResourceConditionally(Module module, String resourceName, ConfigurationCondition condition) { - access.registerReachabilityHandler(e -> registerResourceIfNeeded(module, resourceName), access.findClassByName(condition.getTypeName())); + access.registerReachabilityHandler(e -> addResource(module, resourceName), access.findClassByName(condition.getTypeName())); } @Override - public void addDirectoryResource(Module module, String dir, String content, boolean fromJar) { - registerResourceIfNeeded(module, dir); + public void addDirectoryResource(Module module, String dir) { + ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, dir); } @Override public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar) { - access.registerReachabilityHandler(e -> registerResourceIfNeeded(module, dir), access.findClassByName(condition.getTypeName())); + access.registerReachabilityHandler(e -> addDirectoryResource(module, dir), access.findClassByName(condition.getTypeName())); } @Override From 2e769954c70500c599d25fae93f96007046fbd18 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 10 Oct 2023 13:51:36 +0200 Subject: [PATCH 55/78] Remove sufficient prints --- .../com/oracle/svm/hosted/ResourcesFeature.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index d982b78dae0f..7c1f41f0e7bf 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -94,7 +94,6 @@ import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributesView; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystem; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystemProvider; -import com.oracle.svm.core.jdk.resources.ResourceStorageEntryBase; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.util.UserError; @@ -586,22 +585,6 @@ boolean moduleNameMatches(String resourceContainerModuleName) { @Override public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; - var entryIter = Resources.singleton().getResourceStorage().getEntries(); - while (entryIter.advance()) { - var key = entryIter.getKey(); - ResourceStorageEntryBase val = entryIter.getValue(); - String valStr; - if (val.isException()) { - valStr = val.getException().getClass().getName(); - } else if (val == Resources.NEGATIVE_QUERY_MARKER) { - valStr = "NEGATIVE_QUERY_MARKER"; - } else if (val == Resources.MISSING_METADATA_MARKER) { - valStr = "MISSING_METADATA_MARKER"; - } else { - valStr = "" + val.getData().stream().map(bytes -> bytes.length).reduce((a, b) -> a + b).get(); - } - System.out.println("RESOURCE: " + key.getRight() + ": " + valStr); - } } @Override From 1635c2b5f46f32e681b270fd7692b3c792dffa73 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 11 Oct 2023 10:52:09 +0200 Subject: [PATCH 56/78] Cover case when same resource name can occur in multiple bundles --- .../oracle/svm/hosted/ResourcesFeature.java | 79 ++++++++++++------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 7c1f41f0e7bf..6c9c58225396 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -211,44 +211,69 @@ public void addResource(Module module, String resourcePath) { // we should ignore if user failed to provide resource return; } - } else { - URL url = imageClassLoader.getClassLoader().getResource(resourcePath); - if (url == null) { - // we should ignore if user failed to provide resource + + if (is == null) { return; } + if (isDirectory) { + Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); + } else { + Resources.singleton().registerResource(module, resourcePath, is, fromJar); + } + try { - is = url.openStream(); - fromJar = url.getProtocol().equalsIgnoreCase("jar"); - isDirectory = resourceIsDirectory(url, fromJar, resourcePath); - // if directory is from jar content should remain empty (same as in scanJar - // function from ClassLoaderSupportImpl) - if (isDirectory) { - content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); - } + is.close(); } catch (IOException e) { + throw new RuntimeException(e); + } + + } else { + Enumeration urls; + try { + urls = imageClassLoader.getClassLoader().getResources(resourcePath); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (!urls.hasMoreElements()) { // we should ignore if user failed to provide resource return; - } catch (URISyntaxException e) { - throw new RuntimeException(e); } - } - if (is == null) { - return; - } + while (urls.hasMoreElements()) { + URL url = urls.nextElement(); + try { + is = url.openStream(); + fromJar = url.getProtocol().equalsIgnoreCase("jar"); + isDirectory = resourceIsDirectory(url, fromJar, resourcePath); + // if directory is from jar content should remain empty (same as in scanJar + // function from ClassLoaderSupportImpl) + if (isDirectory) { + content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); + } + } catch (IOException e) { + // we should ignore if user failed to provide resource + return; + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } - if (isDirectory) { - Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); - } else { - Resources.singleton().registerResource(module, resourcePath, is, fromJar); - } + if (is == null) { + return; + } - try { - is.close(); - } catch (IOException e) { - throw new RuntimeException(e); + if (isDirectory) { + Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); + } else { + Resources.singleton().registerResource(module, resourcePath, is, fromJar); + } + + try { + is.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } } From 52d9e72a94340c0b7930beb641cd07a113120be2 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 11 Oct 2023 16:50:28 +0200 Subject: [PATCH 57/78] Restructure code for better readability --- .../oracle/svm/hosted/ResourcesFeature.java | 166 +++++++++--------- 1 file changed, 80 insertions(+), 86 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 6c9c58225396..ecc611896372 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -188,92 +188,10 @@ public void addResource(Module module, String resourcePath) { return; } - InputStream is; - boolean fromJar; - boolean isDirectory; - String content = ""; if (module != null && module.isNamed()) { - try { - String resourcePackage = jdk.internal.module.Resources.toPackageName(resourcePath); - if (!resourcePackage.isEmpty()) { - if (module.getPackages().contains(resourcePackage)) { - ModuleSupport.accessModuleByClass(ModuleSupport.Access.EXPORT, ResourcesFeature.class, module, resourcePackage); - } - } - - is = module.getResourceAsStream(resourcePath); - fromJar = false; - isDirectory = new File(resourcePath).isDirectory(); - if (isDirectory) { - content = getDirectoryContent(resourcePath, false); - } - } catch (IOException e) { - // we should ignore if user failed to provide resource - return; - } - - if (is == null) { - return; - } - - if (isDirectory) { - Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); - } else { - Resources.singleton().registerResource(module, resourcePath, is, fromJar); - } - - try { - is.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - + processResourceFromModule(module, resourcePath); } else { - Enumeration urls; - try { - urls = imageClassLoader.getClassLoader().getResources(resourcePath); - } catch (IOException e) { - throw new RuntimeException(e); - } - if (!urls.hasMoreElements()) { - // we should ignore if user failed to provide resource - return; - } - - while (urls.hasMoreElements()) { - URL url = urls.nextElement(); - try { - is = url.openStream(); - fromJar = url.getProtocol().equalsIgnoreCase("jar"); - isDirectory = resourceIsDirectory(url, fromJar, resourcePath); - // if directory is from jar content should remain empty (same as in scanJar - // function from ClassLoaderSupportImpl) - if (isDirectory) { - content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); - } - } catch (IOException e) { - // we should ignore if user failed to provide resource - return; - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - - if (is == null) { - return; - } - - if (isDirectory) { - Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); - } else { - Resources.singleton().registerResource(module, resourcePath, is, fromJar); - } - - try { - is.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + processResourceFromClasspath(module, resourcePath); } } @@ -319,8 +237,8 @@ public void addResourceBundles(ConfigurationCondition condition, String basename } public boolean shouldRegisterResource(Module module, String resourceName) { - // we only do this if we are on the classPath if ((module == null || !module.isNamed())) { + // we only do this if we are on the classPath if (!alreadyAddedResources.contains(resourceName)) { alreadyAddedResources.add(resourceName); return true; @@ -328,11 +246,87 @@ public boolean shouldRegisterResource(Module module, String resourceName) { return false; } } else { - // we should always try to register module entries (checked later in addEntries) + // always try to register module entries (duplicates checked later in addEntries) return true; } } + private void processResourceFromModule(Module module, String resourcePath) { + try { + String resourcePackage = jdk.internal.module.Resources.toPackageName(resourcePath); + if (!resourcePackage.isEmpty()) { + // if processing resource package, make sure that module exports that package + if (module.getPackages().contains(resourcePackage)) { + ModuleSupport.accessModuleByClass(ModuleSupport.Access.EXPORT, ResourcesFeature.class, module, resourcePackage); + } + } + + InputStream is = module.getResourceAsStream(resourcePath); + boolean isDirectory = new File(resourcePath).isDirectory(); + String content = ""; + if (isDirectory) { + content = getDirectoryContent(resourcePath, false); + } + + registerResource(module, resourcePath, isDirectory, false, is, content); + } catch (IOException e) { + // we should ignore if user failed to provide resource + } + } + + private void processResourceFromClasspath(Module module, String resourcePath) { + Enumeration urls; + try { + /* + * There is an edge case where same resource name can be present in multiple jars + * (different resources), so we are collecting all resources with given name in all + * jars on classpath + */ + urls = imageClassLoader.getClassLoader().getResources(resourcePath); + } catch (IOException e) { + throw new RuntimeException(e); + } + + while (urls.hasMoreElements()) { + URL url = urls.nextElement(); + try { + InputStream is = url.openStream(); + boolean fromJar = url.getProtocol().equalsIgnoreCase("jar"); + boolean isDirectory = resourceIsDirectory(url, fromJar, resourcePath); + String content = ""; + if (isDirectory) { + content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); + } + + registerResource(module, resourcePath, isDirectory, fromJar, is, content); + } catch (IOException e) { + // we should ignore if user failed to provide resource + return; + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + private void registerResource(Module module, String resourcePath, boolean isDirectory, boolean fromJar, InputStream is, String content) { + if (is == null) { + return; + } + + if (isDirectory) { + Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); + } else { + Resources.singleton().registerResource(module, resourcePath, is, fromJar); + } + + try { + is.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /* Util functions for resource attributes calculations */ private String urlToJarPath(URL url) { try { return ((JarURLConnection) url.openConnection()).getJarFileURL().getFile(); From 0f90072de68d4d80543373f378afd7659fff86ad Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 11 Oct 2023 16:55:48 +0200 Subject: [PATCH 58/78] Add synchronization for alreadyAddedResources check --- .../oracle/svm/hosted/ResourcesFeature.java | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index ecc611896372..fc8a0036f1b1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -94,6 +94,7 @@ import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributesView; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystem; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystemProvider; +import com.oracle.svm.core.jdk.resources.ResourceStorageEntryBase; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.util.UserError; @@ -238,12 +239,16 @@ public void addResourceBundles(ConfigurationCondition condition, String basename public boolean shouldRegisterResource(Module module, String resourceName) { if ((module == null || !module.isNamed())) { - // we only do this if we are on the classPath - if (!alreadyAddedResources.contains(resourceName)) { - alreadyAddedResources.add(resourceName); - return true; - } else { - return false; + // addResources can be called from multiple threads so this should be synchronized + synchronized (alreadyAddedResources) { + // we only do this if we are on the classPath + if (!alreadyAddedResources.contains(resourceName)) { + alreadyAddedResources.add(resourceName); + return true; + } else { + return false; + } + } } else { // always try to register module entries (duplicates checked later in addEntries) @@ -604,6 +609,22 @@ boolean moduleNameMatches(String resourceContainerModuleName) { @Override public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; + var entryIter = Resources.singleton().getResourceStorage().getEntries(); + while (entryIter.advance()) { + var key = entryIter.getKey(); + ResourceStorageEntryBase val = entryIter.getValue(); + String valStr; + if (val.isException()) { + valStr = val.getException().getClass().getName(); + } else if (val == Resources.NEGATIVE_QUERY_MARKER) { + valStr = "NEGATIVE_QUERY_MARKER"; + } else if (val == Resources.MISSING_METADATA_MARKER) { + valStr = "MISSING_METADATA_MARKER"; + } else { + valStr = "" + val.getData().stream().map(bytes -> bytes.length).reduce((a, b) -> a + b).get(); + } + System.out.println("RESOURCE: " + key.getRight() + ": " + valStr); + } } @Override From 17f84cc8d1462e12067d6d2d4f42c06bb045337b Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 11 Oct 2023 17:28:49 +0200 Subject: [PATCH 59/78] Cleanup code --- .../oracle/svm/core/ClassLoaderSupport.java | 5 - .../com/oracle/svm/core/jdk/Resources.java | 2 +- .../svm/hosted/ClassLoaderSupportImpl.java | 106 +++++------------- .../oracle/svm/hosted/ResourcesFeature.java | 29 ----- 4 files changed, 27 insertions(+), 115 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java index 937dd94f0773..aa53efe3871a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java @@ -59,14 +59,9 @@ public interface ResourceCollector { void addResourceConditionally(Module module, String resourceName, ConfigurationCondition condition); - void addDirectoryResource(Module module, String dir); - void registerNegativeQuery(Module module, String resourceName); void registerIOException(Module module, String resourceName, IOException e, boolean linkAtBuildTime); - - void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar); - } public abstract void collectResources(ResourceCollector resourceCollector); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index 3b5d615f4273..f7ec625c6ad2 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -106,7 +106,7 @@ public record ModuleResourcePair(String module, String resource) { * specified in the configuration, but we do not want to throw directly (for example when we try * to check all the modules for a resource). */ - public static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase(); + private static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase(); /** * Embedding a resource into an image is counted as a modification. Since all resources are diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 7b23c73ebad5..3b61ab842bee 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -39,7 +39,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; @@ -116,7 +115,7 @@ public void collectResources(ResourceCollector resourceCollector) { .forEach(lookup -> collectResourceFromModule(resourceCollector, lookup)); /* Collect remaining resources from classpath */ - for (Path classpathFile : classLoaderSupport.classpath()) { + classLoaderSupport.classpath().stream().parallel().forEach(classpathFile -> { boolean includeAll = classLoaderSupport.getJavaPathsToInclude().contains(classpathFile); try { if (Files.isDirectory(classpathFile)) { @@ -127,13 +126,13 @@ public void collectResources(ResourceCollector resourceCollector) { } catch (IOException ex) { throw UserError.abort("Unable to handle classpath element '%s'. Make sure that all classpath entries are either directories or valid jar files.", classpathFile); } - } + }); } private void collectResourceFromModule(ResourceCollector resourceCollector, ResourceLookupInfo info) { ModuleReference moduleReference = info.resolvedModule.reference(); try (ModuleReader moduleReader = moduleReference.open()) { - var includeAll = classLoaderSupport.getJavaModuleNamesToInclude().contains(info.resolvedModule().name()); + boolean includeAll = classLoaderSupport.getJavaModuleNamesToInclude().contains(info.resolvedModule().name()); List resourcesFound = new ArrayList<>(); moduleReader.list().forEach(resourceName -> { List conditions = shouldIncludeEntry(info.module, resourceCollector, resourceName, moduleReference.location().orElse(null), includeAll); @@ -146,11 +145,7 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso ConfigurationCondition condition = entry.condition(); String resName = entry.resourceName(); if (resName.endsWith("/")) { - if (ConfigurationCondition.isAlwaysTrue(condition)) { - resourceCollector.addDirectoryResource(info.module, resName); - } else { - resourceCollector.addResourceConditionally(info.module, resName, condition); - } + includeResource(resourceCollector, info.module, resName, condition); continue; } @@ -161,11 +156,7 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso continue; } - if (ConfigurationCondition.isAlwaysTrue(condition)) { - resourceCollector.addResource(info.module, resName); - } else { - resourceCollector.addResourceConditionally(info.module, resName, condition); - } + includeResource(resourceCollector, info.module, resName, condition); } } catch (IOException e) { @@ -173,24 +164,7 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso } } - private static String makeDirContent(Set allEntries, String dir) { - List dirContent = new ArrayList<>(); - for (String entry : allEntries) { - int last = entry.lastIndexOf(RESOURCES_INTERNAL_PATH_SEPARATOR); - String parentDirectory = last == -1 ? "" : entry.substring(0, last); - if (parentDirectory.equals(dir)) { - dirContent.add(entry.substring(last + 1)); - } - } - - dirContent.sort(Comparator.naturalOrder()); - return String.join(System.lineSeparator(), dirContent); - } - private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) { - List matchedDirectoryResources = new ArrayList<>(); - Set allEntries = new HashSet<>(); - ArrayDeque queue = new ArrayDeque<>(); queue.push(root); while (!queue.isEmpty()) { @@ -198,21 +172,18 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea /* Resources always use / as the separator, as do our resource inclusion patterns */ String relativeFilePath; - List conditions; if (entry != root) { relativeFilePath = root.relativize(entry).toString().replace(File.separatorChar, RESOURCES_INTERNAL_PATH_SEPARATOR); - conditions = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); - allEntries.add(relativeFilePath); } else { relativeFilePath = String.valueOf(RESOURCES_INTERNAL_PATH_SEPARATOR); - conditions = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); } - if (Files.isDirectory(entry)) { - for (ConfigurationCondition condition : conditions) { - matchedDirectoryResources.add(new ConditionalResource(condition, relativeFilePath)); - } + List conditions = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); + for (ConfigurationCondition condition : conditions) { + includeResource(collector, null, relativeFilePath, condition); + } + if (Files.isDirectory(entry)) { if (conditions.isEmpty()) { collector.registerNegativeQuery(null, relativeFilePath); } @@ -226,27 +197,8 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea } catch (IOException resourceException) { collector.registerIOException(null, relativeFilePath, resourceException, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(relativeFilePath)); } - } else { - for (ConfigurationCondition condition : conditions) { - if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addResource(null, relativeFilePath); - } else { - collector.addResourceConditionally(null, relativeFilePath, condition); - } - } } } - - matchedDirectoryResources.forEach(entry -> { - ConfigurationCondition condition = entry.condition(); - String dir = entry.resourceName(); - String contentName = makeDirContent(allEntries, dir); - if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addDirectoryResource(null, dir); - } else { - collector.addDirectoryResourceConditionally(null, dir, condition, contentName, false); - } - }); } private static void scanJar(Path jarPath, ResourceCollector collector, boolean includeAll) throws IOException { @@ -254,39 +206,33 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i Enumeration entries = jf.entries(); while (entries.hasMoreElements()) { JarEntry entry = entries.nextElement(); - URI uri = jarPath.toUri(); + String entryName = entry.getName(); if (entry.isDirectory()) { - String dirName = entry.getName().substring(0, entry.getName().length() - 1); - List conditions = shouldIncludeEntry(null, collector, dirName, jarPath.toUri(), includeAll); - for (ConfigurationCondition condition : conditions) { - // Register the directory with empty content to preserve Java behavior - if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addDirectoryResource(null, dirName); - } else { - collector.addDirectoryResourceConditionally(null, dirName, condition, "", true); - } - } - } else { - List conditions = shouldIncludeEntry(null, collector, entry.getName(), jarPath.toUri(), includeAll); - for (ConfigurationCondition condition : conditions) { - if (ConfigurationCondition.isAlwaysTrue(condition)) { - collector.addResource(null, entry.getName()); - } else { - collector.addResourceConditionally(null, entry.getName(), condition); - } - } + entryName = entryName.substring(0, entry.getName().length() - 1); + } + + List conditions = shouldIncludeEntry(null, collector, entryName, jarPath.toUri(), includeAll); + for (ConfigurationCondition condition : conditions) { + includeResource(collector, null, entryName, condition); } } } } + private static void includeResource(ResourceCollector collector, Module module, String name, ConfigurationCondition condition) { + if (ConfigurationCondition.isAlwaysTrue(condition)) { + collector.addResource(module, name); + } else { + collector.addResourceConditionally(module, name, condition); + } + } + private static List shouldIncludeEntry(Module module, ResourceCollector collector, String fileName, URI uri, boolean includeAll) { - List conditions = collector.isIncluded(module, fileName, uri); if (includeAll && !(fileName.endsWith(".class") || fileName.endsWith(".jar"))) { return Collections.singletonList(ConfigurationCondition.alwaysTrue()); } - return conditions; + return collector.isIncluded(module, fileName, uri); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index fc8a0036f1b1..2e45a088a2ca 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -94,7 +94,6 @@ import com.oracle.svm.core.jdk.resources.NativeImageResourceFileAttributesView; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystem; import com.oracle.svm.core.jdk.resources.NativeImageResourceFileSystemProvider; -import com.oracle.svm.core.jdk.resources.ResourceStorageEntryBase; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.util.UserError; @@ -458,7 +457,6 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { } private static final class ResourceCollectorImpl implements ResourceCollector { - private final DebugContext debugContext; private final Set includePatterns; private final ResourcePattern[] excludePatterns; @@ -471,7 +469,6 @@ private static final class ResourceCollectorImpl implements ResourceCollector { ScheduledExecutorService scheduledExecutor; private ResourceCollectorImpl(DebugContext debugContext, Set includePatterns, ResourcePattern[] excludePatterns) { - this.debugContext = debugContext; this.includePatterns = includePatterns; this.excludePatterns = excludePatterns; @@ -545,16 +542,6 @@ public void addResourceConditionally(Module module, String resourceName, Configu access.registerReachabilityHandler(e -> addResource(module, resourceName), access.findClassByName(condition.getTypeName())); } - @Override - public void addDirectoryResource(Module module, String dir) { - ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(module, dir); - } - - @Override - public void addDirectoryResourceConditionally(Module module, String dir, ConfigurationCondition condition, String content, boolean fromJar) { - access.registerReachabilityHandler(e -> addDirectoryResource(module, dir), access.findClassByName(condition.getTypeName())); - } - @Override public void registerIOException(Module module, String resourceName, IOException e, boolean linkAtBuildTime) { Resources.singleton().registerIOException(module, resourceName, e, linkAtBuildTime); @@ -609,22 +596,6 @@ boolean moduleNameMatches(String resourceContainerModuleName) { @Override public void afterAnalysis(AfterAnalysisAccess access) { sealed = true; - var entryIter = Resources.singleton().getResourceStorage().getEntries(); - while (entryIter.advance()) { - var key = entryIter.getKey(); - ResourceStorageEntryBase val = entryIter.getValue(); - String valStr; - if (val.isException()) { - valStr = val.getException().getClass().getName(); - } else if (val == Resources.NEGATIVE_QUERY_MARKER) { - valStr = "NEGATIVE_QUERY_MARKER"; - } else if (val == Resources.MISSING_METADATA_MARKER) { - valStr = "MISSING_METADATA_MARKER"; - } else { - valStr = "" + val.getData().stream().map(bytes -> bytes.length).reduce((a, b) -> a + b).get(); - } - System.out.println("RESOURCE: " + key.getRight() + ": " + valStr); - } } @Override From 45c27ed0c351ee4277591fba9c345c65763ec3ee Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 12 Oct 2023 18:20:39 +0200 Subject: [PATCH 60/78] Convert ResorucePattern class to record --- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 2e45a088a2ca..625907df378c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -571,14 +571,7 @@ private ResourcePattern makeResourcePattern(String rawPattern) { } } - private static final class ResourcePattern { - final String moduleName; - final Pattern pattern; - - private ResourcePattern(String moduleName, Pattern pattern) { - this.moduleName = moduleName; - this.pattern = pattern; - } + private record ResourcePattern(String moduleName, Pattern pattern) { boolean moduleNameMatches(String resourceContainerModuleName) { if (moduleName == null) { From 30e5c8613ca5a79ef9575cfebda9239673d70168 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 12 Oct 2023 18:22:49 +0200 Subject: [PATCH 61/78] Remove debugg context since it is not used anymore --- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 625907df378c..0f1de2be0283 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -439,8 +439,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { }); } ResourcePattern[] excludePatterns = compilePatterns(excludedResourcePatterns); - DebugContext debugContext = beforeAnalysisAccess.getDebugContext(); - ResourceCollectorImpl collector = new ResourceCollectorImpl(debugContext, includePatterns, excludePatterns); + ResourceCollectorImpl collector = new ResourceCollectorImpl(includePatterns, excludePatterns); try { collector.prepareProgressReporter(); ImageSingletons.lookup(ClassLoaderSupport.class).collectResources(collector); @@ -459,7 +458,6 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { private static final class ResourceCollectorImpl implements ResourceCollector { private final Set includePatterns; private final ResourcePattern[] excludePatterns; - private static final int WATCHDOG_RESET_AFTER_EVERY_N_RESOURCES = 1000; private static final int WATCHDOG_INITIAL_WARNING_AFTER_N_SECONDS = 60; private static final int WATCHDOG_WARNING_AFTER_EVERY_N_SECONDS = 20; @@ -468,7 +466,7 @@ private static final class ResourceCollectorImpl implements ResourceCollector { private volatile String currentlyProcessedEntry; ScheduledExecutorService scheduledExecutor; - private ResourceCollectorImpl(DebugContext debugContext, Set includePatterns, ResourcePattern[] excludePatterns) { + private ResourceCollectorImpl(Set includePatterns, ResourcePattern[] excludePatterns) { this.includePatterns = includePatterns; this.excludePatterns = excludePatterns; From c898d381f92c4c4a52c9a2db5a09497a1cefa61d Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Thu, 12 Oct 2023 18:33:51 +0200 Subject: [PATCH 62/78] Add explanation for shouldRegisterResource function --- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 0f1de2be0283..2cb1473355b2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -236,6 +236,13 @@ public void addResourceBundles(ConfigurationCondition condition, String basename registerConditionalConfiguration(condition, () -> ImageSingletons.lookup(LocalizationFeature.class).prepareBundle(basename, locales)); } + /* + * It is possible that one resource can be registered under different conditions + * (typeReachable). In some cases, few conditions will be satisfied, and we will try to + * register same resource for each satisfied condition. This function will check if the + * resource is already registered and prevent multiple registrations of same resource under + * different conditions + */ public boolean shouldRegisterResource(Module module, String resourceName) { if ((module == null || !module.isNamed())) { // addResources can be called from multiple threads so this should be synchronized From 0853a1c2be6048a0b6137ef6ba79816d05876e4b Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 13 Oct 2023 10:57:43 +0200 Subject: [PATCH 63/78] Add function that is never used just to pass gates --- .../src/com/oracle/svm/core/jdk/Resources.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index f7ec625c6ad2..c067c9a92506 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -198,6 +198,11 @@ private void addEntry(Module module, String resourceName, boolean isDirectory, b } } + @Platforms(Platform.HOSTED_ONLY.class) + public static void registerResource(String resourceName, InputStream is) { + singleton().registerResource(null, resourceName, is, true); + } + @Platforms(Platform.HOSTED_ONLY.class) public void registerResource(Module module, String resourceName, byte[] resourceContent) { addEntry(module, resourceName, false, resourceContent, true, false); From 56bdd98bbfb3780cc18460522b52136e4781b8ad Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 17 Oct 2023 14:35:46 +0200 Subject: [PATCH 64/78] Replace module export with module open access --- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 2cb1473355b2..50a83a082983 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -268,7 +268,8 @@ private void processResourceFromModule(Module module, String resourcePath) { if (!resourcePackage.isEmpty()) { // if processing resource package, make sure that module exports that package if (module.getPackages().contains(resourcePackage)) { - ModuleSupport.accessModuleByClass(ModuleSupport.Access.EXPORT, ResourcesFeature.class, module, resourcePackage); + // Use Access.OPEN to find ALL resources within resource package + ModuleSupport.accessModuleByClass(ModuleSupport.Access.OPEN, ResourcesFeature.class, module, resourcePackage); } } @@ -446,7 +447,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { }); } ResourcePattern[] excludePatterns = compilePatterns(excludedResourcePatterns); - ResourceCollectorImpl collector = new ResourceCollectorImpl(includePatterns, excludePatterns); + ResourceCollectorImpl collector = new ResourceCollectorImpl(includePatterns, excludePatterns, beforeAnalysisAccess); try { collector.prepareProgressReporter(); ImageSingletons.lookup(ClassLoaderSupport.class).collectResources(collector); @@ -468,15 +469,17 @@ private static final class ResourceCollectorImpl implements ResourceCollector { private static final int WATCHDOG_RESET_AFTER_EVERY_N_RESOURCES = 1000; private static final int WATCHDOG_INITIAL_WARNING_AFTER_N_SECONDS = 60; private static final int WATCHDOG_WARNING_AFTER_EVERY_N_SECONDS = 20; + private final FeatureImpl.BeforeAnalysisAccessImpl access; private final LongAdder reachedResourceEntries; private boolean initialReport; private volatile String currentlyProcessedEntry; ScheduledExecutorService scheduledExecutor; - private ResourceCollectorImpl(Set includePatterns, ResourcePattern[] excludePatterns) { + private ResourceCollectorImpl(Set includePatterns, ResourcePattern[] excludePatterns, FeatureImpl.BeforeAnalysisAccessImpl access) { this.includePatterns = includePatterns; this.excludePatterns = excludePatterns; + this.access = access; this.reachedResourceEntries = new LongAdder(); this.initialReport = true; this.currentlyProcessedEntry = null; From e1807e002d6f66de1045998d7fccb1d9939c0af7 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 20 Oct 2023 15:38:45 +0200 Subject: [PATCH 65/78] Update imports according to changes on master --- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 50a83a082983..546a03e133a1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -104,6 +104,15 @@ import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; +import jdk.compiler.graal.api.replacements.SnippetReflectionProvider; +import jdk.compiler.graal.nodes.ValueNode; +import jdk.compiler.graal.nodes.graphbuilderconf.GraphBuilderConfiguration; +import jdk.compiler.graal.nodes.graphbuilderconf.GraphBuilderContext; +import jdk.compiler.graal.nodes.graphbuilderconf.InvocationPlugin; +import jdk.compiler.graal.nodes.graphbuilderconf.InvocationPlugins; +import jdk.compiler.graal.options.Option; +import jdk.compiler.graal.options.OptionType; +import jdk.compiler.graal.phases.util.Providers; import jdk.vm.ci.meta.ResolvedJavaMethod; /** From ec0913fa5862c7961d5fed14d1192e09d848f568 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 24 Oct 2023 13:46:06 +0200 Subject: [PATCH 66/78] Inline function and remove passing null for findModule function --- ...undleContentSubstitutedLocalizationSupport.java | 3 ++- .../localization/OptimizedLocalizationSupport.java | 14 ++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java index 10014521a9ce..1b01e1839001 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java @@ -51,6 +51,7 @@ import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; +import jdk.compiler.graal.debug.GraalError; import sun.util.resources.OpenListResourceBundle; import sun.util.resources.ParallelListResourceBundle; @@ -175,7 +176,7 @@ public boolean isNotIncluded(String bundleName) { @Override public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule, Locale locale) { - super.prepareBundle(bundleName, bundle, null, locale); + super.prepareBundle(bundleName, bundle, findModule, locale); /* Initialize ResourceBundle.keySet eagerly */ bundle.keySet(); this.existingBundles.add(control.toBundleName(bundleName, locale)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java index 15d185dff739..b24464c5f603 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java @@ -30,15 +30,11 @@ import java.util.Locale; import java.util.Map; import java.util.MissingResourceException; -import java.util.Optional; import java.util.ResourceBundle; import java.util.Set; -import java.util.function.Function; import java.util.spi.LocaleServiceProvider; import org.graalvm.collections.Pair; -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.ReflectionUtil; @@ -91,7 +87,8 @@ public void prepareClassResourceBundle(String basename, Class bundleClass) { /*- Set the basename and locale to be consistent with JVM lookup process */ bundleNameField.set(bundle, basename); bundleLocaleField.set(bundle, locale); - prepareBundle(basename, bundle, null, locale); + bundle.keySet(); + this.resourceBundles.put(Pair.create(basename, locale), bundle); } catch (ReflectionUtil.ReflectionUtilError | ReflectiveOperationException e) { throw UserError.abort(e, "Failed to instantiated bundle from class %s, reason %s", bundleClass, e.getCause().getMessage()); } @@ -106,13 +103,6 @@ private static Locale extractLocale(Class bundleClass) { return parseLocaleFromTag(name.substring(split + 1)); } - @Platforms(Platform.HOSTED_ONLY.class) - @Override - public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule, Locale locale) { - bundle.keySet(); - this.resourceBundles.put(Pair.create(bundleName, locale), bundle); - } - @Override public boolean shouldSubstituteLoadLookup(String className) { /*- All bundles are stored in the image heap as objects, no need to keep the content around */ From 9de65c8a0e3f97cb7c349a436eef5d84b07ccbce Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 24 Oct 2023 14:35:32 +0200 Subject: [PATCH 67/78] Avoid using synchronized block all the time --- .../svm/hosted/ClassLoaderSupportImpl.java | 4 ++-- .../com/oracle/svm/hosted/ResourcesFeature.java | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 3b61ab842bee..ce50ed0fda14 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -93,7 +93,7 @@ protected boolean isNativeImageClassLoaderImpl(ClassLoader loader) { return false; } - public record ResourceLookupInfo(ResolvedModule resolvedModule, Module module) { + private record ResourceLookupInfo(ResolvedModule resolvedModule, Module module) { } private static Stream extractModuleLookupData(ModuleLayer layer) { @@ -110,8 +110,8 @@ private static Stream extractModuleLookupData(ModuleLayer la public void collectResources(ResourceCollector resourceCollector) { /* Collect resources from modules */ NativeImageClassLoaderSupport.allLayers(classLoaderSupport.moduleLayerForImageBuild).stream() - .parallel() .flatMap(ClassLoaderSupportImpl::extractModuleLookupData) + .parallel() .forEach(lookup -> collectResourceFromModule(resourceCollector, lookup)); /* Collect remaining resources from classpath */ diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 546a03e133a1..b6d4e8593db8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -253,20 +253,29 @@ public void addResourceBundles(ConfigurationCondition condition, String basename * different conditions */ public boolean shouldRegisterResource(Module module, String resourceName) { + // we only do this if we are on the classPath if ((module == null || !module.isNamed())) { - // addResources can be called from multiple threads so this should be synchronized + /* + * This check is not thread safe! If the resource is not added yet, maybe some other + * thread is attempting to do it, so we have to perform same check again in + * synchronized block (in that case). Anyway this check will cut the case when we + * are sure that resource is added (so we don't need to enter synchronized block) + */ + if (alreadyAddedResources.contains(resourceName)) { + return false; + } + + // addResources can be called from multiple threads synchronized (alreadyAddedResources) { - // we only do this if we are on the classPath if (!alreadyAddedResources.contains(resourceName)) { alreadyAddedResources.add(resourceName); return true; } else { return false; } - } } else { - // always try to register module entries (duplicates checked later in addEntries) + // always try to register module entries (we will check duplicates in addEntries) return true; } } From 2da282fd7a716a62dd8fc0ba40266bd8f96941df Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 24 Oct 2023 14:56:02 +0200 Subject: [PATCH 68/78] Replace multiple calls of string transformation with function --- .../jdk/localization/LocalizationSupport.java | 15 ++++++++++----- .../src/com/oracle/svm/util/StringUtil.java | 8 ++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index fa5cf7ae302a..47c52c0f8bfd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -25,6 +25,9 @@ package com.oracle.svm.core.jdk.localization; +import static com.oracle.svm.util.StringUtil.toDotSeparated; +import static com.oracle.svm.util.StringUtil.toSlashSeparated; + import java.nio.charset.Charset; import java.util.Collection; import java.util.Collections; @@ -53,6 +56,8 @@ import com.oracle.svm.core.jdk.Resources; import com.oracle.svm.core.util.VMError; +import jdk.compiler.graal.debug.GraalError; + /** * Holder for localization information that is computed during image generation and used at run * time. @@ -107,7 +112,7 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Function> packageToModules = ImageSingletons.lookup(ClassLoaderSupport.class).getPackageToModules(); Set modules = packageToModules.getOrDefault(packageName(bundleName), Collections.emptySet()); @@ -121,7 +126,7 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Function module = findModule.apply(bundleNameWithModule[0]); String finalResourceName = resourceName; module.ifPresent(m -> ImageSingletons.lookup(RuntimeResourceSupport.class).addResource(m, finalResourceName)); @@ -136,7 +141,7 @@ public void prepareBundle(String bundleName, ResourceBundle bundle, Function strings) { return joinSingleQuoted(strings.toArray(new String[strings.size()])); } + + public static String toSlashSeparated(String string) { + return string.replace('.', '/'); + } + + public static String toDotSeparated(String string) { + return string.replace('/', '.'); + } } From 4b9c4d7839a1b5e63c81a6a46fbbd70bb1d14852 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 24 Oct 2023 15:36:01 +0200 Subject: [PATCH 69/78] Extract directory registration handling form registerResoruce function --- .../oracle/svm/hosted/ResourcesFeature.java | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index b6d4e8593db8..bf411eb6d07f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -27,7 +27,6 @@ import static com.oracle.svm.core.jdk.Resources.RESOURCES_INTERNAL_PATH_SEPARATOR; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; @@ -292,13 +291,13 @@ private void processResourceFromModule(Module module, String resourcePath) { } InputStream is = module.getResourceAsStream(resourcePath); - boolean isDirectory = new File(resourcePath).isDirectory(); - String content = ""; + boolean isDirectory = Files.isDirectory(Path.of(resourcePath)); if (isDirectory) { - content = getDirectoryContent(resourcePath, false); + String content = getDirectoryContent(resourcePath, false); + Resources.singleton().registerDirectoryResource(module, resourcePath, content, false); + } else { + registerResource(module, resourcePath, false, is); } - - registerResource(module, resourcePath, isDirectory, false, is, content); } catch (IOException e) { // we should ignore if user failed to provide resource } @@ -314,7 +313,7 @@ private void processResourceFromClasspath(Module module, String resourcePath) { */ urls = imageClassLoader.getClassLoader().getResources(resourcePath); } catch (IOException e) { - throw new RuntimeException(e); + throw VMError.shouldNotReachHere("getResources for resourcePath " + resourcePath + " failed", e); } while (urls.hasMoreElements()) { @@ -323,31 +322,27 @@ private void processResourceFromClasspath(Module module, String resourcePath) { InputStream is = url.openStream(); boolean fromJar = url.getProtocol().equalsIgnoreCase("jar"); boolean isDirectory = resourceIsDirectory(url, fromJar, resourcePath); - String content = ""; if (isDirectory) { - content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); + String content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); + Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); + } else { + registerResource(module, resourcePath, fromJar, is); } - - registerResource(module, resourcePath, isDirectory, fromJar, is, content); } catch (IOException e) { // we should ignore if user failed to provide resource return; } catch (URISyntaxException e) { - throw new RuntimeException(e); + throw VMError.shouldNotReachHere("resourceIsDirectory for resourcePath " + resourcePath + " failed", e); } } } - private void registerResource(Module module, String resourcePath, boolean isDirectory, boolean fromJar, InputStream is, String content) { + private void registerResource(Module module, String resourcePath, boolean fromJar, InputStream is) { if (is == null) { return; } - if (isDirectory) { - Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); - } else { - Resources.singleton().registerResource(module, resourcePath, is, fromJar); - } + Resources.singleton().registerResource(module, resourcePath, is, fromJar); try { is.close(); @@ -525,8 +520,6 @@ private void shutDownProgressReporter() { @Override public List isIncluded(Module module, String resourceName, URI resource) { - // Possibly we can have multiple conditions for one resource - List conditions = new ArrayList<>(); this.currentlyProcessedEntry = resource.getScheme().equals("jrt") ? (resource + "/" + resourceName) : resource.toString(); this.reachedResourceEntries.increment(); @@ -542,10 +535,12 @@ public List isIncluded(Module module, String resourceNam continue; } if (rp.pattern.matcher(resourceName).matches() || rp.pattern.matcher(relativePathWithTrailingSlash).matches()) { - return conditions; // returns empty list + return List.of(); // nothing should match excluded resource } } + // Possibly we can have multiple conditions for one resource + List conditions = new ArrayList<>(); for (CompiledConditionalPattern rp : includePatterns) { if (!rp.compiledPattern().moduleNameMatches(moduleName)) { continue; From 2d30af2a2f71ef8f3ef9a71e5c3824c264bf5739 Mon Sep 17 00:00:00 2001 From: Peter Hofer Date: Tue, 24 Oct 2023 14:36:59 +0200 Subject: [PATCH 70/78] Support access-filter-file for native-image-configure. --- .../command/ConfigurationGenerateCommand.java | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/command/ConfigurationGenerateCommand.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/command/ConfigurationGenerateCommand.java index 5f6e146f6d31..a42c37f372ad 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/command/ConfigurationGenerateCommand.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/command/ConfigurationGenerateCommand.java @@ -146,6 +146,7 @@ protected static void generate(Iterator argumentsIterator, boolean accep boolean builtinCallerFilter = true; boolean builtinHeuristicFilter = true; List callerFilters = new ArrayList<>(); + List accessFilters = new ArrayList<>(); ConfigurationFileCollection omittedCollection = new ConfigurationFileCollection(); ConfigurationFileCollection inputCollection = new ConfigurationFileCollection(); @@ -220,6 +221,9 @@ protected static void generate(Iterator argumentsIterator, boolean accep case "--caller-filter-file": callerFilters.add(requirePathUri(option, value)); break; + case "--access-filter-file": + accessFilters.add(requirePathUri(option, value)); + break; case "--": if (acceptTraceFileArgs) { argumentsIterator.forEachRemaining(arg -> traceInputs.add(Paths.get(arg).toUri())); @@ -248,15 +252,12 @@ protected static void generate(Iterator argumentsIterator, boolean accep callersFilterHierarchyFilterNode = AccessAdvisor.copyBuiltinCallerFilterTree(); callersFilter = new ComplexFilter(callersFilterHierarchyFilterNode); } - for (URI uri : callerFilters) { - try { - FilterConfigurationParser parser = new FilterConfigurationParser(callersFilter); - parser.parseAndRegister(uri); - } catch (Exception e) { - throw new ConfigurationUsageException("Cannot parse filter file " + uri + ": " + e); - } - } - callersFilter.getHierarchyFilterNode().removeRedundantNodes(); + parseFilterFiles(callersFilter, callerFilters); + } + ComplexFilter accessFilter = null; + if (!accessFilters.isEmpty()) { + accessFilter = new ComplexFilter(AccessAdvisor.copyBuiltinAccessFilterTree()); + parseFilterFiles(accessFilter, accessFilters); } ConfigurationSet configurationSet; @@ -287,6 +288,9 @@ protected static void generate(Iterator argumentsIterator, boolean accep if (callersFilter != null) { advisor.setCallerFilterTree(callersFilter); } + if (accessFilter != null) { + advisor.setAccessFilterTree(accessFilter); + } TraceProcessor processor = new TraceProcessor(advisor); for (URI uri : traceInputs) { @@ -331,6 +335,17 @@ protected static void generate(Iterator argumentsIterator, boolean accep } } + private static void parseFilterFiles(ComplexFilter filter, List filterFiles) { + for (URI uri : filterFiles) { + try { + new FilterConfigurationParser(filter).parseAndRegister(uri); + } catch (Exception e) { + throw new ConfigurationUsageException("Cannot parse filter file " + uri + ": " + e); + } + } + filter.getHierarchyFilterNode().removeRedundantNodes(); + } + @SuppressWarnings("fallthrough") private static void failIfAgentLockFilesPresent(ConfigurationFileCollection... collections) { Set paths = null; From f54f22cf4dc98787aab755619bb87111b9b54d3e Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 24 Oct 2023 18:02:59 +0200 Subject: [PATCH 71/78] Rebase and fix imports --- ...ContentSubstitutedLocalizationSupport.java | 3 +- .../jdk/localization/LocalizationSupport.java | 3 +- .../oracle/svm/hosted/ResourcesFeature.java | 28 ++++++------------- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java index 1b01e1839001..a178d797270b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/BundleContentSubstitutedLocalizationSupport.java @@ -39,7 +39,6 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import jdk.graal.compiler.debug.GraalError; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -51,7 +50,7 @@ import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; -import jdk.compiler.graal.debug.GraalError; +import jdk.graal.compiler.debug.GraalError; import sun.util.resources.OpenListResourceBundle; import sun.util.resources.ParallelListResourceBundle; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java index 47c52c0f8bfd..a036d9c5520a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/LocalizationSupport.java @@ -42,7 +42,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import jdk.graal.compiler.debug.GraalError; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -56,7 +55,7 @@ import com.oracle.svm.core.jdk.Resources; import com.oracle.svm.core.util.VMError; -import jdk.compiler.graal.debug.GraalError; +import jdk.graal.compiler.debug.GraalError; /** * Holder for localization information that is computed during image generation and used at run diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index bf411eb6d07f..8cc36bcf973e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -62,16 +62,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; -import jdk.graal.compiler.debug.DebugContext; -import jdk.graal.compiler.nodes.ValueNode; -import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; -import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; -import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin; -import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins; -import jdk.graal.compiler.options.Option; -import jdk.graal.compiler.options.OptionType; -import jdk.graal.compiler.phases.util.Providers; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; import org.graalvm.nativeimage.impl.ConfigurationCondition; @@ -103,15 +93,15 @@ import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; -import jdk.compiler.graal.api.replacements.SnippetReflectionProvider; -import jdk.compiler.graal.nodes.ValueNode; -import jdk.compiler.graal.nodes.graphbuilderconf.GraphBuilderConfiguration; -import jdk.compiler.graal.nodes.graphbuilderconf.GraphBuilderContext; -import jdk.compiler.graal.nodes.graphbuilderconf.InvocationPlugin; -import jdk.compiler.graal.nodes.graphbuilderconf.InvocationPlugins; -import jdk.compiler.graal.options.Option; -import jdk.compiler.graal.options.OptionType; -import jdk.compiler.graal.phases.util.Providers; +import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; +import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin; +import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionType; +import jdk.graal.compiler.phases.util.Providers; import jdk.vm.ci.meta.ResolvedJavaMethod; /** From 9f793ce233a5c07de40228df0ed35dba19ba0bcc Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Tue, 24 Oct 2023 19:42:31 +0200 Subject: [PATCH 72/78] Rename includeAll variable to includeCurrent --- .../svm/hosted/ClassLoaderSupportImpl.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index ce50ed0fda14..433b0680c4b1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -116,12 +116,12 @@ public void collectResources(ResourceCollector resourceCollector) { /* Collect remaining resources from classpath */ classLoaderSupport.classpath().stream().parallel().forEach(classpathFile -> { - boolean includeAll = classLoaderSupport.getJavaPathsToInclude().contains(classpathFile); + boolean includeCurrent = classLoaderSupport.getJavaPathsToInclude().contains(classpathFile); try { if (Files.isDirectory(classpathFile)) { - scanDirectory(classpathFile, resourceCollector, includeAll); + scanDirectory(classpathFile, resourceCollector, includeCurrent); } else if (ClasspathUtils.isJar(classpathFile)) { - scanJar(classpathFile, resourceCollector, includeAll); + scanJar(classpathFile, resourceCollector, includeCurrent); } } catch (IOException ex) { throw UserError.abort("Unable to handle classpath element '%s'. Make sure that all classpath entries are either directories or valid jar files.", classpathFile); @@ -132,10 +132,10 @@ public void collectResources(ResourceCollector resourceCollector) { private void collectResourceFromModule(ResourceCollector resourceCollector, ResourceLookupInfo info) { ModuleReference moduleReference = info.resolvedModule.reference(); try (ModuleReader moduleReader = moduleReference.open()) { - boolean includeAll = classLoaderSupport.getJavaModuleNamesToInclude().contains(info.resolvedModule().name()); + boolean includeCurrent = classLoaderSupport.getJavaModuleNamesToInclude().contains(info.resolvedModule().name()); List resourcesFound = new ArrayList<>(); moduleReader.list().forEach(resourceName -> { - List conditions = shouldIncludeEntry(info.module, resourceCollector, resourceName, moduleReference.location().orElse(null), includeAll); + List conditions = shouldIncludeEntry(info.module, resourceCollector, resourceName, moduleReference.location().orElse(null), includeCurrent); for (ConfigurationCondition condition : conditions) { resourcesFound.add(new ConditionalResource(condition, resourceName)); } @@ -164,7 +164,7 @@ private void collectResourceFromModule(ResourceCollector resourceCollector, Reso } } - private static void scanDirectory(Path root, ResourceCollector collector, boolean includeAll) { + private static void scanDirectory(Path root, ResourceCollector collector, boolean includeCurrent) { ArrayDeque queue = new ArrayDeque<>(); queue.push(root); while (!queue.isEmpty()) { @@ -178,7 +178,7 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea relativeFilePath = String.valueOf(RESOURCES_INTERNAL_PATH_SEPARATOR); } - List conditions = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeAll); + List conditions = shouldIncludeEntry(null, collector, relativeFilePath, Path.of(relativeFilePath).toUri(), includeCurrent); for (ConfigurationCondition condition : conditions) { includeResource(collector, null, relativeFilePath, condition); } @@ -201,7 +201,7 @@ private static void scanDirectory(Path root, ResourceCollector collector, boolea } } - private static void scanJar(Path jarPath, ResourceCollector collector, boolean includeAll) throws IOException { + private static void scanJar(Path jarPath, ResourceCollector collector, boolean includeCurrent) throws IOException { try (JarFile jf = new JarFile(jarPath.toFile())) { Enumeration entries = jf.entries(); while (entries.hasMoreElements()) { @@ -211,7 +211,7 @@ private static void scanJar(Path jarPath, ResourceCollector collector, boolean i entryName = entryName.substring(0, entry.getName().length() - 1); } - List conditions = shouldIncludeEntry(null, collector, entryName, jarPath.toUri(), includeAll); + List conditions = shouldIncludeEntry(null, collector, entryName, jarPath.toUri(), includeCurrent); for (ConfigurationCondition condition : conditions) { includeResource(collector, null, entryName, condition); } @@ -227,8 +227,8 @@ private static void includeResource(ResourceCollector collector, Module module, } } - private static List shouldIncludeEntry(Module module, ResourceCollector collector, String fileName, URI uri, boolean includeAll) { - if (includeAll && !(fileName.endsWith(".class") || fileName.endsWith(".jar"))) { + private static List shouldIncludeEntry(Module module, ResourceCollector collector, String fileName, URI uri, boolean includeCurrent) { + if (includeCurrent && !(fileName.endsWith(".class") || fileName.endsWith(".jar"))) { return Collections.singletonList(ConfigurationCondition.alwaysTrue()); } From e006dd45360379ca6e28e90808d0aa45c1e64bd5 Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 25 Oct 2023 10:55:19 +0200 Subject: [PATCH 73/78] Use Resources#registerIOException instead of ignoring exception --- .../src/com/oracle/svm/core/jdk/Resources.java | 4 ++-- .../src/com/oracle/svm/hosted/ResourcesFeature.java | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java index c067c9a92506..069e34e89dfe 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java @@ -227,9 +227,9 @@ public void registerDirectoryResource(Module module, String resourceDirName, Str public void registerIOException(Module module, String resourceName, IOException e, boolean linkAtBuildTime) { if (linkAtBuildTime) { if (SubstrateOptions.ThrowLinkAtBuildTimeIOExceptions.getValue()) { - throw new RuntimeException("Resource " + resourceName + " from module " + module.getName() + " produced an IOException.", e); + throw new RuntimeException("Resource " + resourceName + " from module " + moduleName(module) + " produced an IOException.", e); } else { - LogUtils.warning("Resource " + resourceName + " from module " + module.getName() + " produced the following IOException: " + e.getClass().getTypeName() + ": " + e.getMessage()); + LogUtils.warning("Resource " + resourceName + " from module " + moduleName(module) + " produced the following IOException: " + e.getClass().getTypeName() + ": " + e.getMessage()); } } Pair key = createStorageKey(module, resourceName); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index 8cc36bcf973e..f46a8f9c3d77 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -189,7 +189,7 @@ public void addResource(Module module, String resourcePath) { if (module != null && module.isNamed()) { processResourceFromModule(module, resourcePath); } else { - processResourceFromClasspath(module, resourcePath); + processResourceFromClasspath(resourcePath); } } @@ -289,11 +289,11 @@ private void processResourceFromModule(Module module, String resourcePath) { registerResource(module, resourcePath, false, is); } } catch (IOException e) { - // we should ignore if user failed to provide resource + Resources.singleton().registerIOException(module, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); } } - private void processResourceFromClasspath(Module module, String resourcePath) { + private void processResourceFromClasspath(String resourcePath) { Enumeration urls; try { /* @@ -314,12 +314,12 @@ private void processResourceFromClasspath(Module module, String resourcePath) { boolean isDirectory = resourceIsDirectory(url, fromJar, resourcePath); if (isDirectory) { String content = getDirectoryContent(fromJar ? url.toString() : Paths.get(url.toURI()).toString(), fromJar); - Resources.singleton().registerDirectoryResource(module, resourcePath, content, fromJar); + Resources.singleton().registerDirectoryResource(null, resourcePath, content, fromJar); } else { - registerResource(module, resourcePath, fromJar, is); + registerResource(null, resourcePath, fromJar, is); } } catch (IOException e) { - // we should ignore if user failed to provide resource + Resources.singleton().registerIOException(null, resourcePath, e, LinkAtBuildTimeSupport.singleton().packageOrClassAtBuildTime(resourcePath)); return; } catch (URISyntaxException e) { throw VMError.shouldNotReachHere("resourceIsDirectory for resourcePath " + resourcePath + " failed", e); From 932b40f90433e86ce07cf22bf74f837cae05b14a Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 19 Oct 2023 13:08:16 +0200 Subject: [PATCH 74/78] Set parallelism of common pool within builder. --- .../native-image/BuildOutput.md | 2 +- .../graal/llvm/LLVMNativeImageCodeCache.java | 4 +- .../com/oracle/svm/core/SubstrateOptions.java | 11 ---- .../com/oracle/svm/driver/NativeImage.java | 30 ----------- .../hosted/NativeImageGeneratorRunner.java | 6 +++ .../oracle/svm/hosted/NativeImageOptions.java | 54 +++++++++++++++++++ .../oracle/svm/hosted/ProgressReporter.java | 6 +-- 7 files changed, 66 insertions(+), 47 deletions(-) diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index 39c8fc43a2c1..b26b3b5abd3c 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -137,7 +137,7 @@ By default, the build process tries to only use free memory (to avoid memory pre If less than 8GB of memory are free, the build process falls back to use 85% of total memory. Therefore, consider freeing up memory if your machine is slow during a build, for example, by closing applications that you do not need. -By default, the build process uses all available CPU cores to maximize speed. +By default, the build process uses all available processors to maximize speed, but not more than 32 threads. Use the `--parallelism` option to set the number of threads explicitly (for example, `--parallelism=4`). Use fewer threads to reduce load on your system as well as memory consumption (at the cost of a slower build process). diff --git a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java index d93999e889dc..1611f7ce4ba4 100644 --- a/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java @@ -45,7 +45,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import com.oracle.svm.core.SubstrateOptions; import org.graalvm.collections.Pair; import jdk.graal.compiler.code.CompilationResult; import jdk.graal.compiler.core.common.NumUtil; @@ -72,6 +71,7 @@ import com.oracle.svm.core.graal.llvm.util.LLVMStackMapInfo; import com.oracle.svm.core.heap.SubstrateReferenceMap; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicInteger; +import com.oracle.svm.hosted.NativeImageOptions; import com.oracle.svm.hosted.image.NativeImage.NativeTextSectionImpl; import com.oracle.svm.hosted.image.NativeImageCodeCache; import com.oracle.svm.hosted.image.NativeImageHeap; @@ -153,7 +153,7 @@ private void writeBitcode(BatchExecutor executor) { private int createBitcodeBatches(BatchExecutor executor, DebugContext debug) { batchSize = LLVMOptions.LLVMMaxFunctionsPerBatch.getValue(); - int numThreads = SubstrateOptions.NumberOfThreads.getValue(); + int numThreads = NativeImageOptions.getActualNumberOfThreads(); int idealSize = NumUtil.divideAndRoundUp(methodIndex.length, numThreads); if (idealSize < batchSize) { batchSize = idealSize; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 03788b94e10a..2948aacaa928 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -58,7 +58,6 @@ import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.thread.VMOperationControl; import com.oracle.svm.core.util.UserError; -import com.oracle.svm.core.util.VMError; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -107,16 +106,6 @@ public static boolean parseOnce() { @Option(help = "Name of the output file to be generated", type = OptionType.User)// public static final HostedOptionKey Name = new HostedOptionKey<>(""); - /** - * Configures the number of threads of the common pool (see driver). - */ - @APIOption(name = "parallelism")// - @Option(help = "The maximum number of threads to use concurrently during native image generation.")// - public static final HostedOptionKey NumberOfThreads = new HostedOptionKey<>(Math.max(1, Math.min(Runtime.getRuntime().availableProcessors(), 32)), key -> { - int numberOfThreads = key.getValue(); - VMError.guarantee(numberOfThreads >= 1, "Number of threads must be at least 1. Validation should have happened in driver."); - }); - @APIOption(name = "shared")// @Option(help = "Build shared library")// public static final HostedOptionKey SharedLibrary = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 5641e7ecf1a6..3057cce3bb9b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -268,8 +268,6 @@ private static String oR(OptionKey option) { final String oHInspectServerContentPath = oH(PointstoOptions.InspectServerContentPath); final String oHDeadlockWatchdogInterval = oH(SubstrateOptions.DeadlockWatchdogInterval); - static final String oHNumberOfThreads = oH(SubstrateOptions.NumberOfThreads); - final Map imageBuilderEnvironment = new HashMap<>(); private final ArrayList imageBuilderArgs = new ArrayList<>(); private final LinkedHashSet imageBuilderModulePath = new LinkedHashSet<>(); @@ -1095,20 +1093,6 @@ private int completeImageBuild() { } imageClasspath.addAll(customImageClasspath); - Integer maxNumberOfThreads = getMaxNumberOfThreads(); - if (maxNumberOfThreads != null) { - if (maxNumberOfThreads >= 1) { - /* - * maxNumberOfThreads - 1 because the main thread always helps to process tasks. In - * single-threaded mode (parallelism=0 for common pool), only the main thread - * processes tasks. - */ - imageBuilderJavaArgs.add("-Djava.util.concurrent.ForkJoinPool.common.parallelism=" + (maxNumberOfThreads - 1)); - } else { - throw showError("The number of threads was set to " + maxNumberOfThreads + ". Please set the '--parallelism' option to at least 1."); - } - } - /* * Work around "JDK-8315810: Reimplement * sun.reflect.ReflectionFactory::newConstructorForSerialization with method handles" @@ -1359,20 +1343,6 @@ private static Boolean getHostedOptionFinalBooleanArgumentValue(List arg return result; } - private Integer getMaxNumberOfThreads() { - String numberOfThreadsValue = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHNumberOfThreads); - if (numberOfThreadsValue != null) { - try { - return Integer.parseInt(numberOfThreadsValue); - } catch (NumberFormatException e) { - /* Validated already by CommonOptionParser. */ - throw VMError.shouldNotReachHere(e); - } - } else { - return null; - } - } - private boolean shouldAddCWDToCP() { if (config.buildFallbackImage() || printFlagsOptionQuery != null || printFlagsWithExtraHelpOptionQuery != null) { return false; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java index 2d561753051e..7de0c7713c42 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java @@ -292,6 +292,12 @@ public static ImageClassLoader installNativeImageClassLoader(String[] classpath, */ NativeImageGenerator.setSystemPropertiesForImageEarly(); + /* + * Size the common pool before creating the image class loader because it is the first + * component to use the common pool. + */ + NativeImageOptions.setCommonPoolParallelism(nativeImageClassLoaderSupport.getParsedHostedOptions()); + return new ImageClassLoader(NativeImageGenerator.getTargetPlatform(nativeImageClassLoader), nativeImageClassLoaderSupport); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java index ac2626469a98..56cd319598a5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java @@ -31,7 +31,9 @@ import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.concurrent.ForkJoinPool; +import com.oracle.svm.util.LogUtils; import org.graalvm.collections.EconomicMap; import jdk.graal.compiler.options.Option; import jdk.graal.compiler.options.OptionKey; @@ -186,6 +188,58 @@ public static CStandards getCStandard() { } } + /** + * Configures the number of threads of the common pool (see driver). + */ + private static final String PARALLELISM_OPTION_NAME = "parallelism"; + @APIOption(name = PARALLELISM_OPTION_NAME)// + @Option(help = "The maximum number of threads to use concurrently during native image generation.")// + public static final HostedOptionKey NumberOfThreads = new HostedOptionKey<>(Math.max(1, Math.min(Runtime.getRuntime().availableProcessors(), 32)), key -> { + int numberOfThreads = key.getValue(); + if (numberOfThreads < 1) { + throw UserError.abort("The number of threads was set to %s. Please set the '--%s' option to at least 1.", numberOfThreads, PARALLELISM_OPTION_NAME); + } + }); + + public static int getActualNumberOfThreads() { + int commonThreadParallelism = ForkJoinPool.getCommonPoolParallelism(); + if (NumberOfThreads.getValue() == 1) { + assert commonThreadParallelism == 1 : "Disabled common pool expected to report parallelism of 1"; + commonThreadParallelism = 0; /* A disabled common pool has no actual threads */ + } + /* + * Main thread plus common pool threads. setCommonPoolParallelism() asserts that this number + * matches NumberOfThreads. + */ + return 1 + commonThreadParallelism; + } + + public static void setCommonPoolParallelism(OptionValues optionValues) { + if (NativeImageOptions.NumberOfThreads.hasBeenSet(optionValues)) { + /* + * The main thread always helps to process tasks submitted to the common pool (e.g., see + * ForkJoinPool#awaitTermination()), so subtract one from the number of threads. The + * common pool can be disabled "by setting the parallelism property to zero" (see + * ForkJoinPool's javadoc). + */ + int numberOfCommonPoolThreads = NativeImageOptions.NumberOfThreads.getValue(optionValues) - 1; + String commonPoolParallelismProperty = "java.util.concurrent.ForkJoinPool.common.parallelism"; + assert System.getProperty(commonPoolParallelismProperty) == null : commonPoolParallelismProperty + " already set"; + System.setProperty(commonPoolParallelismProperty, "" + numberOfCommonPoolThreads); + int actualCommonPoolParallelism = ForkJoinPool.commonPool().getParallelism(); + /* + * getParallelism() returns at least 1, even in single-threaded mode where common pool + * is disabled. + */ + boolean isSingleThreadedMode = numberOfCommonPoolThreads == 0 && actualCommonPoolParallelism == 1; + if (!isSingleThreadedMode && actualCommonPoolParallelism != numberOfCommonPoolThreads) { + String warning = "Failed to set parallelism of common pool (actual parallelism is %s).".formatted(actualCommonPoolParallelism); + assert false : warning; + LogUtils.warning(warning); + } + } + } + @Option(help = "Deprecated, option no longer has any effect", deprecated = true, deprecationMessage = "Please use '--parallelism' instead.")// public static final HostedOptionKey NumberOfAnalysisThreads = new HostedOptionKey<>(-1); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 309031cd0a71..d36baa66971a 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -392,13 +392,13 @@ private void printResourceInfo() { maxHeapSuffix = "set via '%s'".formatted(xmxValueOrNull); } - int maxNumberOfThreads = SubstrateOptions.NumberOfThreads.getValue(); + int maxNumberOfThreads = NativeImageOptions.getActualNumberOfThreads(); recordJsonMetric(ResourceUsageKey.PARALLELISM, maxNumberOfThreads); int availableProcessors = runtime.availableProcessors(); recordJsonMetric(ResourceUsageKey.CPU_CORES_TOTAL, availableProcessors); String maxNumberOfThreadsSuffix = "determined at start"; - if (SubstrateOptions.NumberOfThreads.hasBeenSet()) { - maxNumberOfThreadsSuffix = "set via '%s'".formatted(SubstrateOptionsParser.commandArgument(SubstrateOptions.NumberOfThreads, Integer.toString(maxNumberOfThreads))); + if (NativeImageOptions.NumberOfThreads.hasBeenSet()) { + maxNumberOfThreadsSuffix = "set via '%s'".formatted(SubstrateOptionsParser.commandArgument(NativeImageOptions.NumberOfThreads, Integer.toString(maxNumberOfThreads))); } l().printLineSeparator(); From af128f66959d50c80005c61b6550b9d066223018 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 19 Oct 2023 13:11:11 +0200 Subject: [PATCH 75/78] Drop ununsed code. --- .../graal/pointsto/PointsToAnalysis.java | 20 ------ .../DisallowedImageHeapObjectFeature.java | 5 -- .../svm/util/ImageGeneratorThreadMarker.java | 67 ------------------- 3 files changed, 92 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ImageGeneratorThreadMarker.java diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java index 23d0ec04653f..cda851aa7c95 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java @@ -37,8 +37,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.ForkJoinWorkerThread; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLongArray; import java.util.function.Consumer; @@ -75,7 +73,6 @@ import com.oracle.graal.pointsto.util.TimerCollection; import com.oracle.svm.common.meta.MultiMethod; import com.oracle.svm.util.ClassUtil; -import com.oracle.svm.util.ImageGeneratorThreadMarker; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.debug.DebugContext; @@ -723,21 +720,4 @@ public void print() { System.out.println(); } } - - private static class SubstrateWorkerThread extends ForkJoinWorkerThread - implements ImageGeneratorThreadMarker { - private final DebugContext debug; - - SubstrateWorkerThread(ForkJoinPool pool, DebugContext debug) { - super(pool); - this.debug = debug; - } - - @Override - protected void onTermination(Throwable exception) { - if (debug != null) { - debug.closeDumpHandlers(true); - } - } - } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/DisallowedImageHeapObjectFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/DisallowedImageHeapObjectFeature.java index 41597934921c..959bd979b12c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/DisallowedImageHeapObjectFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/DisallowedImageHeapObjectFeature.java @@ -50,7 +50,6 @@ import com.oracle.svm.hosted.FeatureImpl; import com.oracle.svm.hosted.classinitialization.ClassInitializationOptions; import com.oracle.svm.hosted.classinitialization.ClassInitializationSupport; -import com.oracle.svm.util.ImageGeneratorThreadMarker; /** * Complain if there are types that can not move from the image generator heap to the image heap. @@ -113,10 +112,6 @@ private static String[] getDisallowedSubstrings(String... substrings) { } private Object replacer(Object original) { - if (original instanceof Thread && original instanceof ImageGeneratorThreadMarker) { - return ((ImageGeneratorThreadMarker) original).asTerminated(); - } - checkDisallowedMBeanObjects(original); if (original instanceof String && disallowedSubstrings != null) { diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ImageGeneratorThreadMarker.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ImageGeneratorThreadMarker.java deleted file mode 100644 index d11e1e338fd1..000000000000 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ImageGeneratorThreadMarker.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.util; - -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; - -/** - * Marker interface to identify threads that are only used by SubstateVM infrastructure and will not - * be present in the image heap at run time. Each of such threads has a {@link #asTerminated() way} - * to obtain the terminated replacement of itself. - *

- * Not intented to be implemented by customer threads. Allowing background threads that are not - * terminated can lead to very hard to debug errors. If you have background threads running while - * the static analysis is running, then bad race conditions can happen: the static analysis looks at - * a heap snapshot at a "random" time from the point of view of the background thread. If the - * background thread makes new objects available after the last heap snapshot, then the static - * analysis misses objects of the image heap, which leads to either strange errors during image - * generation (when the image heap is written), or even worse strange behavior at run time (when a - * field is, e.g., constant folded to {@code null} because the static analysis only saw the - * {@code null} value). - */ -@Platforms(Platform.HOSTED_ONLY.class) -public interface ImageGeneratorThreadMarker /* extends Thread */ { - default Thread asTerminated() { - return TerminatedThread.SINGLETON; - } -} - -final class TerminatedThread extends Thread { - static final TerminatedThread SINGLETON; - static { - SINGLETON = new TerminatedThread("Terminated Infrastructure Thread"); - SINGLETON.start(); - try { - SINGLETON.join(); - } catch (InterruptedException ex) { - throw new IllegalStateException(ex); - } - } - - TerminatedThread(String name) { - super(name); - } -} From 589c73ac58cec2f794ea19532224be5be6c8911b Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 20 Oct 2023 08:57:06 +0200 Subject: [PATCH 76/78] Drop executorService field from CompletionExecutor. --- .../graal/pointsto/util/CompletionExecutor.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java index 7e87277bc7a7..a57aa9dda6c1 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/CompletionExecutor.java @@ -34,15 +34,14 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; +import com.oracle.graal.pointsto.BigBang; + import jdk.graal.compiler.debug.DebugContext; import jdk.graal.compiler.debug.DebugContext.Activation; import jdk.graal.compiler.debug.DebugContext.Description; import jdk.graal.compiler.debug.DebugContext.Scope; import jdk.graal.compiler.debug.DebugHandlersFactory; import jdk.graal.compiler.options.OptionValues; - -import com.oracle.graal.pointsto.BigBang; - import jdk.vm.ci.common.JVMCIError; /** @@ -63,8 +62,6 @@ private enum State { private List postedBeforeStart; private final CopyOnWriteArrayList exceptions = new CopyOnWriteArrayList<>(); - private final ForkJoinPool executorService; - private final DebugContext debug; private final BigBang bb; private Timing timing; @@ -85,7 +82,6 @@ public interface Timing { public CompletionExecutor(DebugContext debugContext, BigBang bb) { this.debug = debugContext.areScopesEnabled() || debugContext.areMetricsEnabled() ? debugContext : null; this.bb = bb; - executorService = ForkJoinPool.commonPool(); state = new AtomicReference<>(State.UNUSED); postedOperations = new LongAdder(); completedOperations = new LongAdder(); @@ -155,7 +151,7 @@ public void execute(DebugContextRunnable command) { } private void executeService(DebugContextRunnable command) { - executorService.execute(() -> executeCommand(command)); + ForkJoinPool.commonPool().execute(() -> executeCommand(command)); } @SuppressWarnings("try") @@ -211,7 +207,7 @@ public long complete() throws InterruptedException { while (true) { assert state.get() == State.STARTED : state.get(); - boolean quiescent = executorService.awaitQuiescence(100, TimeUnit.MILLISECONDS); + boolean quiescent = ForkJoinPool.commonPool().awaitQuiescence(100, TimeUnit.MILLISECONDS); if (timing != null && !quiescent) { long curTime = System.nanoTime(); if (curTime - lastPrint > timing.getPrintIntervalNanos()) { @@ -247,7 +243,7 @@ public long getPostedOperations() { } public void shutdown() { - assert !executorService.hasQueuedSubmissions() : "There should be no queued submissions on shutdown."; + assert !ForkJoinPool.commonPool().hasQueuedSubmissions() : "There should be no queued submissions on shutdown."; assert completedOperations.sum() == postedOperations.sum() : "Posted operations (" + postedOperations.sum() + ") must match completed (" + completedOperations.sum() + ") operations"; setState(State.UNUSED); } From 72876c81526869f201ac6a1ddedf7ffdd5c5c5fa Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 20 Oct 2023 11:21:16 +0200 Subject: [PATCH 77/78] Ensure option descriptor is loaded when accessed. --- .../jdk/graal/compiler/options/OptionKey.java | 16 +++++++++++++++- .../svm/core/option/SubstrateOptionsParser.java | 6 ++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/OptionKey.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/OptionKey.java index 12471587bdef..8e485bb53ec1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/OptionKey.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/OptionKey.java @@ -50,12 +50,26 @@ public final void setDescriptor(OptionDescriptor descriptor) { /** * Returns the descriptor for this option, if it has been set by - * {@link #setDescriptor(OptionDescriptor)}. + * {@link #setDescriptor(OptionDescriptor)}. As descriptors are loaded lazily, this method will + * return {@code null} if the descriptors have not been loaded. Use {@link #loadDescriptor} + * instead to ensure a non-null descriptor is returned if available. */ public final OptionDescriptor getDescriptor() { return descriptor; } + /** + * Returns the descriptor for this option, triggering loading of descriptors if this descriptor + * is null. Note that it's still possible for this method to return null if this option does not + * have a descriptor created by a service loader. + */ + public final OptionDescriptor loadDescriptor() { + if (descriptor == null) { + Lazy.init(); + } + return descriptor; + } + /** * Checks that a descriptor exists for this key after triggering loading of descriptors. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/SubstrateOptionsParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/SubstrateOptionsParser.java index 9b4bb45e80f0..5c2151bddc45 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/SubstrateOptionsParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/SubstrateOptionsParser.java @@ -175,16 +175,18 @@ public static String commandArgument(OptionKey option, String value) { */ @Platforms(Platform.HOSTED_ONLY.class) public static String commandArgument(OptionKey option, String value, String apiOptionName) { + /* Ensure descriptor is loaded */ + OptionDescriptor optionDescriptor = option.loadDescriptor(); Field field; try { - field = option.getDescriptor().getDeclaringClass().getDeclaredField(option.getDescriptor().getFieldName()); + field = optionDescriptor.getDeclaringClass().getDeclaredField(optionDescriptor.getFieldName()); } catch (ReflectiveOperationException ex) { throw VMError.shouldNotReachHere(ex); } APIOption[] apiOptions = field.getAnnotationsByType(APIOption.class); - if (option.getDescriptor().getOptionValueType() == Boolean.class) { + if (optionDescriptor.getOptionValueType() == Boolean.class) { VMError.guarantee(value.equals("+") || value.equals("-"), "Boolean option value can be only + or -"); for (APIOption apiOption : apiOptions) { String selected = selectVariant(apiOption, apiOptionName); From b55218140f03522cafea2b746899f30eafa7a3ee Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Wed, 25 Oct 2023 17:29:41 +0200 Subject: [PATCH 78/78] Revert overridden prepareBundle function because it can be called from other places --- .../OptimizedLocalizationSupport.java | 16 ++++++++++++++-- .../jdk/localization/LocalizationFeature.java | 12 ++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java index b24464c5f603..5a8a29f4a1b4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/localization/OptimizedLocalizationSupport.java @@ -30,11 +30,15 @@ import java.util.Locale; import java.util.Map; import java.util.MissingResourceException; +import java.util.Optional; import java.util.ResourceBundle; import java.util.Set; +import java.util.function.Function; import java.util.spi.LocaleServiceProvider; import org.graalvm.collections.Pair; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.ReflectionUtil; @@ -87,13 +91,21 @@ public void prepareClassResourceBundle(String basename, Class bundleClass) { /*- Set the basename and locale to be consistent with JVM lookup process */ bundleNameField.set(bundle, basename); bundleLocaleField.set(bundle, locale); - bundle.keySet(); - this.resourceBundles.put(Pair.create(basename, locale), bundle); + + // override in this class does not use findModule + prepareBundle(basename, bundle, null, locale); } catch (ReflectionUtil.ReflectionUtilError | ReflectiveOperationException e) { throw UserError.abort(e, "Failed to instantiated bundle from class %s, reason %s", bundleClass, e.getCause().getMessage()); } } + @Platforms(Platform.HOSTED_ONLY.class) + @Override + public void prepareBundle(String bundleName, ResourceBundle bundle, Function> findModule, Locale locale) { + bundle.keySet(); + this.resourceBundles.put(Pair.create(bundleName, locale), bundle); + } + private static Locale extractLocale(Class bundleClass) { String name = bundleClass.getName(); int split = name.lastIndexOf('_'); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java index 7aabef6e0565..89dc8f45fdcf 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java @@ -61,12 +61,6 @@ import java.util.stream.Collectors; import org.graalvm.collections.Pair; -import jdk.graal.compiler.nodes.ValueNode; -import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; -import jdk.graal.compiler.nodes.graphbuilderconf.NodePlugin; -import jdk.graal.compiler.options.Option; -import jdk.graal.compiler.options.OptionStability; -import jdk.graal.compiler.options.OptionType; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -91,6 +85,12 @@ import com.oracle.svm.hosted.FeatureImpl.DuringSetupAccessImpl; import com.oracle.svm.hosted.ImageClassLoader; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import jdk.graal.compiler.nodes.graphbuilderconf.NodePlugin; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionStability; +import jdk.graal.compiler.options.OptionType; import jdk.internal.access.SharedSecrets; import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod;