From dc865d774bc61eed2f292301ceb085997a9f80b9 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 12:08:17 +0100 Subject: [PATCH 01/14] Fix realpath usage on systems with bsd coreutils --- Tools/wasm/emscripten/__main__.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 9ce8dd6a364ad6..b99b74e3661060 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -215,12 +215,29 @@ def configure_emscripten_python(context, working_dir): exec_script = working_dir / "python.sh" exec_script.write_text( dedent( - f"""\ + """\ #!/bin/sh + # Macs come with free BSD coreutils which doesn't have the -s option + # so feature detect and work around it. + if which grealpath > /dev/null; then + # It has brew installed gnu core utils, use that + REALPATH="grealpath -s" + elif which realpath > /dev/null && realpath --version | grep GNU > /dev/null; then + # realpath points to GNU realpath so use it. + REALPATH="realpath -s" + else + # Shim for macs without GNU coreutils + abs_path () { + echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" + } + REALPATH=abs_path + fi + """ + f""" # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. - exec {host_runner} {node_entry} "$(realpath -s $0)" "$@" + exec {host_runner} {node_entry} "$($REALPATH "$0")" "$@" """ ) ) From 18f0424ffa22e4898ed9f589fee59dab95f43448 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 12:29:39 +0100 Subject: [PATCH 02/14] Mount all native directories that aren't known to cause trouble This mounts all native directories into the Emscripten file system except for dev, lib, and proc. --- Tools/wasm/emscripten/node_entry.mjs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index cb1c6ff3cba6aa..295fbcf6002268 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -1,6 +1,5 @@ import EmscriptenModule from "./python.mjs"; -import { dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import fs from "node:fs"; if (process?.versions?.node) { const nodeVersion = Number(process.versions.node.split(".", 1)[0]); @@ -12,11 +11,24 @@ if (process?.versions?.node) { } } +function rootDirsToMount(Module) { + return fs + .readdirSync("/") + .filter((dir) => !["dev", "lib", "proc"].includes(dir)).map(dir => "/" + dir); +} + +function mountDirectories(Module) { + for (const dir of rootDirsToMount(Module)) { + Module.FS.mkdirTree(dir); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir); + } +} + const settings = { preRun(Module) { - const __dirname = dirname(fileURLToPath(import.meta.url)); - Module.FS.mkdirTree("/lib/"); - Module.FS.mount(Module.FS.filesystems.NODEFS, { root: __dirname + "/lib/" }, "/lib/"); + Module.FS.mkdirTree("/home/"); + mountDirectories(Module); + Module.FS.chdir(process.cwd()); }, // The first three arguments are: "node", path to this file, path to // python.sh. After that come the arguments the user passed to python.sh. From 38b1f7a6bbc0f9c57157f9d9608ce4173ebe39f9 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 13:05:33 +0100 Subject: [PATCH 03/14] Pass environment variables into Emscripten --- Tools/wasm/emscripten/node_entry.mjs | 1 + configure | 2 +- configure.ac | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index 295fbcf6002268..a899ba1ce93e49 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -29,6 +29,7 @@ const settings = { Module.FS.mkdirTree("/home/"); mountDirectories(Module); Module.FS.chdir(process.cwd()); + Object.assign(Module.ENV, process.env); }, // The first three arguments are: "node", path to this file, path to // python.sh. After that come the arguments the user passed to python.sh. diff --git a/configure b/configure index 4e4043260ed2df..5024aa9081de5e 100755 --- a/configure +++ b/configure @@ -9430,7 +9430,7 @@ fi as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS" + as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,ENV" if test "x$enable_wasm_dynamic_linking" = xyes then : diff --git a/configure.ac b/configure.ac index 4cfced10432491..936c542f7e64fd 100644 --- a/configure.ac +++ b/configure.ac @@ -2328,7 +2328,7 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,ENV"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) From 4d4527967388d5752ab5324546570f9d11792136 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 13:12:17 +0100 Subject: [PATCH 04/14] Change make target from commoninstall to all --- Tools/wasm/emscripten/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index b99b74e3661060..ecde70bf945323 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -250,7 +250,7 @@ def configure_emscripten_python(context, working_dir): def make_emscripten_python(context, working_dir): """Run `make` for the emscripten/host build.""" call( - ["make", "--jobs", str(cpu_count()), "commoninstall"], + ["make", "--jobs", str(cpu_count()), "all"], env=updated_env(), quiet=context.quiet, ) From af9960355b1dc6d6747db63849dd38589d942d8d Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 13:22:59 +0100 Subject: [PATCH 05/14] Pass flags in NODEFLAGS environment variable to node Useful for: --experimental flags, --inspect flags, etc. --- Tools/wasm/emscripten/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index ecde70bf945323..de41bab7eb3b8a 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -237,7 +237,8 @@ def configure_emscripten_python(context, working_dir): f""" # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. - exec {host_runner} {node_entry} "$($REALPATH "$0")" "$@" + # Intentionally allow word splitting on NODEFLAGS. + exec {host_runner} {node_entry} ${NODEFLAGS} "$($REALPATH "$0")" "$@" """ ) ) From f9411929e42d87e00bfef148e32d156373e9de80 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 16:53:46 +0100 Subject: [PATCH 06/14] Apply mdboom's review comments --- Tools/wasm/emscripten/__main__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index de41bab7eb3b8a..2e230e4d00afde 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -218,7 +218,7 @@ def configure_emscripten_python(context, working_dir): """\ #!/bin/sh - # Macs come with free BSD coreutils which doesn't have the -s option + # Macs come with FreeBSD coreutils which doesn't have the -s option # so feature detect and work around it. if which grealpath > /dev/null; then # It has brew installed gnu core utils, use that @@ -228,13 +228,12 @@ def configure_emscripten_python(context, working_dir): REALPATH="realpath -s" else # Shim for macs without GNU coreutils - abs_path () { + abs_path () {{ echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" - } + }} REALPATH=abs_path fi - """ - f""" + # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. From 351aa4f1e701331ae69c7d23ea9a4e768aa4707f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 16:54:21 +0100 Subject: [PATCH 07/14] Fix NODEFLAGS reference --- Tools/wasm/emscripten/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 2e230e4d00afde..31ddede3336f1a 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -237,7 +237,7 @@ def configure_emscripten_python(context, working_dir): # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} {node_entry} ${NODEFLAGS} "$($REALPATH "$0")" "$@" + exec {host_runner} {node_entry} $NODEFLAGS "$($REALPATH "$0")" "$@" """ ) ) From 1e434938b78feff62d0b1837164fad5a01f473e5 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 2 Dec 2024 19:57:46 +0100 Subject: [PATCH 08/14] Fix NODEFLAGS --- Tools/wasm/emscripten/__main__.py | 2 +- Tools/wasm/emscripten/node_entry.mjs | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 31ddede3336f1a..9d73555072db0d 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -237,7 +237,7 @@ def configure_emscripten_python(context, working_dir): # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} {node_entry} $NODEFLAGS "$($REALPATH "$0")" "$@" + exec node $NODEFLAGS /home/rchatham/cpython/cross-build/wasm32-emscripten/node_entry.mjs "$($REALPATH "$0")" "$@" """ ) ) diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index a899ba1ce93e49..e3efb4a0324fce 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -24,6 +24,8 @@ function mountDirectories(Module) { } } +const pythonShIndex = process.argv.findIndex(x => x.endsWith("python.sh")); + const settings = { preRun(Module) { Module.FS.mkdirTree("/home/"); @@ -31,13 +33,12 @@ const settings = { Module.FS.chdir(process.cwd()); Object.assign(Module.ENV, process.env); }, - // The first three arguments are: "node", path to this file, path to - // python.sh. After that come the arguments the user passed to python.sh. - arguments: process.argv.slice(3), // Ensure that sys.executable, sys._base_executable, etc point to python.sh // not to this file. To properly handle symlinks, python.sh needs to compute // its own path. - thisProgram: process.argv[2], + thisProgram: process.argv[pythonShIndex], + // After python.sh come the arguments that the user passed to python.sh. + arguments: process.argv.slice(pythonShIndex + 1), }; await EmscriptenModule(settings); From af17844e0072dd8ce3a731d308cb2f44bde4ea8f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 3 Dec 2024 11:00:55 +0100 Subject: [PATCH 09/14] Fix python.sh template --- Tools/wasm/emscripten/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 9d73555072db0d..071c5ff26ea722 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -237,7 +237,7 @@ def configure_emscripten_python(context, working_dir): # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. - exec node $NODEFLAGS /home/rchatham/cpython/cross-build/wasm32-emscripten/node_entry.mjs "$($REALPATH "$0")" "$@" + exec {host_runner} $NODEFLAGS {node_entry} "$($REALPATH "$0")" "$@" """ ) ) From ca741bce4979e88ba1b2d024433ec040b6b4f982 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 3 Dec 2024 11:01:54 +0100 Subject: [PATCH 10/14] Be more careful with handling of the program name Previous version only worked if the name was python.sh. If we symlink something else to it (say `python`) it would fail. This update makes it not depend on the name of the link. --- Tools/wasm/emscripten/__main__.py | 2 +- Tools/wasm/emscripten/node_entry.mjs | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 071c5ff26ea722..d6d06b953d7be0 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -237,7 +237,7 @@ def configure_emscripten_python(context, working_dir): # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} $NODEFLAGS {node_entry} "$($REALPATH "$0")" "$@" + exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" """ ) ) diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index e3efb4a0324fce..56c9303ffdadfa 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -24,7 +24,8 @@ function mountDirectories(Module) { } } -const pythonShIndex = process.argv.findIndex(x => x.endsWith("python.sh")); +const thisProgram = "--this-program="; +const thisProgramIndex = process.argv.findIndex(x => x.startsWith(thisProgram)); const settings = { preRun(Module) { @@ -36,9 +37,9 @@ const settings = { // Ensure that sys.executable, sys._base_executable, etc point to python.sh // not to this file. To properly handle symlinks, python.sh needs to compute // its own path. - thisProgram: process.argv[pythonShIndex], - // After python.sh come the arguments that the user passed to python.sh. - arguments: process.argv.slice(pythonShIndex + 1), + thisProgram: process.argv[thisProgramIndex], + // After python.sh come the arguments thatthe user passed to python.sh. + arguments: process.argv.slice(thisProgramIndex + 1), }; await EmscriptenModule(settings); From 60875c04c139b01876a6b82a1e24aec0f81fc1a3 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 3 Dec 2024 11:03:31 +0100 Subject: [PATCH 11/14] Apply code formatter --- Tools/wasm/emscripten/node_entry.mjs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index 56c9303ffdadfa..072ae5cef4303b 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -4,17 +4,18 @@ import fs from "node:fs"; if (process?.versions?.node) { const nodeVersion = Number(process.versions.node.split(".", 1)[0]); if (nodeVersion < 18) { - process.stderr.write( - `Node version must be >= 18, got version ${process.version}\n`, - ); - process.exit(1); + process.stderr.write( + `Node version must be >= 18, got version ${process.version}\n`, + ); + process.exit(1); } } function rootDirsToMount(Module) { return fs - .readdirSync("/") - .filter((dir) => !["dev", "lib", "proc"].includes(dir)).map(dir => "/" + dir); + .readdirSync("/") + .filter((dir) => !["dev", "lib", "proc"].includes(dir)) + .map((dir) => "/" + dir); } function mountDirectories(Module) { @@ -25,7 +26,9 @@ function mountDirectories(Module) { } const thisProgram = "--this-program="; -const thisProgramIndex = process.argv.findIndex(x => x.startsWith(thisProgram)); +const thisProgramIndex = process.argv.findIndex((x) => + x.startsWith(thisProgram), +); const settings = { preRun(Module) { From 63d57c83ccaa6b249c392afa5599ac3c3349c0af Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 3 Dec 2024 11:04:24 +0100 Subject: [PATCH 12/14] Redirect realpath --version to /dev/null again --- Tools/wasm/emscripten/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index d6d06b953d7be0..5e05620d8e8d58 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -223,7 +223,7 @@ def configure_emscripten_python(context, working_dir): if which grealpath > /dev/null; then # It has brew installed gnu core utils, use that REALPATH="grealpath -s" - elif which realpath > /dev/null && realpath --version | grep GNU > /dev/null; then + elif which realpath > /dev/null && realpath --version 2&>1 | grep GNU > /dev/null; then # realpath points to GNU realpath so use it. REALPATH="realpath -s" else From f9c9ace49400c8583a59039aae6cf0c2a01bada2 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 3 Dec 2024 11:07:24 +0100 Subject: [PATCH 13/14] Cleanup --- Tools/wasm/emscripten/node_entry.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index 072ae5cef4303b..40ab1515cf28c1 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -32,7 +32,6 @@ const thisProgramIndex = process.argv.findIndex((x) => const settings = { preRun(Module) { - Module.FS.mkdirTree("/home/"); mountDirectories(Module); Module.FS.chdir(process.cwd()); Object.assign(Module.ENV, process.env); From ca2abc32122930401e07e33f6860e29f224d09e1 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 4 Dec 2024 11:05:11 +0100 Subject: [PATCH 14/14] Put back in f string --- Tools/wasm/emscripten/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 5e05620d8e8d58..c998ed71309dad 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -215,7 +215,7 @@ def configure_emscripten_python(context, working_dir): exec_script = working_dir / "python.sh" exec_script.write_text( dedent( - """\ + f"""\ #!/bin/sh # Macs come with FreeBSD coreutils which doesn't have the -s option