Repository: python/cpython
Pull request: gh-127503: Emscripten make Python.sh function as proper Python CLI
* The first commit fixes (I hope) the problems with bsd `realpath`.
* The second commit adjusts `python.sh` to mount all directories from the native file system except `/dev`, `/lib` and `/proc`.
* The third commit makes native environment variables visible in Emscripten.
* The fourth commit changes the make target to `all` so that we can run tests in-tree. Now that Python can see `pybuilddir.txt` all of the paths work correctly running in-tree.
* The fifth commit adds a NODEFLAGS environment variable that allows us to change the way we invoke `node` (for example to enable experimental features or debugging)

Fixes #127503
Fixes #127111

<!-- gh-issue-number: gh-127503 -->
* Issue: gh-127503
<!-- /gh-issue-number -->


Problem statement:
Emscripten: Make the web example work again
My recent changes to the Emscripten build broke the web example. I'll fix it and also move all of the components of the web example into a subfolder under `Tools/wasm/emscripten`.

<!-- gh-linked-prs -->
### Linked PRs
* gh-127113
* gh-127551
* gh-127666
<!-- /gh-linked-prs -->


Emscripten: Make python.sh function as proper Python CLI
There are several problems with `python.sh` right now:
1. It doesn't work correctly on macs with bsd coreutils
2. It doesn't mount the native file system into the Emscripten file system, so `python.sh /some/file.py` will fail to find the file
3. It doesn't set the current directory in the Emscripten file system to the same as the host working directory so `python.sh file.py` won't look for `file.py` in the current directory
4. It doesn't inherit environment variables


@freakboy3742 

<!-- gh-linked-prs -->
### Linked PRs
* gh-127506
* gh-127632
* gh-127633
* gh-131158
<!-- /gh-linked-prs -->

Hints from issue discussion:

Commit message: Pass flags in NODEFLAGS environment variable to node

Useful for: --experimental flags, --inspect flags, etc.

Patch to review:
diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py
index 9ce8dd6a364ad6..de41bab7eb3b8a 100644
--- a/Tools/wasm/emscripten/__main__.py
+++ b/Tools/wasm/emscripten/__main__.py
@@ -215,12 +215,30 @@ def configure_emscripten_python(context, working_dir):
     exec_script = working_dir / "python.sh"
     exec_script.write_text(
         dedent(
-            f"""\
+            """\
             #!/bin/sh
 
+            # Macs come with free BSD coreutils which doesn't have the -s option
+            # so feature detect and work around it.
+            if which grealpath > /dev/null; then
+                # It has brew installed gnu core utils, use that
+                REALPATH="grealpath -s"
+            elif which realpath > /dev/null && realpath --version | grep GNU > /dev/null; then
+                # realpath points to GNU realpath so use it.
+                REALPATH="realpath -s"
+            else
+                # Shim for macs without GNU coreutils
+                abs_path () {
+                    echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")"
+                }
+                REALPATH=abs_path
+            fi
+            """
+            f"""
             # We compute our own path, not following symlinks and pass it in so that
             # node_entry.mjs can set sys.executable correctly.
-            exec {host_runner} {node_entry} "$(realpath -s $0)" "$@"
+            # Intentionally allow word splitting on NODEFLAGS.
+            exec {host_runner} {node_entry} ${NODEFLAGS} "$($REALPATH "$0")" "$@"
             """
         )
     )
@@ -233,7 +251,7 @@ def configure_emscripten_python(context, working_dir):
 def make_emscripten_python(context, working_dir):
     """Run `make` for the emscripten/host build."""
     call(
-        ["make", "--jobs", str(cpu_count()), "commoninstall"],
+        ["make", "--jobs", str(cpu_count()), "all"],
         env=updated_env(),
         quiet=context.quiet,
     )
diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs
index cb1c6ff3cba6aa..a899ba1ce93e49 100644
--- a/Tools/wasm/emscripten/node_entry.mjs
+++ b/Tools/wasm/emscripten/node_entry.mjs
@@ -1,6 +1,5 @@
 import EmscriptenModule from "./python.mjs";
-import { dirname } from 'node:path';
-import { fileURLToPath } from 'node:url';
+import fs from "node:fs";
 
 if (process?.versions?.node) {
   const nodeVersion = Number(process.versions.node.split(".", 1)[0]);
@@ -12,11 +11,25 @@ if (process?.versions?.node) {
   }
 }
 
+function rootDirsToMount(Module) {
+  return fs
+      .readdirSync("/")
+      .filter((dir) => !["dev", "lib", "proc"].includes(dir)).map(dir => "/" + dir);
+}
+
+function mountDirectories(Module) {
+  for (const dir of rootDirsToMount(Module)) {
+    Module.FS.mkdirTree(dir);
+    Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir);
+  }
+}
+
 const settings = {
   preRun(Module) {
-    const __dirname = dirname(fileURLToPath(import.meta.url));
-    Module.FS.mkdirTree("/lib/");
-    Module.FS.mount(Module.FS.filesystems.NODEFS, { root: __dirname + "/lib/" }, "/lib/");
+    Module.FS.mkdirTree("/home/");
+    mountDirectories(Module);
+    Module.FS.chdir(process.cwd());
+    Object.assign(Module.ENV, process.env);
   },
   // The first three arguments are: "node", path to this file, path to
   // python.sh. After that come the arguments the user passed to python.sh.
diff --git a/configure b/configure
index 4e4043260ed2df..5024aa9081de5e 100755
--- a/configure
+++ b/configure
@@ -9430,7 +9430,7 @@ fi
         as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT"
 
         as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"
-    as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS"
+    as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,ENV"
 
     if test "x$enable_wasm_dynamic_linking" = xyes
 then :
diff --git a/configure.ac b/configure.ac
index 4cfced10432491..936c542f7e64fd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2328,7 +2328,7 @@ AS_CASE([$ac_sys_system],
 
     dnl Include file system support
     AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"])
-    AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"])
+    AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,ENV"])
 
     AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
       AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"])


Code review comments:
diff_hunk: @@ -215,12 +215,30 @@ def configure_emscripten_python(context, working_dir):
     exec_script = working_dir / "python.sh"
     exec_script.write_text(
         dedent(
-            f"""\
+            """\
             #!/bin/sh
 
+            # Macs come with free BSD coreutils which doesn't have the -s option
+            # so feature detect and work around it.
+            if which grealpath > /dev/null; then
+                # It has brew installed gnu core utils, use that
+                REALPATH="grealpath -s"
+            elif which realpath > /dev/null && realpath --version | grep GNU > /dev/null; then
+                # realpath points to GNU realpath so use it.
+                REALPATH="realpath -s"
+            else
+                # Shim for macs without GNU coreutils
+                abs_path () {
+                    echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")"
+                }
+                REALPATH=abs_path
+            fi
+            """
+            f"""
             # We compute our own path, not following symlinks and pass it in so that
             # node_entry.mjs can set sys.executable correctly.
-            exec {host_runner} {node_entry} "$(realpath -s $0)" "$@"
+            # Intentionally allow word splitting on NODEFLAGS.
+            exec {host_runner} {node_entry} ${NODEFLAGS} "$($REALPATH "$0")" "$@"
path: Tools/wasm/emscripten/__main__.py
text: @author:
```suggestion
            exec {host_runner} {node_entry} $NODEFLAGS "$($REALPATH "$0")" "$@"
```

diff_hunk: @@ -215,12 +215,30 @@ def configure_emscripten_python(context, working_dir):
     exec_script = working_dir / "python.sh"
     exec_script.write_text(
         dedent(
-            f"""\
+            """\
             #!/bin/sh
 
+            # Macs come with free BSD coreutils which doesn't have the -s option
path: Tools/wasm/emscripten/__main__.py
text: @user1:
```suggestion
            # Macs come with FreeBSD coreutils which doesn't have the -s option
```

diff_hunk: @@ -215,12 +215,30 @@ def configure_emscripten_python(context, working_dir):
     exec_script = working_dir / "python.sh"
     exec_script.write_text(
         dedent(
-            f"""\
+            """\
             #!/bin/sh
 
+            # Macs come with free BSD coreutils which doesn't have the -s option
+            # so feature detect and work around it.
+            if which grealpath > /dev/null; then
+                # It has brew installed gnu core utils, use that
+                REALPATH="grealpath -s"
+            elif which realpath > /dev/null && realpath --version | grep GNU > /dev/null; then
+                # realpath points to GNU realpath so use it.
+                REALPATH="realpath -s"
+            else
+                # Shim for macs without GNU coreutils
+                abs_path () {
+                    echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")"
+                }
+                REALPATH=abs_path
+            fi
+            """
+            f"""
path: Tools/wasm/emscripten/__main__.py
text: @user1:
This 2 strings with implicit concatenation is a bit subtle.  Can we just do one string (which probably requires replacing `{` with `{{` above).