Repository: pyodide/pyodide
Pull request: Add url parameter to install_xbuildenv
### Description

Resolve #3567 

Adds url paramter to `install_xbuildenv`, which helps installing xbuildenv from arbitrary URL. This is mostly for developers / maintainers who want to test tot xbuildenv.

This PR also adds a private CLI entrypoint `pyodide xbuildenv install`.

### Checklists

- [x] Add / update tests


Problem statement:
Option to install arbitrary version of xbuildenv for debug purpose
When testing out-of-tree builds with tot pyodide-build, it is sometimes annoying to create a cross-build environment.

So I think it would be nice to have the option to install an arbitrary version of cross-build env, such as:

```
pyodide install-xbuildenv --version x.x.x
```

This should be a private entry point only for maintainers/contributors, since installing an incompatible version of cross-build env will definitely break the build. Those who use this entry point should know what they are doing.

Hints from issue discussion:

Commit message: Revert changes in initialize_pyodide_root

Patch to review:
diff --git a/pyodide-build/pyodide_build/cli/xbuildenv.py b/pyodide-build/pyodide_build/cli/xbuildenv.py
new file mode 100644
index 00000000000..36e7ccaa725
--- /dev/null
+++ b/pyodide-build/pyodide_build/cli/xbuildenv.py
@@ -0,0 +1,32 @@
+from pathlib import Path
+
+import typer
+
+from ..install_xbuildenv import install
+
+app = typer.Typer(hidden=True)
+
+
+@app.callback()
+def callback():
+    """
+    Create or install cross build environment
+    """
+
+
+@app.command("install")  # type: ignore[misc]
+def _install(
+    path: Path = typer.Option(".pyodide-xbuildenv", help="path to xbuildenv directory"),
+    download: bool = typer.Option(False, help="download xbuildenv before installing"),
+    url: str = typer.Option(None, help="URL to download xbuildenv from"),
+) -> None:
+    """
+    Install xbuildenv.
+
+    The isntalled environment is the same as the one that would result from
+    `PYODIDE_PACKAGES='scipy' make` except that it is much faster.
+    The goal is to enable out-of-tree builds for binary packages that depend
+    on numpy or scipy.
+    Note: this is a private endpoint that should not be used outside of the Pyodide Makefile.
+    """
+    install(path, download=download, url=url)
diff --git a/pyodide-build/pyodide_build/install_xbuildenv.py b/pyodide-build/pyodide_build/install_xbuildenv.py
index 23e6a68d91f..a1e4419e5c4 100644
--- a/pyodide-build/pyodide_build/install_xbuildenv.py
+++ b/pyodide-build/pyodide_build/install_xbuildenv.py
@@ -1,11 +1,12 @@
 import argparse
 import json
+import os
 import shutil
 import subprocess
 from pathlib import Path
 from urllib.request import urlopen, urlretrieve
 
-from .common import exit_with_stdio, get_make_flag, get_pyodide_root
+from .common import exit_with_stdio, get_make_flag
 from .create_pypa_index import create_pypa_index
 from .logger import logger
 
@@ -20,19 +21,27 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
         "Note: this is a private endpoint that should not be used outside of the Pyodide Makefile."
     )
     parser.add_argument("--download", action="store_true", help="Download xbuild env")
+    parser.add_argument("--url", help="URL to download xbuild env from", default=None)
     parser.add_argument("xbuildenv", type=str, nargs=1)
     return parser
 
 
-def download_xbuildenv(version: str, xbuildenv_path: Path) -> None:
+def download_xbuildenv(
+    version: str, xbuildenv_path: Path, *, url: str | None = None
+) -> None:
     from shutil import rmtree, unpack_archive
     from tempfile import NamedTemporaryFile
 
     logger.info("Downloading xbuild environment")
     rmtree(xbuildenv_path, ignore_errors=True)
+
+    xbuildenv_url = (
+        url
+        or f"https://github.com/pyodide/pyodide/releases/download/{version}/xbuildenv-{version}.tar.bz2"
+    )
     with NamedTemporaryFile(suffix=".tar") as f:
         urlretrieve(
-            f"https://github.com/pyodide/pyodide/releases/download/{version}/xbuildenv-{version}.tar.bz2",
+            xbuildenv_url,
             f.name,
         )
         unpack_archive(f.name, xbuildenv_path)
@@ -40,12 +49,13 @@ def download_xbuildenv(version: str, xbuildenv_path: Path) -> None:
 
 def install_xbuildenv(version: str, xbuildenv_path: Path) -> None:
     logger.info("Installing xbuild environment")
+
     xbuildenv_path = xbuildenv_path / "xbuildenv"
-    pyodide_root = get_pyodide_root()
     xbuildenv_root = xbuildenv_path / "pyodide-root"
-    host_site_packages = xbuildenv_root / Path(
-        get_make_flag("HOSTSITEPACKAGES")
-    ).relative_to(pyodide_root)
+
+    os.environ["PYODIDE_ROOT"] = str(xbuildenv_root)
+
+    host_site_packages = Path(get_make_flag("HOSTSITEPACKAGES"))
     host_site_packages.mkdir(exist_ok=True, parents=True)
     result = subprocess.run(
         [
@@ -79,11 +89,14 @@ def install_xbuildenv(version: str, xbuildenv_path: Path) -> None:
     create_pypa_index(repodata["packages"], xbuildenv_root, cdn_base)
 
 
-def main(args: argparse.Namespace) -> None:
+def install(path: Path, *, download: bool = False, url: str | None = None) -> None:
     from . import __version__
 
-    xbuildenv_path = Path(args.xbuildenv[0])
     version = __version__
-    if args.download:
-        download_xbuildenv(version, xbuildenv_path)
-    install_xbuildenv(version, xbuildenv_path)
+    if download:
+        download_xbuildenv(version, path, url=url)
+    install_xbuildenv(version, path)
+
+
+def main(args: argparse.Namespace) -> None:
+    install(Path(args.xbuildenv[0]), download=args.download, url=args.url)
diff --git a/pyodide-build/pyodide_build/tests/test_cli.py b/pyodide-build/pyodide_build/tests/test_cli.py
index 75e30033ba8..4d05abedd34 100644
--- a/pyodide-build/pyodide_build/tests/test_cli.py
+++ b/pyodide-build/pyodide_build/tests/test_cli.py
@@ -9,7 +9,14 @@
 from typer.testing import CliRunner  # type: ignore[import]
 
 from pyodide_build import common
-from pyodide_build.cli import build, build_recipes, config, create_zipfile, skeleton
+from pyodide_build.cli import (
+    build,
+    build_recipes,
+    config,
+    create_zipfile,
+    skeleton,
+    xbuildenv,
+)
 
 from .fixture import temp_python_lib
 
@@ -328,3 +335,24 @@ def test_create_zipfile_compile(temp_python_lib, tmp_path):
     with ZipFile(output) as zf:
         assert "module1.pyc" in zf.namelist()
         assert "module2.pyc" in zf.namelist()
+
+
+def test_xbuildenv_install(tmp_path):
+    envpath = Path(tmp_path) / ".xbuildenv"
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "install",
+            "--path",
+            str(envpath),
+            "--download",
+            "--url",
+            "https://github.com/pyodide/pyodide/releases/download/0.22.1/xbuildenv-0.22.1.tar.bz2",
+        ],
+    )
+
+    assert result.exit_code == 0, result.stdout
+    assert "Downloading xbuild environment" in result.stdout, result.stdout
+    assert "Installing xbuild environment" in result.stdout, result.stdout
+    assert (envpath / "xbuildenv" / "pyodide-root").is_dir()
+    assert (envpath / "xbuildenv" / "site-packages-extras").is_dir()
diff --git a/pyodide-build/setup.cfg b/pyodide-build/setup.cfg
index 43ff2b3adc9..c1957ea3c58 100644
--- a/pyodide-build/setup.cfg
+++ b/pyodide-build/setup.cfg
@@ -54,6 +54,7 @@ pyodide.cli =
     py-compile = pyodide_build.cli.py_compile:main
     config = pyodide_build.cli.config:app
     create-zipfile = pyodide_build.cli.create_zipfile:main
+    xbuildenv = pyodide_build.cli.xbuildenv:app
 
 [options.extras_require]
 test =


Code review comments:
diff_hunk: @@ -328,3 +335,24 @@ def test_create_zipfile_compile(temp_python_lib, tmp_path):
     with ZipFile(output) as zf:
         assert "module1.pyc" in zf.namelist()
         assert "module2.pyc" in zf.namelist()
+
+
+def test_xbuildenv_install(tmp_path):
+    envpath = Path(tmp_path) / ".xbuildenv"
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "install",
+            "--path",
+            str(envpath),
+            "--download",
+            "--url",
+            "https://github.com/pyodide/pyodide/releases/download/0.22.1/xbuildenv-0.22.1.tar.bz2",
path: pyodide-build/pyodide_build/tests/test_cli.py
text: @user1:
It's a 10MB file, not sure about downloading it each time this test is run. At least maybe we could put it behind some pytest mark and not run it by default?

If we cannot create a similar archive for testing at runtime that is.

@author:
> If we cannot create a similar archive for testing at runtime that is.

I'll try this approach.