Repository: pyodide/pyodide

Issue:
Option to install arbitrary version of xbuildenv for debug purpose
When testing out-of-tree builds with tot pyodide-build, it is sometimes annoying to create a cross-build environment.

So I think it would be nice to have the option to install an arbitrary version of cross-build env, such as:

```
pyodide install-xbuildenv --version x.x.x
```

This should be a private entry point only for maintainers/contributors, since installing an incompatible version of cross-build env will definitely break the build. Those who use this entry point should know what they are doing.



Discussion:

Resolving patch:
diff --git a/pyodide-build/pyodide_build/cli/xbuildenv.py b/pyodide-build/pyodide_build/cli/xbuildenv.py
new file mode 100644
--- /dev/null
+++ b/pyodide-build/pyodide_build/cli/xbuildenv.py
@@ -0,0 +1,32 @@
+from pathlib import Path
+
+import typer
+
+from ..install_xbuildenv import install
+
+app = typer.Typer(hidden=True)
+
+
+@app.callback()
+def callback():
+    """
+    Create or install cross build environment
+    """
+
+
+@app.command("install")  # type: ignore[misc]
+def _install(
+    path: Path = typer.Option(".pyodide-xbuildenv", help="path to xbuildenv directory"),
+    download: bool = typer.Option(False, help="download xbuildenv before installing"),
+    url: str = typer.Option(None, help="URL to download xbuildenv from"),
+) -> None:
+    """
+    Install xbuildenv.
+
+    The isntalled environment is the same as the one that would result from
+    `PYODIDE_PACKAGES='scipy' make` except that it is much faster.
+    The goal is to enable out-of-tree builds for binary packages that depend
+    on numpy or scipy.
+    Note: this is a private endpoint that should not be used outside of the Pyodide Makefile.
+    """
+    install(path, download=download, url=url)
diff --git a/pyodide-build/pyodide_build/install_xbuildenv.py b/pyodide-build/pyodide_build/install_xbuildenv.py
--- a/pyodide-build/pyodide_build/install_xbuildenv.py
+++ b/pyodide-build/pyodide_build/install_xbuildenv.py
@@ -1,11 +1,12 @@
 import argparse
 import json
+import os
 import shutil
 import subprocess
 from pathlib import Path
 from urllib.request import urlopen, urlretrieve
 
-from .common import exit_with_stdio, get_make_flag, get_pyodide_root
+from .common import exit_with_stdio, get_make_flag
 from .create_pypa_index import create_pypa_index
 from .logger import logger
 
@@ -20,19 +21,27 @@ def make_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
         "Note: this is a private endpoint that should not be used outside of the Pyodide Makefile."
     )
     parser.add_argument("--download", action="store_true", help="Download xbuild env")
+    parser.add_argument("--url", help="URL to download xbuild env from", default=None)
     parser.add_argument("xbuildenv", type=str, nargs=1)
     return parser
 
 
-def download_xbuildenv(version: str, xbuildenv_path: Path) -> None:
+def download_xbuildenv(
+    version: str, xbuildenv_path: Path, *, url: str | None = None
+) -> None:
     from shutil import rmtree, unpack_archive
     from tempfile import NamedTemporaryFile
 
     logger.info("Downloading xbuild environment")
     rmtree(xbuildenv_path, ignore_errors=True)
+
+    xbuildenv_url = (
+        url
+        or f"https://github.com/pyodide/pyodide/releases/download/{version}/xbuildenv-{version}.tar.bz2"
+    )
     with NamedTemporaryFile(suffix=".tar") as f:
         urlretrieve(
-            f"https://github.com/pyodide/pyodide/releases/download/{version}/xbuildenv-{version}.tar.bz2",
+            xbuildenv_url,
             f.name,
         )
         unpack_archive(f.name, xbuildenv_path)
@@ -40,12 +49,13 @@ def download_xbuildenv(version: str, xbuildenv_path: Path) -> None:
 
 def install_xbuildenv(version: str, xbuildenv_path: Path) -> None:
     logger.info("Installing xbuild environment")
+
     xbuildenv_path = xbuildenv_path / "xbuildenv"
-    pyodide_root = get_pyodide_root()
     xbuildenv_root = xbuildenv_path / "pyodide-root"
-    host_site_packages = xbuildenv_root / Path(
-        get_make_flag("HOSTSITEPACKAGES")
-    ).relative_to(pyodide_root)
+
+    os.environ["PYODIDE_ROOT"] = str(xbuildenv_root)
+
+    host_site_packages = Path(get_make_flag("HOSTSITEPACKAGES"))
     host_site_packages.mkdir(exist_ok=True, parents=True)
     result = subprocess.run(
         [
@@ -68,8 +78,8 @@ def install_xbuildenv(version: str, xbuildenv_path: Path) -> None:
         xbuildenv_path / "site-packages-extras", host_site_packages, dirs_exist_ok=True
     )
     cdn_base = f"https://cdn.jsdelivr.net/pyodide/v{version}/full/"
-    if (xbuildenv_root / "repodata.json").exists():
-        repodata_bytes = (xbuildenv_root / "repodata.json").read_bytes()
+    if (repodata_json := xbuildenv_root / "dist" / "repodata.json").exists():
+        repodata_bytes = repodata_json.read_bytes()
     else:
         repodata_url = cdn_base + "repodata.json"
         with urlopen(repodata_url) as response:
@@ -79,11 +89,33 @@ def install_xbuildenv(version: str, xbuildenv_path: Path) -> None:
     create_pypa_index(repodata["packages"], xbuildenv_root, cdn_base)
 
 
-def main(args: argparse.Namespace) -> None:
+def install(path: Path, *, download: bool = False, url: str | None = None) -> None:
+    """
+    Install cross-build environment.
+
+    Parameters
+    ----------
+    path
+        A path to the cross-build environment.
+    download
+        Whether to download the cross-build environment before installing it.
+    url
+        URL to download the cross-build environment from. This is only used
+        if `download` is True. The URL should point to a tarball containing
+        the cross-build environment. If not specified, the corresponding
+        release on GitHub is used.
+
+        Warning: if you are downloading from a version that is not the same
+        as the current version of pyodide-build, make sure that the cross-build
+        environment is compatible with the current version of Pyodide.
+    """
     from . import __version__
 
-    xbuildenv_path = Path(args.xbuildenv[0])
     version = __version__
-    if args.download:
-        download_xbuildenv(version, xbuildenv_path)
-    install_xbuildenv(version, xbuildenv_path)
+    if download:
+        download_xbuildenv(version, path, url=url)
+    install_xbuildenv(version, path)
+
+
+def main(args: argparse.Namespace) -> None:
+    install(Path(args.xbuildenv[0]), download=args.download, url=args.url)


Test patch:
diff --git a/pyodide-build/pyodide_build/tests/fixture.py b/pyodide-build/pyodide_build/tests/fixture.py
--- a/pyodide-build/pyodide_build/tests/fixture.py
+++ b/pyodide-build/pyodide_build/tests/fixture.py
@@ -1,7 +1,12 @@
+import json
+import shutil
 from pathlib import Path
+from typing import Any
 
 import pytest
 
+from ..common import chdir
+
 
 @pytest.fixture(scope="module")
 def temp_python_lib(tmp_path_factory):
@@ -20,3 +25,53 @@ def temp_python_lib(tmp_path_factory):
     (path / "hello_pyodide.py").write_text("def hello(): return 'hello'")
 
     yield libdir
+
+
+def mock_repodata_json() -> dict[str, Any]:
+    # TODO: use pydantic
+
+    return {
+        "info": {
+            "version": "0.22.1",
+        },
+        "packages": {},
+    }
+
+
+@pytest.fixture(scope="module")
+def temp_xbuildenv(tmp_path_factory):
+    """
+    Create a temporary xbuildenv archive
+    """
+    base = tmp_path_factory.mktemp("base")
+
+    path = Path(base)
+
+    xbuildenv = path / "xbuildenv"
+    xbuildenv.mkdir()
+
+    pyodide_root = xbuildenv / "pyodide-root"
+    site_packages_extra = xbuildenv / "site-packages-extras"
+    requirements_txt = xbuildenv / "requirements.txt"
+
+    pyodide_root.mkdir()
+    site_packages_extra.mkdir()
+    requirements_txt.touch()
+
+    (pyodide_root / "Makefile.envs").write_text(
+        """
+export HOSTSITEPACKAGES=$(PYODIDE_ROOT)/packages/.artifacts/lib/python$(PYMAJOR).$(PYMINOR)/site-packages
+
+.output_vars:
+	set
+"""
+    )
+    (pyodide_root / "dist").mkdir()
+    (pyodide_root / "dist" / "repodata.json").write_text(
+        json.dumps(mock_repodata_json())
+    )
+
+    with chdir(base):
+        archive_name = shutil.make_archive("xbuildenv", "tar")
+
+    yield base, archive_name
diff --git a/pyodide-build/pyodide_build/tests/test_cli.py b/pyodide-build/pyodide_build/tests/test_cli.py
--- a/pyodide-build/pyodide_build/tests/test_cli.py
+++ b/pyodide-build/pyodide_build/tests/test_cli.py
@@ -5,13 +5,21 @@
 from pathlib import Path
 
 import pytest
+from pytest_pyodide import spawn_web_server
 import typer
 from typer.testing import CliRunner  # type: ignore[import]
 
 from pyodide_build import common
-from pyodide_build.cli import build, build_recipes, config, create_zipfile, skeleton
+from pyodide_build.cli import (
+    build,
+    build_recipes,
+    config,
+    create_zipfile,
+    skeleton,
+    xbuildenv,
+)
 
-from .fixture import temp_python_lib
+from .fixture import temp_python_lib, temp_xbuildenv
 
 only_node = pytest.mark.xfail_browsers(
     chrome="node only", firefox="node only", safari="node only"
@@ -328,3 +336,30 @@ def test_create_zipfile_compile(temp_python_lib, tmp_path):
     with ZipFile(output) as zf:
         assert "module1.pyc" in zf.namelist()
         assert "module2.pyc" in zf.namelist()
+
+
+def test_xbuildenv_install(tmp_path, temp_xbuildenv):
+    envpath = Path(tmp_path) / ".xbuildenv"
+
+    xbuildenv_url_base, xbuildenv_filename = temp_xbuildenv
+
+    with spawn_web_server(xbuildenv_url_base) as (hostname, port, _):
+        xbuildenv_url = f"http://{hostname}:{port}/{xbuildenv_filename}"
+        result = runner.invoke(
+            xbuildenv.app,
+            [
+                "install",
+                "--path",
+                str(envpath),
+                "--download",
+                "--url",
+                xbuildenv_url,
+            ],
+        )
+
+    assert result.exit_code == 0, result.stdout
+    assert "Downloading xbuild environment" in result.stdout, result.stdout
+    assert "Installing xbuild environment" in result.stdout, result.stdout
+    assert (envpath / "xbuildenv" / "pyodide-root").is_dir()
+    assert (envpath / "xbuildenv" / "site-packages-extras").is_dir()
+    assert (envpath / "xbuildenv" / "requirements.txt").exists()