Repository: pyodide/pyodide

Issue:
pyodide-build test suite fails locally
I'm trying to run the pyodide-build test suite locally inside Docker,
```
source pyodide_env.sh
pytest pyodide-build
```

and so far it has had multiple failures,
```
FAILED pyodide-build/pyodide_build/tests/test_build_env.py::TestOutOfTree::get_build_environment_vars[node] - PermissionError: [Errno 13] Permission denied: '/packages'
FAILED pyodide-build/pyodide_build/tests/test_build_env.py::TestOutOfTree::get_build_flag[node] - PermissionError: [Errno 13] Permission denied: '/packages'
FAILED pyodide-build/pyodide_build/tests/test_build_env.py::TestOutOfTree::init_environment[node] - PermissionError: [Errno 13] Permission denied: '/packages'
FAILED pyodide-build/pyodide_build/tests/test_build_env.py::TestOutOfTree::get_pyodide_root[node] - PermissionError: [Errno 13] Permission denied: '/packages'
FAILED pyodide-build/pyodide_build/tests/test_build_env.py::TestOutOfTree::in_xbuildenv[node] - PermissionError: [Errno 13] Permission denied: '/packages'
FAILED pyodide-build/pyodide_build/tests/test_pypi.py::fetch_or_build_pypi[node] - AssertionError: * Creating virtualenv isolated environment...
FAILED pyodide-build/pyodide_build/tests/test_pypi.py::fetch_or_build_pypi_with_deps_and_extras[node] - AssertionError: Successfully fetched: eth_hash-0.5.2-py3-none-any.whl
FAILED pyodide-build/pyodide_build/tests/test_pypi.py::fake_pypi_succeed[node] - AssertionError: 127.0.0.1 - - [23/Jun/2023 14:06:56] "GET /simple/resolves-package/ HTTP/1.1" 200 -
FAILED pyodide-build/pyodide_build/tests/test_pypi.py::fake_pypi_extras_build[node] - AssertionError: 127.0.0.1 - - [23/Jun/2023 14:07:02] "GET /simple/pkg-b/ HTTP/1.1" 200 -
```

Maybe I'm doing something wrong or I should spend time reading through what we are doing in the CircleCI config but IMO the above should have worked, or else we need to update [developer instructions](https://pyodide.org/en/stable/development/testing.html#testing-and-benchmarking).


Anyway, I can probably figure it out by spending enough time on it, but we probably need to improve the docs as this is not very developer friendly.


Discussion:
Probably `PYODIDE_ROOT` is defined and empty for some reason?
What happens if you say `export PYODIDE_ROOT=$(pwd)` first?
Okay it still fails.
I think made this bug while working on #3922. Will open a PR that fix this.

Resolving patch:
diff --git a/pyodide-build/pyodide_build/build_env.py b/pyodide-build/pyodide_build/build_env.py
--- a/pyodide-build/pyodide_build/build_env.py
+++ b/pyodide-build/pyodide_build/build_env.py
@@ -199,6 +199,7 @@ def _get_make_environment_vars(*, pyodide_root: Path | None = None) -> dict[str,
         ["make", "-f", str(PYODIDE_ROOT / "Makefile.envs"), ".output_vars"],
         capture_output=True,
         text=True,
+        env={"PYODIDE_ROOT": str(PYODIDE_ROOT)},
     )
 
     if result.returncode != 0:


Test patch:
diff --git a/pyodide-build/pyodide_build/tests/fixture.py b/pyodide-build/pyodide_build/tests/fixture.py
--- a/pyodide-build/pyodide_build/tests/fixture.py
+++ b/pyodide-build/pyodide_build/tests/fixture.py
@@ -1,11 +1,14 @@
 import json
+import os
 import shutil
 from pathlib import Path
 from typing import Any
 
 import pytest
 
-from ..common import chdir
+from conftest import ROOT_PATH
+from pyodide_build import build_env
+from pyodide_build.common import chdir
 
 
 @pytest.fixture(scope="module")
@@ -89,3 +92,59 @@ def temp_xbuildenv(tmp_path_factory):
         archive_name = shutil.make_archive("xbuildenv", "tar")
 
     yield base, archive_name
+
+
+@pytest.fixture(scope="function")
+def reset_env_vars():
+    # Will reset the environment variables to their original values after each test.
+
+    os.environ.pop("PYODIDE_ROOT", None)
+    old_environ = dict(os.environ)
+
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_environ)
+
+
+@pytest.fixture(scope="function")
+def reset_cache():
+    # Will remove all caches before each test.
+
+    build_env.get_pyodide_root.cache_clear()
+    build_env.get_build_environment_vars.cache_clear()
+    build_env.get_unisolated_packages.cache_clear()
+
+    yield
+
+
+@pytest.fixture(scope="function")
+def xbuildenv(selenium, tmp_path, reset_env_vars, reset_cache):
+    import subprocess as sp
+
+    assert "PYODIDE_ROOT" not in os.environ
+
+    envpath = Path(tmp_path) / ".pyodide-xbuildenv"
+    result = sp.run(
+        [
+            "pyodide",
+            "xbuildenv",
+            "create",
+            str(envpath),
+            "--root",
+            ROOT_PATH,
+            "--skip-missing-files",
+        ]
+    )
+
+    assert result.returncode == 0
+
+    cur_dir = os.getcwd()
+
+    os.chdir(tmp_path)
+
+    try:
+        yield tmp_path
+    finally:
+        os.chdir(cur_dir)
diff --git a/pyodide-build/pyodide_build/tests/test_build_env.py b/pyodide-build/pyodide_build/tests/test_build_env.py
--- a/pyodide-build/pyodide_build/tests/test_build_env.py
+++ b/pyodide-build/pyodide_build/tests/test_build_env.py
@@ -1,40 +1,15 @@
+# flake8: noqa
 # This file contains tests that ensure build environment is properly initialized in
 # both in-tree and out-of-tree builds.
 
-# TODO: move functions that are tested here to a separate module
-
 import os
-from pathlib import Path
 
 import pytest
 
 from conftest import ROOT_PATH
 from pyodide_build import build_env, common
 
-
-@pytest.fixture(scope="function")
-def reset_env_vars():
-    # Will reset the environment variables to their original values after each test.
-
-    os.environ.pop("PYODIDE_ROOT", None)
-    old_environ = dict(os.environ)
-
-    try:
-        yield
-    finally:
-        os.environ.clear()
-        os.environ.update(old_environ)
-
-
-@pytest.fixture(scope="function")
-def reset_cache():
-    # Will remove all caches before each test.
-
-    build_env.get_pyodide_root.cache_clear()
-    build_env.get_build_environment_vars.cache_clear()
-    build_env.get_unisolated_packages.cache_clear()
-
-    yield
+from .fixture import reset_cache, reset_env_vars, xbuildenv
 
 
 class TestInTree:
@@ -96,6 +71,13 @@ def test_get_build_environment_vars(self, reset_env_vars, reset_cache):
         for var in extra_vars:
             assert var in build_vars, f"Missing {var}"
 
+    def test_get_make_environment_vars(self, reset_env_vars, reset_cache):
+        make_vars = build_env._get_make_environment_vars()
+        assert make_vars["PYODIDE_ROOT"] == str(ROOT_PATH)
+
+        make_vars = build_env._get_make_environment_vars(pyodide_root=ROOT_PATH)
+        assert make_vars["PYODIDE_ROOT"] == str(ROOT_PATH)
+
     def test_get_build_flag(self, reset_env_vars, reset_cache):
         for key, val in build_env.get_build_environment_vars().items():
             assert build_env.get_build_flag(key) == val
@@ -141,41 +123,6 @@ def test_get_build_environment_vars_host_env(
 
 
 class TestOutOfTree(TestInTree):
-    # TODO: selenium fixture is a hack to make these tests run only after building Pyodide.
-    @pytest.fixture(scope="function", autouse=True)
-    def xbuildenv(self, selenium, tmp_path, reset_env_vars, reset_cache):
-        import subprocess as sp
-
-        assert "PYODIDE_ROOT" not in os.environ
-
-        envpath = Path(tmp_path) / ".pyodide-xbuildenv"
-        result = sp.run(
-            [
-                "pyodide",
-                "xbuildenv",
-                "create",
-                str(envpath),
-                "--root",
-                ROOT_PATH,
-                "--skip-missing-files",
-            ]
-        )
-
-        assert result.returncode == 0
-
-        yield tmp_path
-
-    @pytest.fixture(scope="function", autouse=True)
-    def chdir_xbuildenv(self, xbuildenv):
-        cur_dir = os.getcwd()
-
-        os.chdir(xbuildenv)
-
-        try:
-            yield
-        finally:
-            os.chdir(cur_dir)
-
     # Note: other tests are inherited from TestInTree
 
     def test_init_environment(self, xbuildenv, reset_env_vars, reset_cache):
@@ -196,9 +143,17 @@ def test_get_pyodide_root(self, xbuildenv, reset_env_vars, reset_cache):
             == xbuildenv / ".pyodide-xbuildenv/xbuildenv/pyodide-root"
         )
 
-    def test_in_xbuildenv(self, reset_env_vars, reset_cache):
+    def test_in_xbuildenv(self, xbuildenv, reset_env_vars, reset_cache):
         assert build_env.in_xbuildenv()
 
+    def test_get_make_environment_vars(self, xbuildenv, reset_env_vars, reset_cache):
+        xbuildenv_root = xbuildenv / ".pyodide-xbuildenv/xbuildenv/pyodide-root"
+        make_vars = build_env._get_make_environment_vars()
+        assert make_vars["PYODIDE_ROOT"] == str(xbuildenv_root)
+
+        make_vars = build_env._get_make_environment_vars(pyodide_root=xbuildenv_root)
+        assert make_vars["PYODIDE_ROOT"] == str(xbuildenv_root)
+
 
 def test_check_emscripten_version(monkeypatch):
     s = None
diff --git a/pyodide-build/pyodide_build/tests/test_pypi.py b/pyodide-build/pyodide_build/tests/test_pypi.py
--- a/pyodide-build/pyodide_build/tests/test_pypi.py
+++ b/pyodide-build/pyodide_build/tests/test_pypi.py
@@ -1,4 +1,5 @@
-import os
+# flake8: noqa
+
 import re
 import subprocess
 import sys
@@ -14,7 +15,7 @@
 from typer.testing import CliRunner
 
 from pyodide_build.cli import build
-from pyodide_build.common import chdir
+from .fixture import reset_cache, reset_env_vars, xbuildenv
 
 runner = CliRunner()
 
@@ -213,16 +214,15 @@ def fake_pypi_url(fake_pypi_server):
     pyodide_build.out_of_tree.pypi._PYPI_INDEX = pypi_old
 
 
-def test_fetch_or_build_pypi(selenium, tmp_path):
+def test_fetch_or_build_pypi(xbuildenv):
     # TODO: - make test run without pyodide
-    output_dir = tmp_path / "dist"
+    output_dir = xbuildenv / "dist"
     # one pure-python package (doesn't need building) and one sdist package (needs building)
     pkgs = ["pytest-pyodide", "pycryptodome==3.15.0"]
 
     app = typer.Typer()
     app.command()(build.main)
 
-    os.chdir(tmp_path)
     for p in pkgs:
         result = runner.invoke(
             app,
@@ -234,16 +234,15 @@ def test_fetch_or_build_pypi(selenium, tmp_path):
     assert len(built_wheels) == len(pkgs)
 
 
-def test_fetch_or_build_pypi_with_deps_and_extras(selenium, tmp_path):
+def test_fetch_or_build_pypi_with_deps_and_extras(xbuildenv):
     # TODO: - make test run without pyodide
-    output_dir = tmp_path / "dist"
+    output_dir = xbuildenv / "dist"
     # one pure-python package (doesn't need building) which depends on one sdist package (needs building)
     pkgs = ["eth-hash[pycryptodome]==0.5.1", "safe-pysha3 (>=1.0.0)"]
 
     app = typer.Typer()
     app.command()(build.main)
 
-    os.chdir(tmp_path)
     for p in pkgs:
         result = runner.invoke(
             app,
@@ -255,38 +254,36 @@ def test_fetch_or_build_pypi_with_deps_and_extras(selenium, tmp_path):
     assert len(built_wheels) == 3
 
 
-def test_fake_pypi_succeed(selenium, tmp_path, fake_pypi_url):
+def test_fake_pypi_succeed(xbuildenv, fake_pypi_url):
     # TODO: - make test run without pyodide
-    output_dir = tmp_path / "dist"
+    output_dir = xbuildenv / "dist"
     # build package that resolves right
     app = typer.Typer()
     app.command()(build.main)
 
-    with chdir(tmp_path):
-        result = runner.invoke(
-            app,
-            ["resolves-package", "--build-dependencies"],
-        )
+    result = runner.invoke(
+        app,
+        ["resolves-package", "--build-dependencies"],
+    )
 
-        assert result.exit_code == 0, str(result.stdout) + str(result)
+    assert result.exit_code == 0, str(result.stdout) + str(result)
 
     built_wheels = set(output_dir.glob("*.whl"))
     assert len(built_wheels) == 5
 
 
-def test_fake_pypi_resolve_fail(selenium, tmp_path, fake_pypi_url):
-    # TODO: - make test run without pyodide
-    output_dir = tmp_path / "dist"
+def test_fake_pypi_resolve_fail(xbuildenv, fake_pypi_url):
+    output_dir = xbuildenv / "dist"
+
     # build package that resolves right
 
     app = typer.Typer()
     app.command()(build.main)
 
-    with chdir(tmp_path):
-        result = runner.invoke(
-            app,
-            ["fails-package", "--build-dependencies"],
-        )
+    result = runner.invoke(
+        app,
+        ["fails-package", "--build-dependencies"],
+    )
 
     # this should fail and should not build any wheels
     assert result.exit_code != 0, result.stdout
@@ -294,18 +291,17 @@ def test_fake_pypi_resolve_fail(selenium, tmp_path, fake_pypi_url):
     assert len(built_wheels) == 0
 
 
-def test_fake_pypi_extras_build(selenium, tmp_path, fake_pypi_url):
+def test_fake_pypi_extras_build(xbuildenv, fake_pypi_url):
     # TODO: - make test run without pyodide
-    output_dir = tmp_path / "dist"
+    output_dir = xbuildenv / "dist"
     # build package that resolves right
     app = typer.Typer()
     app.command()(build.main)
 
-    with chdir(tmp_path):
-        result = runner.invoke(
-            app,
-            ["pkg-b[docs]", "--build-dependencies"],
-        )
+    result = runner.invoke(
+        app,
+        ["pkg-b[docs]", "--build-dependencies"],
+    )
 
     # this should work
     assert result.exit_code == 0, result.stdout
@@ -313,16 +309,16 @@ def test_fake_pypi_extras_build(selenium, tmp_path, fake_pypi_url):
     assert len(built_wheels) == 2
 
 
-def test_fake_pypi_repeatable_build(selenium, tmp_path, fake_pypi_url):
-    # TODO: - make test run without pyodide
-    output_dir = tmp_path / "dist"
+def test_fake_pypi_repeatable_build(xbuildenv, fake_pypi_url):
+    output_dir = xbuildenv / "dist"
+
     # build package that resolves right
     app = typer.Typer()
     app.command()(build.main)
 
     # override a dependency version and build
     # pkg-a
-    with open(tmp_path / "requirements.txt", "w") as req_file:
+    with open("requirements.txt", "w") as req_file:
         req_file.write(
             """
 # Whole line comment
@@ -330,17 +326,17 @@ def test_fake_pypi_repeatable_build(selenium, tmp_path, fake_pypi_url):
 pkg-a
             """
         )
-    with chdir(tmp_path):
-        result = runner.invoke(
-            app,
-            [
-                "-r",
-                "requirements.txt",
-                "--build-dependencies",
-                "--output-lockfile",
-                "lockfile.txt",
-            ],
-        )
+
+    result = runner.invoke(
+        app,
+        [
+            "-r",
+            "requirements.txt",
+            "--build-dependencies",
+            "--output-lockfile",
+            "lockfile.txt",
+        ],
+    )
     # this should work
     assert result.exit_code == 0, result.stdout
     built_wheels = list(output_dir.glob("*.whl"))
@@ -354,11 +350,10 @@ def test_fake_pypi_repeatable_build(selenium, tmp_path, fake_pypi_url):
 
     # rebuild from package-versions lockfile and
     # check it outputs the same version number
-    with chdir(tmp_path):
-        result = runner.invoke(
-            app,
-            ["-r", str(tmp_path / "lockfile.txt")],
-        )
+    result = runner.invoke(
+        app,
+        ["-r", "lockfile.txt"],
+    )
 
     # should still have built 1.0.0 of pkg-c
     built_wheels = list(output_dir.glob("*.whl"))
@@ -369,7 +364,7 @@ def test_fake_pypi_repeatable_build(selenium, tmp_path, fake_pypi_url):
     assert len(built_wheels) == 2, result.stdout
 
 
-def test_bad_requirements_text(selenium, tmp_path):
+def test_bad_requirements_text(xbuildenv):
     app = typer.Typer()
     app.command()(build.main)
     # test 1 - error on URL location in requirements
@@ -377,11 +372,11 @@ def test_bad_requirements_text(selenium, tmp_path):
     # test 3 - error on editable install of package
     bad_lines = [" pkg-c@http://www.pkg-c.org", "  -r bob.txt", "   -e pkg-c"]
     for line in bad_lines:
-        with open(tmp_path / "requirements.txt", "w") as req_file:
+        with open("requirements.txt", "w") as req_file:
             req_file.write(line + "\n")
-        with chdir(tmp_path):
-            result = runner.invoke(
-                app,
-                ["-r", "requirements.txt"],
-            )
-            assert result.exit_code != 0 and line.strip() in str(result)
+
+        result = runner.invoke(
+            app,
+            ["-r", "requirements.txt"],
+        )
+        assert result.exit_code != 0 and line.strip() in str(result)