internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: pyodide/pyodide

Issue:
Remove loadPackagesFromImports from runPythonAsync?
This would be a backwards incompatible API change but I think it's for the best.


Discussion:

Resolving patch:
diff --git a/conftest.py b/conftest.py
--- a/conftest.py
+++ b/conftest.py
@@ -181,6 +181,7 @@ def run(self, code):
     def run_async(self, code):
         return self.run_js(
             f"""
+            await pyodide.loadPackagesFromImports({code!r})
             let result = await pyodide.runPythonAsync({code!r});
             if(result && result.toJs){{
                 let converted_result = result.toJs();


Test patch:
diff --git a/packages/micropip/test_micropip.py b/packages/micropip/test_micropip.py
--- a/packages/micropip/test_micropip.py
+++ b/packages/micropip/test_micropip.py
@@ -14,7 +14,7 @@ def selenium_standalone_micropip(selenium_standalone):
     selenium_standalone.run_js(
         """
         await pyodide.loadPackage("micropip");
-        await pyodide.runPythonAsync("import micropip");
+        await pyodide.runPython("import micropip");
         """
     )
     yield selenium_standalone
diff --git a/packages/numpy/test_numpy.py b/packages/numpy/test_numpy.py
--- a/packages/numpy/test_numpy.py
+++ b/packages/numpy/test_numpy.py
@@ -197,7 +197,8 @@ def test_runwebworker_numpy(selenium_webworker_standalone):
 def test_get_buffer(selenium):
     selenium.run_js(
         """
-        await pyodide.runPythonAsync(`
+        await pyodide.loadPackage(['numpy']);
+        await pyodide.runPython(`
             import numpy as np
             x = np.arange(24)
             z1 = x.reshape([8,3])
@@ -246,7 +247,8 @@ def test_get_buffer(selenium):
 def test_get_buffer_roundtrip(selenium, arg):
     selenium.run_js(
         f"""
-        await pyodide.runPythonAsync(`
+        await pyodide.loadPackage(['numpy']);
+        await pyodide.runPython(`
             import numpy as np
             x = {arg}
         `);
@@ -288,7 +290,8 @@ def test_get_buffer_roundtrip(selenium, arg):
 def test_get_buffer_big_endian(selenium):
     selenium.run_js(
         """
-        window.a = await pyodide.runPythonAsync(`
+        await pyodide.loadPackage(['numpy']);
+        window.a = await pyodide.runPython(`
             import numpy as np
             np.arange(24, dtype="int16").byteswap().newbyteorder()
         `);
@@ -315,7 +318,8 @@ def test_get_buffer_error_messages(selenium):
     with pytest.raises(Exception, match="Javascript has no Float16 support"):
         selenium.run_js(
             """
-            await pyodide.runPythonAsync(`
+            await pyodide.loadPackage(['numpy']);
+            await pyodide.runPython(`
                 import numpy as np
                 x = np.ones(2, dtype=np.float16)
             `);
Repository: pyodide/pyodide

Issue:
exception in asyncio.wait_for()
## 🐛 Bug

Waiting for an awaitable using `asyncio.wait_for()` prints exceptions to the browser's console:

```
pyodide.asm.js:14 Exception in callback None()
pyodide.asm.js:14 handle: <Handle cancelled>
```

This happens whether the awaitable finishes before the timeout or not. `asyncio.wait_for()` appears to still function as intended, but this makes applications that use `asyncio.wait_for()` frequently very spammy.

### To Reproduce

In [Pyodide console application](https://pyodide.org/en/stable/console.html), enter the following:

```
import asyncio
await asyncio.wait_for(asyncio.sleep(1), 2)
```

### Expected behavior

No exceptions are raised and no output is printed to the console.

### Environment

- Pyodide Version: 0.18.1 from https://cdn.jsdelivr.net/pyodide/v0.18.1/full/
- Browser version: Chrome 96.0.4664.55
- Any other relevant information:




Discussion:
Thanks for the report! This is probably a bug in our event loop.

Resolving patch:
diff --git a/src/py/pyodide/webloop.py b/src/py/pyodide/webloop.py
--- a/src/py/pyodide/webloop.py
+++ b/src/py/pyodide/webloop.py
@@ -143,7 +143,13 @@ def call_later(
         if delay < 0:
             raise ValueError("Can't schedule in the past")
         h = asyncio.Handle(callback, args, self, context=context)
-        setTimeout(create_once_callable(h._run), delay * 1000)
+
+        def run_handle():
+            if h.cancelled():
+                return
+            h._run()
+
+        setTimeout(create_once_callable(run_handle), delay * 1000)
         return h
 
     def call_at(


Test patch:
diff --git a/src/tests/test_webloop.py b/src/tests/test_webloop.py
--- a/src/tests/test_webloop.py
+++ b/src/tests/test_webloop.py
@@ -33,6 +33,26 @@ async def sleep_task():
     )
 
 
+def test_cancel_handle(selenium_standalone):
+    selenium_standalone.run_js(
+        """
+        await pyodide.runPythonAsync(`
+            import asyncio
+            loop = asyncio.get_event_loop()
+            exc = []
+            def exception_handler(loop, context):
+                exc.append(context)
+            loop.set_exception_handler(exception_handler)
+            try:
+                await asyncio.wait_for(asyncio.sleep(1), 2)
+            finally:
+                loop.set_exception_handler(None)
+            assert not exc
+        `);
+        """
+    )
+
+
 def test_return_result(selenium):
     # test return result
     run_with_resolve(
@@ -172,7 +192,9 @@ def f():
     )
 
 
-def test_webloop_exception_handler(selenium):
+@pytest.mark.xfail(reason="Works locally but failing in test suite as of #2022.")
+def test_webloop_exception_handler(selenium_standalone):
+    selenium = selenium_standalone
     selenium.run_async(
         """
         import asyncio
Repository: pyodide/pyodide

Issue:
0.19 release
Opening an issue to track the 0.19 release.

I think we are fairly ready to make the 0.19.0 release. Ideally, the following would be nice to do before the 0.19 alpha release (but not critical),
 - update browser versions used for testing https://github.com/pyodide/pyodide/pull/1952 
 - which should hopefully unblock emscripten update https://github.com/pyodide/pyodide/pull/2035
 - run the update `meta.yaml` script for all pure python packages 

For instance, maybe we could plan to have a
 - A release candidate 0.19.0rc0: 2021/12/22 or 23  -> to make it easier still use the main branch up to the final release
 - A final release 0.19.0: 2021/12/30
 
?

cc @hoodmane 


Discussion:
We have some new APIs that are related to downloading files/scripts and importing them.

- pyodide.http.pyfetch (#1865)
- pyodide.unpackArchive (#2018)
- pyodide.pyimport (#1944)

These APIs can benefit many users, so I think it would be great to update documentation of using those APIs before releasing 0.19.

Maybe we could have two new sections about:

- File download (ongoing PR?: #1861)
- Importing python script


> These APIs can benefit many users, so I think it would be great to update documentation of using those APIs before releasing 0.19.

Yes, definitely. Thanks for mentioning it @ryanking13 !
The 0.19.0a1 was released by @hoodmane yesterday, but we probably still address some of the above topics before the release, and certainly the documentation.
I think we are good to go for the 0.19.0 release. The only remaining thing is to write a short blog post with major highlights from this release. If you has bandwidth @hoodmane @ryanking13 feel free to start it, otherwise I can look into it tomorrow.
Okay, I will try to get to it.
Maybe I will update the roadmap a bit too.
Okay I made a very rough sketch of some release notes:
https://github.com/pyodide/pyodide-blog/pull/18

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -127,7 +127,7 @@
 epub_exclude_files = ["search.html"]
 
 if "READTHEDOCS" in os.environ:
-    env = {"PYODIDE_BASE_URL": "https://cdn.jsdelivr.net/pyodide/dev/full/"}
+    env = {"PYODIDE_BASE_URL": "https://cdn.jsdelivr.net/pyodide/v0.19.0/full/"}
     os.makedirs("_build/html", exist_ok=True)
     res = subprocess.check_output(
         ["make", "-C", "..", "docs/_build/html/console.html"],
diff --git a/src/py/pyodide/__init__.py b/src/py/pyodide/__init__.py
--- a/src/py/pyodide/__init__.py
+++ b/src/py/pyodide/__init__.py
@@ -40,7 +40,7 @@
     asyncio.set_event_loop_policy(WebLoopPolicy())
 
 
-__version__ = "0.19.0dev0"
+__version__ = "0.19.0"
 
 __all__ = [
     "open_url",


Test patch:
Repository: pyodide/pyodide

Issue:
Update yt



Discussion:
We hit this issue: https://github.com/yt-project/yt/issues/3477
So I guess this has to wait on yt version 4.0.2.

Resolving patch:
diff --git a/packages/matplotlib/src/html5_canvas_backend.py b/packages/matplotlib/src/html5_canvas_backend.py
--- a/packages/matplotlib/src/html5_canvas_backend.py
+++ b/packages/matplotlib/src/html5_canvas_backend.py
@@ -355,7 +355,8 @@ def _get_font(self, prop):
     def get_text_width_height_descent(self, s, prop, ismath):
         if ismath:
             image, d = self.mathtext_parser.parse(s, self.dpi, prop)
-            w, h = image.get_width(), image.get_height()
+            image_arr = np.asarray(image)
+            h, w = image_arr.shape
         else:
             font, _ = self._get_font(prop)
             font.set_text(s, 0.0, flags=LOAD_NO_HINTING)


Test patch:
diff --git a/packages/matplotlib/test_matplotlib.py b/packages/matplotlib/test_matplotlib.py
--- a/packages/matplotlib/test_matplotlib.py
+++ b/packages/matplotlib/test_matplotlib.py
@@ -83,7 +83,7 @@ def test_svg(selenium):
     selenium.run("fd = io.BytesIO()")
     selenium.run("plt.savefig(fd, format='svg')")
     content = selenium.run("fd.getvalue().decode('utf8')")
-    assert len(content) == 16283
+    assert len(content) == 14998
     assert content.startswith("<?xml")
 
 
diff --git a/packages/test_packages_common.py b/packages/test_packages_common.py
--- a/packages/test_packages_common.py
+++ b/packages/test_packages_common.py
@@ -33,7 +33,7 @@ def registered_packages_meta():
 UNSUPPORTED_PACKAGES: dict[str, list[str]] = {
     "chrome": [],
     "firefox": [],
-    "node": [],
+    "node": ["cmyt"],
 }
 if "CI" in os.environ:
     UNSUPPORTED_PACKAGES["chrome"].extend(["statsmodels"])
Repository: pyodide/pyodide

Issue:
https://pyodide.org/en/latest/console.html doesn't show latest `main` version
## 🐛 Bug

The https://pyodide.org/en/latest/console.html console is stuck at `v0.21.0a3`. I believe this is because the version wasn't bumped to the next 'dev' version after the `v0.21.0a3` release, so somebody probably needs to run
```
./tools/bump_version.py --new-version 0.21.0.dev0

```
Without `dev` in the version, the documentation's console uses the release:
https://github.com/pyodide/pyodide/blob/7231cab3ffc83f6221fafb7458f9b223d2a7c759/docs/conf.py#L20-L23

### To Reproduce

Go to https://pyodide.org/en/latest/console.html and load a package added since v0.21.0a3, e.g., `import rebound`




Discussion:
Thanks for the report @jobovy. Probably we should use `READTHEDOCS_VERSION` to check if we are building for a `latest` version.

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -17,7 +17,7 @@
 copyright = "2019-2022, Pyodide contributors and Mozilla"
 pyodide_version = "0.21.0a3"
 
-if ".dev" in pyodide_version:
+if ".dev" in pyodide_version or os.environ.get("READTHEDOCS_VERSION") == "latest":
     CDN_URL = "https://cdn.jsdelivr.net/pyodide/dev/full/"
 else:
     CDN_URL = f"https://cdn.jsdelivr.net/pyodide/v{pyodide_version}/full/"


Test patch:
Repository: pyodide/pyodide

Issue:
Scipy build failing with Cython error
## 🐛 Bug

Scipy build is failing in [main](https://app.circleci.com/pipelines/github/ryanking13/pyodide/705/workflows/f75c57c6-95c0-4437-85ca-47889051954d/jobs/11444?invite=true#step-105-1365). I guess it is related to the new Cython release.

```
_stats.pyx:173:0: Referring to a memoryview typed argument directly in a nested closure function is not supported in Cython 0.x. Either upgrade to Cython 3, or assign the argument to a local variable and use that in the nested function.
Traceback (most recent call last):
  File "/root/repo/packages/scipy/build/scipy-1.8.1/tools/cythonize.py", line 317, in <module>
    main()
  File "/root/repo/packages/scipy/build/scipy-1.8.1/tools/cythonize.py", line 313, in main
    find_process_files(root_dir)
  File "/root/repo/packages/scipy/build/scipy-1.8.1/tools/cythonize.py", line 302, in find_process_files
    for result in pool.imap_unordered(lambda args: process(*args), jobs):
  File "/usr/local/lib/python3.10/multiprocessing/pool.py", line 870, in next
    raise value
  File "/usr/local/lib/python3.10/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
  File "/root/repo/packages/scipy/build/scipy-1.8.1/tools/cythonize.py", line 302, in <lambda>
    for result in pool.imap_unordered(lambda args: process(*args), jobs):
  File "/root/repo/packages/scipy/build/scipy-1.8.1/tools/cythonize.py", line 236, in process
    processor_function(fromfile, tofile, cwd=path)
  File "/root/repo/packages/scipy/build/scipy-1.8.1/tools/cythonize.py", line 102, in process_pyx
    raise Exception('Cython failed')
Exception: Cython failed
```



Discussion:
Upstream (scipy) issue tracking this bug: https://github.com/scipy/scipy/issues/16718
We should pin the cython version used, as it's likely to be forward incompatible, particularly once the long-awaited 3.0 version is released.
Or we can think of adding a new patch for scipy until they release 1.9.0.
As you prefer. But it's really upstream job to fix these things, we are just building the latest version that is known to work. So there is no real advantage for us to run the latest Cython in this particular case.

Resolving patch:
diff --git a/pyodide-build/pyodide_build/pypabuild.py b/pyodide-build/pyodide_build/pypabuild.py
--- a/pyodide-build/pyodide_build/pypabuild.py
+++ b/pyodide-build/pyodide_build/pypabuild.py
@@ -58,7 +58,12 @@ def install_reqs(env: IsolatedEnv, reqs: set[str]) -> None:
     # only recythonize if it is present. We need them to always recythonize so
     # we always install cython. If the reqs included some cython version already
     # then this won't do anything.
-    env.install(["cython", "pythran"])
+    env.install(
+        [
+            "cython<0.29.31",  # cython 0.29.31 is incompatible with scipy 1.8.1. TODO: remove this after the scipy update.
+            "pythran",
+        ]
+    )
 
 
 def _build_in_isolated_env(


Test patch:
Repository: pyodide/pyodide

Issue:
Release 0.21
I went through issues and tagged things for the 0.21 release https://github.com/pyodide/pyodide/milestone/11 

Mostly naming related subjects need to be discussed/addressed before the release IMO, since once we release they will be harder to change.


Discussion:
Are there any remaining blockers for this @rth @ryanking13 ?
I think #2893 was the last blocker for this.

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -15,7 +15,7 @@
 
 project = "Pyodide"
 copyright = "2019-2022, Pyodide contributors and Mozilla"
-pyodide_version = "0.21.0a3"
+pyodide_version = "0.21.0"
 
 if ".dev" in pyodide_version or os.environ.get("READTHEDOCS_VERSION") == "latest":
     CDN_URL = "https://cdn.jsdelivr.net/pyodide/dev/full/"
diff --git a/pyodide-build/pyodide_build/__init__.py b/pyodide-build/pyodide_build/__init__.py
--- a/pyodide-build/pyodide_build/__init__.py
+++ b/pyodide-build/pyodide_build/__init__.py
@@ -1 +1 @@
-__version__ = "0.21.0a3"
+__version__ = "0.21.0"
diff --git a/src/py/pyodide/__init__.py b/src/py/pyodide/__init__.py
--- a/src/py/pyodide/__init__.py
+++ b/src/py/pyodide/__init__.py
@@ -10,7 +10,7 @@
 # This package is imported by the test suite as well, and currently we don't use
 # pytest mocks for js or pyodide_js, so make sure to test "if IN_BROWSER" before
 # importing from these.
-__version__ = "0.21.0a3"
+__version__ = "0.21.0"
 
 __all__ = ["__version__"]
 


Test patch:
Repository: pyodide/pyodide

Issue:
Add lzma
As mentioned by @hoodmane in https://github.com/pyodide/pyodide/discussions/2930#discussioncomment-3316181

> Is there an issue open about lzma? What is our position on it again? That we want it but there is no emscripten port and we haven't gotten to it?

I think the main concern was the size increase for everyone vs few people actually needing it. Depending on the size maybe we could make it an unvendored stdlib package (or include by default if the size is negligible).


Discussion:

Resolving patch:
diff --git a/pyodide-build/pyodide_build/common.py b/pyodide-build/pyodide_build/common.py
--- a/pyodide-build/pyodide_build/common.py
+++ b/pyodide-build/pyodide_build/common.py
@@ -80,6 +80,7 @@ def find_matching_wheels(wheel_paths: Iterable[Path]) -> Iterator[Path]:
     "sharedlib-test-py",
     "cpp-exceptions-test",
     "ssl",
+    "lzma",
     "pytest",
     "tblib",
 }


Test patch:
diff --git a/packages/lzma/test_lzma.py b/packages/lzma/test_lzma.py
new file mode 100644
--- /dev/null
+++ b/packages/lzma/test_lzma.py
@@ -0,0 +1,17 @@
+from pytest_pyodide import run_in_pyodide
+
+
+@run_in_pyodide(packages=["test", "lzma"], pytest_assert_rewrites=False)
+def test_lzma(selenium):
+    # TODO: libregrtest.main(["test_lzma"]) doesn't collect any tests for some unknown reason.
+
+    import test.test_lzma
+    import unittest
+
+    suite = unittest.TestSuite(
+        [unittest.TestLoader().loadTestsFromModule(test.test_lzma)]
+    )
+
+    runner = unittest.TextTestRunner(verbosity=2)
+    result = runner.run(suite)
+    assert result.wasSuccessful()
Repository: pyodide/pyodide

Issue:
Export version from pyodide JS module
## 🚀 Feature

In `pyodide.d.ts` I see `declare let version: string;` but it's not exported. It'd be great if it was exported so that I can do this:

```ts
import {version} from "pyodide";
```

### Motivation

I have [some code](https://github.com/alexmojaki/pyodide-worker-runner/blob/e7dd3d0ee1dff457bf9d6104944477840a83e5a7/lib/index.ts#L16) that roughly looks like this:

```ts
import {loadPyodide, PyodideInterface} from "pyodide";

const version = "0.21.1";
const indexURL = `https://cdn.jsdelivr.net/pyodide/v${version}/full/`;
const pyodide: PyodideInterface = await loadPyodide({indexURL});
if (pyodide.version !== version) {
  throw new Error(
    `loadPyodide loaded version ${pyodide.version} instead of ${version}`,
  );
}
```

I'd like to import `version` instead of setting it manually, so that it always automatically matches whatever version of pyodide is installed.


Discussion:
Thanks for the suggestion @alexmojaki!

https://github.com/pyodide/pyodide/blob/3e2d7f29a5b242882f2b74bf89319220815136b1/src/js/pyodide.ts#L147

It seems like we are setting the version string of the JS package dynamically during the `loadPyodide` using the version of the Pyodide Python package.

Probably we should have a static version string for JS as you suggested, and maybe we should reject if the requested Pyodide Python package version mismatches JS package version.

Resolving patch:
diff --git a/tools/bump_version.py b/tools/bump_version.py
--- a/tools/bump_version.py
+++ b/tools/bump_version.py
@@ -64,6 +64,16 @@ def build_version_pattern(pattern):
         build_version_pattern(r"version\s*=\s*{{{python_version}}}"),
         prerelease=False,
     ),
+    Target(
+        ROOT / "src/js/version.ts",
+        build_version_pattern('version: string = "{python_version}"'),
+        prerelease=True,
+    ),
+    Target(
+        ROOT / "src/core/pre.js",
+        build_version_pattern('API.version = "{python_version}"'),
+        prerelease=True,
+    ),
 ]
 
 JS_TARGETS = [


Test patch:
diff --git a/src/tests/test_pyodide.py b/src/tests/test_pyodide.py
--- a/src/tests/test_pyodide.py
+++ b/src/tests/test_pyodide.py
@@ -1126,6 +1126,24 @@ def test_home_directory(selenium_standalone_noload):
     )
 
 
+def test_version_variable(selenium):
+    js_version = selenium.run_js(
+        """
+        return pyodide.version
+        """
+    )
+
+    core_version = selenium.run_js(
+        """
+        return pyodide._api.version
+        """
+    )
+
+    from pyodide import __version__ as py_version
+
+    assert js_version == py_version == core_version
+
+
 def test_sys_path0(selenium):
     selenium.run_js(
         """
Repository: pyodide/pyodide

Issue:
The content area in the docs is too narrow
## 📚 Documentation

In the documentation strings, rendered code examples only fit 63 characters of width. It would be nice if we could make the content area a bit larger so that code examples fit at least 80 characters. On my screen, the content area is exactly the middle third of the screen, with the left and right thirds devoted to menus.



Discussion:
Sounds good to me. It looks like we had wider area in [0.18.0](https://pyodide.org/en/0.18.0/usage/quickstart.html).
Yes that's way better. I wonder what changed.
You might need to add
```
panels_add_bootstrap_css= False
```
to the ``conf.py`` file? See

https://sphinx-panels.readthedocs.io/en/latest/#sphinx-configuration

and 

https://github.com/executablebooks/sphinx-book-theme/issues/194#issuecomment-697028667

``sphinx_panels`` was added in #1960, so that seems consistent with when the width changed... I can't test this locally though, because I can't easily get the docs to build.
> because I can't easily get the docs to build.

Maybe that should be a separate issue. I agree that building the docs locally is a bit of a pain.
> add `panels_add_bootstrap_css= False`

This fixes the problem, thanks @jobovy!

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -11,6 +11,8 @@
 from typing import Any
 from unittest import mock
 
+panels_add_bootstrap_css = False
+
 # -- Project information -----------------------------------------------------
 
 project = "Pyodide"


Test patch:
Repository: pyodide/pyodide

Issue:
`pyodide skeleton pypi --update` doesn't compare versions correctly
## 🐛 Bug

`pyodide skeleton pypi --update` doesn't compare versions correctly (this also was a problem with the old `make -C packages update-all`).

### To Reproduce

```
/src$ pyodide skeleton pypi --update numcodecs
numcodecs already up to date. Local: 0.9.1 PyPI: 0.11.0
```

### Expected behavior

Update numcodecs to the never v0.11.0



Discussion:

Resolving patch:
diff --git a/pyodide-build/pyodide_build/mkpkg.py b/pyodide-build/pyodide_build/mkpkg.py
--- a/pyodide-build/pyodide_build/mkpkg.py
+++ b/pyodide-build/pyodide_build/mkpkg.py
@@ -16,6 +16,7 @@
 from typing import Any, Literal, NoReturn, TypedDict
 from urllib import request
 
+from packaging.version import Version
 from ruamel.yaml import YAML
 
 from .common import parse_top_level_import_name
@@ -264,8 +265,8 @@ def update_package(
         old_fmt = "sdist"
 
     pypi_metadata = _get_metadata(package, version)
-    pypi_ver = pypi_metadata["info"]["version"]
-    local_ver = yaml_content["package"]["version"]
+    pypi_ver = Version(pypi_metadata["info"]["version"])
+    local_ver = Version(yaml_content["package"]["version"])
     already_up_to_date = pypi_ver <= local_ver and (
         source_fmt is None or source_fmt == old_fmt
     )


Test patch: