Repository: pyodide/pyodide

Issue:
Import hook for repodata packages shows incorrect results.
## 🐛 Bug

The import hook for repodata packages is showing incorrect results for some packages.

For example, shown in https://github.com/pyodide/pyodide/discussions/3419:

```py
import gdal # from osgeo import gdal

PythonError: Traceback (most recent call last):
  File "/lib/python3.10/_pyodide/_base.py", line 460, in eval_code
    .run(globals, locals)
  File "/lib/python3.10/_pyodide/_base.py", line 304, in run
    coroutine = eval(self.code, globals, locals)
  File "<exec>", line 8, in <module>
ModuleNotFoundError: The module 'gdal' is included in the Pyodide distribution, but it is not installed.
```

This is not intended, as `gdal` is a shared library not a Python package (at least until we have Python bindings for gdal).

I'll make a fix, this is mostly a FYI.




Discussion:

Resolving patch:
diff --git a/pyodide-build/pyodide_build/buildall.py b/pyodide-build/pyodide_build/buildall.py
--- a/pyodide-build/pyodide_build/buildall.py
+++ b/pyodide-build/pyodide_build/buildall.py
@@ -571,6 +571,7 @@ def generate_packagedata(
             "file_name": pkg.file_name,
             "install_dir": pkg.install_dir,
             "sha256": _generate_package_hash(Path(output_dir, pkg.file_name)),
+            "imports": [],
         }
 
         pkg_type = pkg.package_type
@@ -582,9 +583,11 @@ def generate_packagedata(
             )
 
         pkg_entry["depends"] = [x.lower() for x in pkg.run_dependencies]
-        pkg_entry["imports"] = (
-            pkg.meta.package.top_level if pkg.meta.package.top_level else [name]
-        )
+
+        if pkg.package_type not in ("static_library", "shared_library"):
+            pkg_entry["imports"] = (
+                pkg.meta.package.top_level if pkg.meta.package.top_level else [name]
+            )
 
         packages[name.lower()] = pkg_entry
 
diff --git a/src/py/_pyodide/_importhook.py b/src/py/_pyodide/_importhook.py
--- a/src/py/_pyodide/_importhook.py
+++ b/src/py/_pyodide/_importhook.py
@@ -134,26 +134,15 @@ def register_js_finder() -> None:
 
 
 STDLIBS = sys.stdlib_module_names | {"test"}
-UNVENDORED_STDLIBS_MAP = {
-    "distutils": ["distutils"],
-    "ssl": ["ssl", "_ssl"],
-    "lzma": ["lzma", "_lzma"],
-    "sqlite3": ["sqlite3", "_sqlite3"],
-    "hashlib": ["_hashlib"],
-}
-UNVENDORED_STDLIBS = list(UNVENDORED_STDLIBS_MAP.keys())
+# TODO: Move this list to js side
+UNVENDORED_STDLIBS = ["distutils", "ssl", "lzma", "sqlite3", "hashlib"]
 UNVENDORED_STDLIBS_AND_TEST = UNVENDORED_STDLIBS + ["test"]
-UNVENDORED_STDLIBS_MODULES = {
-    importname: pkgname
-    for pkgname, importnames in UNVENDORED_STDLIBS_MAP.items()
-    for importname in importnames
-} | {"test": "test"}
 
 
 from importlib import _bootstrap  # type: ignore[attr-defined]
 
 orig_get_module_not_found_error: Any = None
-REPODATA_PACKAGES: list[str] = []
+REPODATA_PACKAGES_IMPORT_TO_PACKAGE_NAME: dict[str, str] = {}
 
 SEE_PACKAGE_LOADING = (
     "\nSee https://pyodide.org/en/stable/usage/loading-packages.html for more details."
@@ -161,29 +150,34 @@ def register_js_finder() -> None:
 
 YOU_CAN_INSTALL_IT_BY = """
 You can install it by calling:
-  await micropip.install("{name}") in Python, or
-  await pyodide.loadPackage("{name}") in JavaScript\
+  await micropip.install("{package_name}") in Python, or
+  await pyodide.loadPackage("{package_name}") in JavaScript\
 """
 
 
-def get_module_not_found_error(name):
-    if name not in REPODATA_PACKAGES and name not in STDLIBS:
-        return orig_get_module_not_found_error(name)
+def get_module_not_found_error(import_name):
 
-    if name in UNVENDORED_STDLIBS_MODULES:
-        msg = "The module '{name}' is unvendored from the Python standard library in the Pyodide distribution."
-        msg += YOU_CAN_INSTALL_IT_BY
-        name = UNVENDORED_STDLIBS_MODULES[name]
-    elif name in REPODATA_PACKAGES:
-        msg = "The module '{name}' is included in the Pyodide distribution, but it is not installed."
+    package_name = REPODATA_PACKAGES_IMPORT_TO_PACKAGE_NAME.get(import_name, "")
+
+    if not package_name and import_name not in STDLIBS:
+        return orig_get_module_not_found_error(import_name)
+
+    if package_name in UNVENDORED_STDLIBS_AND_TEST:
+        msg = "The module '{package_name}' is unvendored from the Python standard library in the Pyodide distribution."
         msg += YOU_CAN_INSTALL_IT_BY
-    else:
+    elif import_name in STDLIBS:
         msg = (
-            "The module '{name}' is removed from the Python standard library in the"
+            "The module '{import_name}' is removed from the Python standard library in the"
             " Pyodide distribution due to browser limitations."
         )
+    else:
+        msg = "The module '{package_name}' is included in the Pyodide distribution, but it is not installed."
+        msg += YOU_CAN_INSTALL_IT_BY
+
     msg += SEE_PACKAGE_LOADING
-    return ModuleNotFoundError(msg.format(name=name))
+    return ModuleNotFoundError(
+        msg.format(import_name=import_name, package_name=package_name)
+    )
 
 
 def register_module_not_found_hook(packages: Any) -> None:
@@ -194,48 +188,7 @@ def register_module_not_found_hook(packages: Any) -> None:
     in order to prevent any unexpected side effects.
     """
     global orig_get_module_not_found_error
-    global REPODATA_PACKAGES
-    REPODATA_PACKAGES = packages.to_py()
+    global REPODATA_PACKAGES_IMPORT_TO_PACKAGE_NAME
+    REPODATA_PACKAGES_IMPORT_TO_PACKAGE_NAME = packages.to_py()
     orig_get_module_not_found_error = _bootstrap._get_module_not_found_error
     _bootstrap._get_module_not_found_error = get_module_not_found_error
-
-
-class RepodataPackagesFinder(MetaPathFinder):
-    """
-    A MetaPathFinder that handles packages in repodata.json.
-
-    This class simply raises an error if a package is in repodata.json but not loaded yet.
-    This needs to be added to the end of sys.meta_path, so if a package
-    is already loaded via pyodide.loadPackage, it can be handled by the existing finder.
-    """
-
-    def __init__(self, packages: dict[str, Any]) -> None:
-        self.repodata_packages = packages
-
-    def find_spec(
-        self,
-        fullname: str,
-        path: Sequence[bytes | str] | None,
-        target: ModuleType | None = None,
-    ) -> ModuleSpec | None:
-        [parent, _, _] = fullname.partition(".")
-
-        if not parent or parent in sys.modules or parent not in self.repodata_packages:
-            return None
-
-        return None
-
-
-def register_repodata_packages_finder(packages: Any) -> None:
-    """
-    A function that adds RepodataPackagesFinder to the end of sys.meta_path.
-
-    Note that this finder must be placed in the end of meta_paths
-    in order to prevent any unexpected side effects.
-    """
-
-    for importer in sys.meta_path:
-        if isinstance(importer, RepodataPackagesFinder):
-            raise RuntimeError("RepodataPackagesFinder already registered")
-
-    sys.meta_path.append(RepodataPackagesFinder(packages.to_py()))


Test patch:
diff --git a/pyodide-build/pyodide_build/tests/_test_recipes/libtest_shared/meta.yaml b/pyodide-build/pyodide_build/tests/_test_recipes/libtest_shared/meta.yaml
new file mode 100644
--- /dev/null
+++ b/pyodide-build/pyodide_build/tests/_test_recipes/libtest_shared/meta.yaml
@@ -0,0 +1,9 @@
+package:
+  name: libtest_shared
+  version: 1.0.0
+source:
+  sha256: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef
+  url: https://dummy-url-that-not-exists.com/libtest.tar.gz
+build:
+  type: shared_library
+  script: "pass"
diff --git a/pyodide-build/pyodide_build/tests/_test_recipes/pkg_1/meta.yaml b/pyodide-build/pyodide_build/tests/_test_recipes/pkg_1/meta.yaml
--- a/pyodide-build/pyodide_build/tests/_test_recipes/pkg_1/meta.yaml
+++ b/pyodide-build/pyodide_build/tests/_test_recipes/pkg_1/meta.yaml
@@ -8,9 +8,11 @@ requirements:
   host:
     - pkg_1_1
     - pkg_3
+    - libtest_shared
   run:
     - pkg_1_1
     - pkg_3
+    - libtest_shared
 test:
   imports:
     - pkg_1
diff --git a/pyodide-build/pyodide_build/tests/test_buildall.py b/pyodide-build/pyodide_build/tests/test_buildall.py
--- a/pyodide-build/pyodide_build/tests/test_buildall.py
+++ b/pyodide-build/pyodide_build/tests/test_buildall.py
@@ -53,7 +53,7 @@ def from_yaml(cls, path):
 
 def test_generate_repodata(tmp_path):
     pkg_map = buildall.generate_dependency_graph(
-        RECIPE_DIR, {"pkg_1", "pkg_2", "libtest"}
+        RECIPE_DIR, {"pkg_1", "pkg_2", "libtest", "libtest_shared"}
     )
     hashes = {}
     for pkg in pkg_map.values():
@@ -77,17 +77,23 @@ def test_generate_repodata(tmp_path):
         "pkg_2",
         "pkg_3",
         "pkg_3_1",
+        "libtest_shared",
     }
     assert package_data["packages"]["pkg_1"] == {
         "name": "pkg_1",
         "version": "1.0.0",
         "file_name": "pkg_1.whl",
-        "depends": ["pkg_1_1", "pkg_3"],
+        "depends": ["pkg_1_1", "pkg_3", "libtest_shared"],
         "imports": ["pkg_1"],
         "install_dir": "site",
         "sha256": hashes["pkg_1"],
     }
 
+    sharedlib_imports = package_data["packages"]["libtest_shared"]["imports"]
+    assert not sharedlib_imports, (
+        "shared libraries should not have any imports, but got " f"{sharedlib_imports}"
+    )
+
 
 @pytest.mark.parametrize("n_jobs", [1, 4])
 def test_build_dependencies(n_jobs, monkeypatch):
@@ -111,6 +117,7 @@ def build(self, args: Any) -> None:
         "pkg_2",
         "pkg_3",
         "pkg_3_1",
+        "libtest_shared",
     }
     assert build_list.index("pkg_1_1") < build_list.index("pkg_1")
     assert build_list.index("pkg_3") < build_list.index("pkg_1")
diff --git a/src/tests/test_pyodide.py b/src/tests/test_pyodide.py
--- a/src/tests/test_pyodide.py
+++ b/src/tests/test_pyodide.py
@@ -1424,6 +1424,14 @@ def test_module_not_found_hook(selenium_standalone):
     with pytest.raises(ModuleNotFoundError, match="No module named"):
         importlib.import_module("pytest.there_is_no_such_module")
 
+    # liblzma and openssl are libraries not python packages, so it should just fail.
+    for pkg in ["liblzma", "openssl"]:
+        with pytest.raises(ModuleNotFoundError, match="No module named"):
+            importlib.import_module(pkg)
+
+    with pytest.raises(ModuleNotFoundError, match=r'loadPackage\("hashlib"\)'):
+        importlib.import_module("_hashlib")
+
 
 def test_args(selenium_standalone_noload):
     selenium = selenium_standalone_noload