Repository: pyodide/pyodide

Issue:
API Normalize package names in pyodide-lock.json
In https://github.com/pyodide/pyodide-lock/pull/20#issuecomment-1735331697 we discussed that it would be good to normalize the package names. This was also supported by @bollwyvl's comment in https://github.com/pyodide/pyodide-lock/pull/18#discussion_r1336556106 and also what the [PEP 691](https://peps.python.org/pep-0691/) for Simple JSON API recommends. 

In the case of pyodide, I imagine that would mean doing,

```py
packages: {
    <normalized_name>: {
              name : <normalized_name>
              depends: [<normalized_names>]
     }
}
```
so we would be repeating the normalized name twice once as the dict key, and the second time as the inner name field. 


This however means that if say we do `micropip.install('scikit-learn')` (or if it's pulled as a dependency) in the logs we would see `installing scikit_learn` which I personally find strange. Unless we de-normalize the name somehow when displaying (at least replacing `_` with `-`).

For the input `meta.yaml` this souldn't change anything, and we would only normalize the output name.

cc @ryanking13 @joemarshall


Discussion:
```
packages: {
    <normalized_name>: {
              name : <original_name>
              depends: [<normalized_names>]
     }
}
```

Actually, I was thinking like this, so that a package can be queried using normalized name, but we can still get the original name of a package from the value.
I would be happy with this, and I was thinking about this as well.  Would the above work for you @bollwyvl, which means we can still remove normalization in https://github.com/pyodide/pyodide-lock/pull/18#discussion_r1332985361?
Sure, keeping the verbatim `name` sounds good, along with some human-readable text to this effect that makes it all the way into the schema.

It turns out _dict keyed by strings matching {pattern}_, and even _list of strings matching {pattern}_  is... non-trivial to support for _both_ major versions of `pydantic`. As `pyodide` still ships `pydantic 1.10.7`, and going to `2.x` would take a bunch more rust work, it likely makes sense to constrain `pydantic <2` for now.

Resolving patch:
diff --git a/pyodide-build/pyodide_build/buildall.py b/pyodide-build/pyodide_build/buildall.py
--- a/pyodide-build/pyodide_build/buildall.py
+++ b/pyodide-build/pyodide_build/buildall.py
@@ -19,6 +19,7 @@
 from time import perf_counter, sleep
 from typing import Any
 
+from packaging.utils import canonicalize_name
 from pyodide_lock import PyodideLockSpec
 from pyodide_lock.spec import PackageSpec as PackageLockSpec
 from pyodide_lock.utils import update_package_sha256
@@ -637,6 +638,8 @@ def generate_packagedata(
 ) -> dict[str, PackageLockSpec]:
     packages: dict[str, PackageLockSpec] = {}
     for name, pkg in pkg_map.items():
+        normalized_name = canonicalize_name(name)
+
         if not pkg.file_name or pkg.package_type == "static_library":
             continue
         if not Path(output_dir, pkg.file_name).exists():
@@ -666,10 +669,10 @@ def generate_packagedata(
                 pkg.meta.package.top_level if pkg.meta.package.top_level else [name]
             )
 
-        packages[name.lower()] = pkg_entry
+        packages[normalized_name.lower()] = pkg_entry
 
         if pkg.unvendored_tests:
-            packages[name.lower()].unvendored_tests = True
+            packages[normalized_name.lower()].unvendored_tests = True
 
             # Create the test package if necessary
             pkg_entry = PackageLockSpec(
@@ -682,7 +685,7 @@ def generate_packagedata(
 
             update_package_sha256(pkg_entry, output_dir / pkg.unvendored_tests.name)
 
-            packages[name.lower() + "-tests"] = pkg_entry
+            packages[normalized_name.lower() + "-tests"] = pkg_entry
 
     # sort packages by name
     packages = dict(sorted(packages.items()))


Test patch:
diff --git a/packages/micropip/test_micropip.py b/packages/micropip/test_micropip.py
--- a/packages/micropip/test_micropip.py
+++ b/packages/micropip/test_micropip.py
@@ -5,6 +5,8 @@
 import pytest
 from pytest_pyodide import run_in_pyodide, spawn_web_server
 
+from conftest import package_is_built
+
 cpver = f"cp{sys.version_info.major}{sys.version_info.minor}"
 
 
@@ -243,3 +245,18 @@ async def run_test(selenium, url, wheel_name):
             ]
 
         run_test(selenium_standalone_micropip, url, SNOWBALL_WHEEL)
+
+
+def test_install_non_normalized_package(selenium_standalone_micropip):
+    if not package_is_built("ruamel-yaml"):
+        pytest.skip("ruamel.yaml not built")
+
+    selenium = selenium_standalone_micropip
+
+    selenium.run_async(
+        """
+        import micropip
+        await micropip.install("ruamel.yaml")
+        import ruamel.yaml
+        """
+    )
diff --git a/pyodide-build/pyodide_build/tests/test_buildall.py b/pyodide-build/pyodide_build/tests/test_buildall.py
--- a/pyodide-build/pyodide_build/tests/test_buildall.py
+++ b/pyodide-build/pyodide_build/tests/test_buildall.py
@@ -71,14 +71,14 @@ def test_generate_lockfile(tmp_path):
     assert package_data.info.platform.startswith("emscripten")
 
     assert set(package_data.packages) == {
-        "pkg_1",
-        "pkg_1_1",
-        "pkg_2",
-        "pkg_3",
-        "pkg_3_1",
-        "libtest_shared",
+        "pkg-1",
+        "pkg-1-1",
+        "pkg-2",
+        "pkg-3",
+        "pkg-3-1",
+        "libtest-shared",
     }
-    assert package_data.packages["pkg_1"] == PackageSpec(
+    assert package_data.packages["pkg-1"] == PackageSpec(
         name="pkg_1",
         version="1.0.0",
         file_name="pkg_1-1.0.0-py3-none-any.whl",
@@ -89,9 +89,9 @@ def test_generate_lockfile(tmp_path):
         sha256=hashes["pkg_1"],
     )
 
-    assert package_data.packages["libtest_shared"].package_type == "shared_library"
+    assert package_data.packages["libtest-shared"].package_type == "shared_library"
 
-    sharedlib_imports = package_data.packages["libtest_shared"].imports
+    sharedlib_imports = package_data.packages["libtest-shared"].imports
     assert not sharedlib_imports, (
         "shared libraries should not have any imports, but got " f"{sharedlib_imports}"
     )
diff --git a/src/js/test/unit/common.test.ts b/src/js/test/unit/common.test.ts
new file mode 100644
--- /dev/null
+++ b/src/js/test/unit/common.test.ts
@@ -0,0 +1,71 @@
+import * as chai from "chai";
+import {
+  canonicalizePackageName,
+  uriToPackageData,
+} from "../../packaging-utils";
+
+describe("canonicalizePackageName", () => {
+  it("should return lower case", () => {
+    chai.assert.equal(canonicalizePackageName("ABC"), "abc");
+  });
+  it("should replace -, _, . with -", () => {
+    chai.assert.equal(
+      canonicalizePackageName("pytest-pyodide"),
+      "pytest-pyodide",
+    );
+    chai.assert.equal(canonicalizePackageName("ruamel.yaml"), "ruamel-yaml");
+    chai.assert.equal(
+      canonicalizePackageName("pytest_benchmark"),
+      "pytest-benchmark",
+    );
+    chai.assert.equal(canonicalizePackageName("a_b-c.d"), "a-b-c-d");
+  });
+});
+
+describe("uriToPackageData", () => {
+  it("should return the correct package data if a correct wheel URI is given", () => {
+    const testcases: [
+      string,
+      { name: string; version: string; fileName: string },
+    ][] = [
+      [
+        "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl",
+        {
+          name: "requests",
+          version: "2.31.0",
+          fileName: "requests-2.31.0-py3-none-any.whl",
+        },
+      ],
+      [
+        "https://example.com/srtrain-2.3.0-py3-none-any.whl",
+        {
+          name: "srtrain",
+          version: "2.3.0",
+          fileName: "srtrain-2.3.0-py3-none-any.whl",
+        },
+      ],
+      [
+        "https://test.net/numpy-1.25.2-cp311-cp311-emscripten_3_1_45_wasm32.whl",
+        {
+          name: "numpy",
+          version: "1.25.2",
+          fileName: "numpy-1.25.2-cp311-cp311-emscripten_3_1_45_wasm32.whl",
+        },
+      ],
+    ];
+
+    testcases.forEach((tc) => {
+      const [url, expected] = tc;
+      const pkgData = uriToPackageData(url);
+      chai.assert.equal(pkgData?.name, expected.name);
+      chai.assert.equal(pkgData?.version, expected.version);
+      chai.assert.equal(pkgData?.fileName, expected.fileName);
+    });
+  });
+
+  it("should return undefined if URI is not a valid wheel URI", () => {
+    chai.assert.equal(uriToPackageData("requests"), undefined);
+    chai.assert.equal(uriToPackageData("pyodide-lock"), undefined);
+    chai.assert.equal(uriToPackageData("pytest_benchmark"), undefined);
+  });
+});
diff --git a/src/tests/test_package_loading.py b/src/tests/test_package_loading.py
--- a/src/tests/test_package_loading.py
+++ b/src/tests/test_package_loading.py
@@ -588,3 +588,40 @@ def test_custom_lockfile(selenium_standalone_noload):
         )
     finally:
         custom_lockfile.unlink()
+
+
+@pytest.mark.parametrize(
+    "load_name, normalized_name, real_name",
+    [
+        # TODO: find a better way to test this without relying on the core packages set
+        ("fpcast-test", "fpcast-test", "fpcast-test"),
+        ("fpcast_test", "fpcast-test", "fpcast-test"),
+        ("Jinja2", "jinja2", "Jinja2"),
+        ("jinja2", "jinja2", "Jinja2"),
+        ("pydoc_data", "pydoc-data", "pydoc_data"),
+        ("pydoc-data", "pydoc-data", "pydoc_data"),
+    ],
+)
+def test_normalized_name(selenium_standalone, load_name, normalized_name, real_name):
+    selenium = selenium_standalone
+
+    selenium.run_js(
+        f"""
+        const msgs = [];
+        await pyodide.loadPackage(
+            "{load_name}",
+            {{
+                messageCallback: (msg) => msgs.push(msg),
+            }}
+        )
+
+        const loaded = Object.keys(pyodide.loadedPackages);
+        assert(() => loaded.includes("{real_name}"));
+
+        const loadStartMsgs = msgs.filter((msg) => msg.startsWith("Loading"));
+        const loadEndMsgs = msgs.filter((msg) => msg.startsWith("Loaded"));
+
+        assert(() => loadStartMsgs.some((msg) => msg.includes("{real_name}")));
+        assert(() => loadEndMsgs.some((msg) => msg.includes("{real_name}")));
+        """
+    )