Repository: pyodide/pyodide

Issue:
Build C libraries with the Python build system
Currently numerous C libraries (e.g. lz4, libxml) and also some packages (e.g. six, parso)  are packaged with Makefiles in the root folder of pyodide. I this can be confusing for new contributors and is not sustainable long term, as it makes it hard to distinguish what is pyodide code and what are packages.

Related to https://github.com/iodide-project/pyodide/issues/38

I think we should,
 - [x] move all of those libraires under `packages/` folder (while keeping the use of Makefiles). This would be just a path change. Done in https://github.com/iodide-project/pyodide/pull/714
 - [x] migrate these libraries to be defined with a `meta.yaml` and build them with the Python build system. This would allow managing dependencies better (e.g. build LAPACK only when scipy is built etc).


Discussion:
To give a more concrete example for step 2, consider [`packages/zlib/Makefile`](https://github.com/iodide-project/pyodide/blob/master/packages/zlib/Makefile), we would want to replace it with something along the lines of,

**packages/zlib/meta.yaml**
```yaml
package:
  name: zlib
  version: 1.2.11
source:
  sha256: c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
  url: https://zlib.net/zlib-1.2.11.tar.gz
build:
  script:
    - emconfigure ./configure  --prefix=$(SRC)
    - emmake make install -j ${PYODIDE_JOBS:-3} 
```

and then,
 1. adjust [pyodide_build/buildpkg.py](https://github.com/iodide-project/pyodide/blob/master/pyodide_build/buildpkg.py#L230) to these build script commands if they are available and fall back to the `python setup.py` command otherwise as it's done now.
 2. figure out how to pass environment variables such as `PYODIDE_JOBS` and also some paths variables (here `$SRC` which is the current dir). There is probably some standard approaches of doing this with conda `meta.yaml`. It would require reading the `meta.yaml` spec and maybe also browsing a few examples on conda-forge.
 3. Add `zlib` as a requirements to any package that needs it, here lxml.
 4. Remove the corresponding command form the [main Makefile](https://github.com/iodide-project/pyodide/blob/bc3ffaae427e35b9ecd9a6216fadad031a363a47/Makefile#L268).

**Note:** zlib is required by libxml, so it would likely need to be ported at the same time.  Or one could start with a different package, for instance lz4.

Resolving patch:
diff --git a/pyodide_build/buildall.py b/pyodide_build/buildall.py
--- a/pyodide_build/buildall.py
+++ b/pyodide_build/buildall.py
@@ -40,37 +40,28 @@ def __init__(self, pkgdir: Path):
 
     def build(self, outputdir: Path, args) -> None:
         with open(self.pkgdir / "build.log", "w") as f:
-            if self.library:
-                p = subprocess.run(
-                    ["make"],
-                    cwd=self.pkgdir,
-                    check=False,
-                    stdout=f,
-                    stderr=subprocess.STDOUT,
-                )
-            else:
-                p = subprocess.run(
-                    [
-                        sys.executable,
-                        "-m",
-                        "pyodide_build",
-                        "buildpkg",
-                        str(self.pkgdir / "meta.yaml"),
-                        "--cflags",
-                        args.cflags,
-                        "--cxxflags",
-                        args.cxxflags,
-                        "--ldflags",
-                        args.ldflags,
-                        "--target",
-                        args.target,
-                        "--install-dir",
-                        args.install_dir,
-                    ],
-                    check=False,
-                    stdout=f,
-                    stderr=subprocess.STDOUT,
-                )
+            p = subprocess.run(
+                [
+                    sys.executable,
+                    "-m",
+                    "pyodide_build",
+                    "buildpkg",
+                    str(self.pkgdir / "meta.yaml"),
+                    "--cflags",
+                    args.cflags,
+                    "--cxxflags",
+                    args.cxxflags,
+                    "--ldflags",
+                    args.ldflags,
+                    "--target",
+                    args.target,
+                    "--install-dir",
+                    args.install_dir,
+                ],
+                check=False,
+                stdout=f,
+                stderr=subprocess.STDOUT,
+            )
 
         try:
             p.check_returncode()
diff --git a/pyodide_build/buildpkg.py b/pyodide_build/buildpkg.py
--- a/pyodide_build/buildpkg.py
+++ b/pyodide_build/buildpkg.py
@@ -92,7 +92,8 @@ def download_and_extract(
                 tarballname = tarballname[: -len(extension)]
                 break
 
-        return buildpath / tarballname
+        return buildpath / pkg["source"].get("extract_dir", tarballname)
+
     elif "path" in pkg["source"]:
         srcdir = Path(pkg["source"]["path"])
 
@@ -212,6 +213,23 @@ def package_files(buildpath: Path, srcpath: Path, pkg: Dict[str, Any], args):
         fd.write(b"\n")
 
 
+def run_script(buildpath: Path, srcpath: Path, pkg: Dict[str, Any]):
+    # We don't really do packaging, but needs_rebuild checks .packaged to
+    # determine if it needs to rebuild
+    if (buildpath / ".packaged").is_file():
+        return
+
+    orig_path = Path.cwd()
+    os.chdir(srcpath)
+    try:
+        subprocess.run(["bash", "-c", pkg["build"]["script"]], check=True)
+    finally:
+        os.chdir(orig_path)
+
+    with open(buildpath / ".packaged", "wb") as fd:
+        fd.write(b"\n")
+
+
 def needs_rebuild(pkg: Dict[str, Any], path: Path, buildpath: Path) -> bool:
     """
     Determines if a package needs a rebuild because its meta.yaml, patches, or
@@ -240,7 +258,7 @@ def build_package(path: Path, args):
     name = pkg["package"]["name"]
     t0 = datetime.now()
     print("[{}] Building package {}...".format(t0.strftime("%Y-%m-%d %H:%M:%S"), name))
-    packagedir = name + "-" + pkg["package"]["version"]
+    packagedir = name + "-" + str(pkg["package"]["version"])
     dirpath = path.parent
     orig_path = Path.cwd()
     os.chdir(dirpath)
@@ -254,8 +272,11 @@ def build_package(path: Path, args):
             os.makedirs(buildpath)
         srcpath = download_and_extract(buildpath, packagedir, pkg, args)
         patch(path, srcpath, pkg, args)
-        compile(path, srcpath, pkg, args)
-        package_files(buildpath, srcpath, pkg, args)
+        if pkg.get("build", {}).get("library"):
+            run_script(buildpath, srcpath, pkg)
+        else:
+            compile(path, srcpath, pkg, args)
+            package_files(buildpath, srcpath, pkg, args)
     finally:
         os.chdir(orig_path)
         t1 = datetime.now()
diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py
--- a/pyodide_build/pywasmcross.py
+++ b/pyodide_build/pywasmcross.py
@@ -287,7 +287,7 @@ def handle_command(line, args, dryrun=False):
             output = arg
 
         # Fix for scipy to link to the correct BLAS/LAPACK files
-        if arg.startswith("-L") and "CLAPACK-WA" in arg:
+        if arg.startswith("-L") and "CLAPACK" in arg:
             out_idx = line.index("-o")
             out_idx += 1
             module_name = line[out_idx]


Test patch: