Repository: pypa/cibuildwheel

Issue:
Use Nox?
In bin, we have some code that likely could be run by nox, and at least one piece of code that could be replaced by Nox. This could simplify some of the scripts - mostly looking at the first half of update_dependencies.py. It could be then run either from a Nox docker container, using the Nox action, or locally if someone has installed all the required versions of Python, without having the current `--no-docker` duplication.

This could possibly be augmented over time to take support running all our development actions, so a new developer could easily run these in a controlled environment without having to set up anything, and get a matching environment to the CI. I wasn't that fond of tox - too rigid with too many options and assumptions , but nox is not bad, much cleaner and more natural with nicer output. `pip` uses nox and tox (slowly transitioning to nox-only, I think). `packaging` and `packaging.python.org` use nox. Pretty much all the other `pypa` projects use tox.

Common dev actions would mostly be still available as a script (though the nox runner would install the requirements for you), though some actions (esp. ones like update_dependencies) could only run in Nox (mostly ones that require multiple Python versions).

Thoughts?

PS: personally, I run `pipx run nox` which means I don't have to install nox.


Discussion:
For an example of something very similar for manylinux, see https://github.com/pypa/manylinux/pull/1055 - noxfile https://github.com/pypa/manylinux/pull/1055/files#diff-f7a16a65f061822bcc73b8296f4dc837353d379d8d9cc5307982cb6941442835
Another nox example: https://github.com/scikit-build/scikit-build-sample-projects/pull/12/files
nox looks like a great tool! I never used tox myself, it always had this 'heavy' feeling, probably because of the configuration files. 

> It could be then run either from a Nox docker container, using the Nox action, or locally if someone has installed all the required versions of Python, without having the current --no-docker duplication.

If this is possible, that would be great.


Resolving patch:
diff --git a/bin/update_dependencies.py b/bin/update_dependencies.py
--- a/bin/update_dependencies.py
+++ b/bin/update_dependencies.py
@@ -1,141 +1,43 @@
 #!/usr/bin/env python3
-from __future__ import annotations
+# This file supports 3.6+
 
-import configparser
 import os
 import shutil
 import subprocess
 import sys
-from typing import NamedTuple
+from pathlib import Path
 
-import requests
+DIR = Path(__file__).parent.resolve()
+RESOURCES = DIR.parent / "cibuildwheel/resources"
 
-os.chdir(os.path.dirname(__file__))
-os.chdir("..")
+python_version = "".join(str(v) for v in sys.version_info[:2])
+
+env = os.environ.copy()
 
 # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to
 # regenerate the constraints files
-os.environ["CUSTOM_COMPILE_COMMAND"] = "bin/update_dependencies.py"
-
-PYTHON_VERSIONS = ["36", "37", "38", "39"]
-
-if "--no-docker" in sys.argv:
-    for python_version in PYTHON_VERSIONS:
-        subprocess.run(
-            [
-                f"./env{python_version}/bin/pip-compile",
-                "--allow-unsafe",
-                "--upgrade",
-                "cibuildwheel/resources/constraints.in",
-                f"--output-file=cibuildwheel/resources/constraints-python{python_version}.txt",
-            ],
-            check=True,
-        )
-else:
-    image_runner = "quay.io/pypa/manylinux2010_x86_64:latest"
-    subprocess.run(["docker", "pull", image_runner], check=True)
-    for python_version in PYTHON_VERSIONS:
-        abi_flags = "" if int(python_version) >= 38 else "m"
-        python_path = f"/opt/python/cp{python_version}-cp{python_version}{abi_flags}/bin/"
-        command = (
-            f"{python_path}pip install pip-tools && "
-            f"{python_path}pip-compile --allow-unsafe --upgrade "
-            "cibuildwheel/resources/constraints.in "
-            f"--output-file cibuildwheel/resources/constraints-python{python_version}.txt"
-        )
-        subprocess.run(
-            [
-                "docker",
-                "run",
-                "--rm",
-                "--env=CUSTOM_COMPILE_COMMAND",
-                f"--volume={os.getcwd()}:/volume",
-                "--workdir=/volume",
-                image_runner,
-                "bash",
-                "-c",
-                command,
-            ],
-            check=True,
-        )
-
-# default constraints.txt
-shutil.copyfile(
-    f"cibuildwheel/resources/constraints-python{PYTHON_VERSIONS[-1]}.txt",
-    "cibuildwheel/resources/constraints.txt",
+env["CUSTOM_COMPILE_COMMAND"] = "bin/update_dependencies.py"
+
+if python_version == "36":
+    # Bug with click and Python 3.6
+    env["LC_ALL"] = "C.UTF-8"
+    env["LANG"] = "C.UTF-8"
+
+subprocess.run(
+    [
+        "pip-compile",
+        "--allow-unsafe",
+        "--upgrade",
+        f"{RESOURCES}/constraints.in",
+        f"--output-file={RESOURCES}/constraints-python{python_version}.txt",
+    ],
+    check=True,
+    env=env,
 )
 
-
-class Image(NamedTuple):
-    manylinux_version: str
-    platform: str
-    image_name: str
-    tag: str | None
-
-
-images = [
-    Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None),
-    Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None),
-    # 2010 images
-    Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None),
-    Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None),
-    Image("manylinux2010", "pypy_x86_64", "pypywheels/manylinux2010-pypy_x86_64", None),
-    # 2014 images
-    Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None),
-    Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None),
-    Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None),
-    Image("manylinux2014", "ppc64le", "quay.io/pypa/manylinux2014_ppc64le", None),
-    Image("manylinux2014", "s390x", "quay.io/pypa/manylinux2014_s390x", None),
-    # 2_24 images
-    Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None),
-    Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None),
-    Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None),
-    Image("manylinux_2_24", "ppc64le", "quay.io/pypa/manylinux_2_24_ppc64le", None),
-    Image("manylinux_2_24", "s390x", "quay.io/pypa/manylinux_2_24_s390x", None),
-]
-
-config = configparser.ConfigParser()
-
-for image in images:
-    # get the tag name whose digest matches 'latest'
-    if image.tag is not None:
-        # image has been pinned, do not update
-        tag_name = image.tag
-    elif image.image_name.startswith("quay.io/"):
-        _, _, repository_name = image.image_name.partition("/")
-        response = requests.get(
-            f"https://quay.io/api/v1/repository/{repository_name}?includeTags=true"
-        )
-        response.raise_for_status()
-        repo_info = response.json()
-        tags_dict = repo_info["tags"]
-
-        latest_tag = tags_dict.pop("latest")
-        # find the tag whose manifest matches 'latest'
-        tag_name = next(
-            name
-            for (name, info) in tags_dict.items()
-            if info["manifest_digest"] == latest_tag["manifest_digest"]
-        )
-    else:
-        response = requests.get(f"https://hub.docker.com/v2/repositories/{image.image_name}/tags")
-        response.raise_for_status()
-        tags = response.json()["results"]
-
-        latest_tag = next(tag for tag in tags if tag["name"] == "latest")
-        # i don't know what it would mean to have multiple images per tag
-        assert len(latest_tag["images"]) == 1
-        digest = latest_tag["images"][0]["digest"]
-
-        pinned_tag = next(
-            tag for tag in tags if tag != latest_tag and tag["images"][0]["digest"] == digest
-        )
-        tag_name = pinned_tag["name"]
-
-    if not config.has_section(image.platform):
-        config[image.platform] = {}
-
-    config[image.platform][image.manylinux_version] = f"{image.image_name}:{tag_name}"
-
-with open("cibuildwheel/resources/pinned_docker_images.cfg", "w") as f:
-    config.write(f)
+# default constraints.txt
+if python_version == "39":
+    shutil.copyfile(
+        RESOURCES / f"constraints-python{python_version}.txt",
+        RESOURCES / "constraints.txt",
+    )
diff --git a/bin/update_docker.py b/bin/update_docker.py
new file mode 100755
--- /dev/null
+++ b/bin/update_docker.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import configparser
+from pathlib import Path
+from typing import NamedTuple
+
+import requests
+
+DIR = Path(__file__).parent.resolve()
+RESOURCES = DIR.parent / "cibuildwheel/resources"
+
+
+class Image(NamedTuple):
+    manylinux_version: str
+    platform: str
+    image_name: str
+    tag: str | None  # Set this to pin the image
+
+
+images = [
+    Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None),
+    Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None),
+    # 2010 images
+    Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None),
+    Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None),
+    Image("manylinux2010", "pypy_x86_64", "pypywheels/manylinux2010-pypy_x86_64", None),
+    # 2014 images
+    Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None),
+    Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None),
+    Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None),
+    Image("manylinux2014", "ppc64le", "quay.io/pypa/manylinux2014_ppc64le", None),
+    Image("manylinux2014", "s390x", "quay.io/pypa/manylinux2014_s390x", None),
+    # 2_24 images
+    Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None),
+    Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None),
+    Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None),
+    Image("manylinux_2_24", "ppc64le", "quay.io/pypa/manylinux_2_24_ppc64le", None),
+    Image("manylinux_2_24", "s390x", "quay.io/pypa/manylinux_2_24_s390x", None),
+]
+
+config = configparser.ConfigParser()
+
+for image in images:
+    # get the tag name whose digest matches 'latest'
+    if image.tag is not None:
+        # image has been pinned, do not update
+        tag_name = image.tag
+    elif image.image_name.startswith("quay.io/"):
+        _, _, repository_name = image.image_name.partition("/")
+        response = requests.get(
+            f"https://quay.io/api/v1/repository/{repository_name}?includeTags=true"
+        )
+        response.raise_for_status()
+        repo_info = response.json()
+        tags_dict = repo_info["tags"]
+
+        latest_tag = tags_dict.pop("latest")
+        # find the tag whose manifest matches 'latest'
+        tag_name = next(
+            name
+            for (name, info) in tags_dict.items()
+            if info["manifest_digest"] == latest_tag["manifest_digest"]
+        )
+    else:
+        response = requests.get(f"https://hub.docker.com/v2/repositories/{image.image_name}/tags")
+        response.raise_for_status()
+        tags = response.json()["results"]
+
+        latest_tag = next(tag for tag in tags if tag["name"] == "latest")
+        # i don't know what it would mean to have multiple images per tag
+        assert len(latest_tag["images"]) == 1
+        digest = latest_tag["images"][0]["digest"]
+
+        pinned_tag = next(
+            tag for tag in tags if tag != latest_tag and tag["images"][0]["digest"] == digest
+        )
+        tag_name = pinned_tag["name"]
+
+    if not config.has_section(image.platform):
+        config[image.platform] = {}
+
+    config[image.platform][image.manylinux_version] = f"{image.image_name}:{tag_name}"
+
+with open(RESOURCES / "pinned_docker_images.cfg", "w") as f:
+    config.write(f)
diff --git a/noxfile.py b/noxfile.py
new file mode 100644
--- /dev/null
+++ b/noxfile.py
@@ -0,0 +1,93 @@
+import shutil
+import sys
+from pathlib import Path
+
+import nox
+
+nox.options.sessions = ["lint", "tests"]
+
+PYTHON_ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
+
+DIR = Path(__file__).parent.resolve()
+
+
+@nox.session
+def tests(session):
+    """
+    Run the unit and regular tests.
+    """
+    unit_test_args = ["--run-docker"] if sys.platform.startswith("linux") else []
+    session.install("-e", ".[test]")
+    session.run("pytest", "unit_test", *unit_test_args)
+    session.run("pytest", "test", "-x", "--durations", "0", "--timeout=2400", "test")
+
+
+@nox.session
+def lint(session):
+    """
+    Run the linter.
+    """
+    session.install("pre-commit")
+    session.run("pre-commit", "run", "--all-files")
+
+
+@nox.session(python=PYTHON_ALL_VERSIONS)
+def update_constraints(session):
+    """
+    Update the dependencies inplace.
+    """
+    session.install("requests", "pip-tools")
+    session.run("python", "bin/update_dependencies.py")
+
+
+@nox.session
+def update_pins(session):
+    """
+    Update the python and docker pins version inplace.
+    """
+    session.install("-e", ".[dev]")
+    session.run("python", "bin/update_pythons.py", "--force")
+    session.run("python", "bin/update_docker.py")
+
+
+@nox.session
+def update_proj(session):
+    """
+    Update the README inplace.
+    """
+    session.install("-e", ".[dev]")
+    session.run("./bin/projects.py", "docs/data/projects.yml", "--readme=README.md")
+
+
+@nox.session
+def docs(session):
+    """
+    Build the docs.
+    """
+    session.install("-e", ".[docs]")
+
+    if session.posargs:
+        if "serve" in session.posargs:
+            session.run("mkdocs", "serve")
+        else:
+            print("Unrecognized args, use 'serve'")
+    else:
+        session.run("mkdocs", "build")
+
+
+@nox.session
+def build(session):
+    """
+    Build an SDist and wheel.
+    """
+
+    build_p = DIR.joinpath("build")
+    if build_p.exists():
+        shutil.rmtree(build_p)
+
+    dist_p = DIR.joinpath("dist")
+    if dist_p.exists():
+        shutil.rmtree(dist_p)
+
+    session.install("build")
+    session.run("python", "-m", "build")


Test patch: