Repository: python-pillow/Pillow

Issue:
Specify build backend via pyproject.toml
For https://github.com/python-pillow/Pillow/issues/6941.

https://github.com/python-pillow/Pillow/pull/7188 added setuptools to the CI installation for Python 3.12, because 3.12 dropped setuptools:

* https://docs.python.org/3.12/whatsnew/3.12.html#removed

This is actually a bit bigger problem than the CI because users could theoretically have a system that does not have setuptools installed.

---

With pyproject.toml, we can specify the "build backend". In our case, this is setuptools. (The "build frontend" is what reads this file and does the install, often pip.)

So when installing, the frontend will install the required backend, so we don't need to worry whether users have already installed it. We can also specify a minimum version. I just went with the current latest, and it also seems we can remove the extra `PILLOW_VERSION` double quotes hack we needed for Windows on Python 3.8.

It's also possible to move more config from `setup.cfg` to `pyproject.toml`, but I ran into some problems when moving `[options]` and `[options.extras_require]` over, so let's leave those for later. We can also move the tools config, but that can also wait to keep the diff smaller here and the PR focused.

---

About `winbuild/build_prepare.py`: it had a direct invocation of `setup.py` which is a no-go because it assumes setuptools is already installed. So we need `pip install` instead (reminder of https://blog.ganssle.io/tag/setuptools.html).

I went with `--global-option` for now, to follow the current approach elsewhere, but that will need changing as part of https://github.com/python-pillow/Pillow/issues/7167.

I did hit one blocker though: `test-windows.yml` builds a wheel, it calls:

```
winbuild\\build\\build_pillow.cmd --disable-imagequant bdist_wheel
```

Before, that was handled by this: 

```
        r'"{python_dir}\{python_exe}" setup.py build_ext --vendor-raqm --vendor-fribidi %*',  # noqa: E501
```

Which wrote out a command that was called like:

```
2023-04-28T14:37:47.1413330Z D:\a\Pillow\Pillow>"C:\hostedtoolcache\windows\Python\3.8.10\x86\python.exe" setup.py build_ext --vendor-raqm --vendor-fribidi --disable-imagequant bdist_wheel 
```

Now, those two args `--disable-imagequant bdist_wheel` each need wrapping as `--global-option="--disable-imagequant" --global-option="bdist_wheel"`

But this PR does:

```python
        r'"{python_dir}\{python_exe}" -m pip install --upgrade pip',
        r'"{python_dir}\{python_exe}" -m pip install . '
        r'--global-option="--vendor-raqm" '
        r'--global-option="--vendor-fribidi" '
        r'--global-option="%*"',
```

And those two args were both passed into `%*` meaning they both were wrapped in a single `global-option` which doesn't work, so it doesn't build the wheel:

```
2023-05-27T14:56:57.1800310Z D:\a\Pillow\Pillow>"C:\hostedtoolcache\windows\Python\3.8.10\x86\python.exe" -m pip install . --global-option="--vendor-raqm" --global-option="--vendor-fribidi" --global-option="--disable-imagequant bdist_wheel" 
```

Any ideas?

Maybe replace the `build_pillow.cmd` call entirely, and use something like https://pypi.org/project/build/ to call `python -m build --wheel`?



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -137,7 +137,6 @@ class RequiredDependencyException(Exception):
 
 
 PLATFORM_MINGW = os.name == "nt" and "GCC" in sys.version
-PLATFORM_PYPY = hasattr(sys, "pypy_version_info")
 
 
 def _dbg(s, tp=None):
@@ -848,14 +847,7 @@ def build_extensions(self):
         if struct.unpack("h", b"\0\1")[0] == 1:
             defs.append(("WORDS_BIGENDIAN", None))
 
-        if (
-            sys.platform == "win32"
-            and sys.version_info < (3, 9)
-            and not (PLATFORM_PYPY or PLATFORM_MINGW)
-        ):
-            defs.append(("PILLOW_VERSION", f'"\\"{PILLOW_VERSION}\\""'))
-        else:
-            defs.append(("PILLOW_VERSION", f'"{PILLOW_VERSION}"'))
+        defs.append(("PILLOW_VERSION", f'"{PILLOW_VERSION}"'))
 
         self._update_extension("PIL._imaging", libs, defs)
 
diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py
--- a/winbuild/build_prepare.py
+++ b/winbuild/build_prepare.py
@@ -5,7 +5,6 @@
 import shutil
 import struct
 import subprocess
-import sys
 
 
 def cmd_cd(path):
@@ -103,13 +102,6 @@ def cmd_msbuild(
     "ARM64": {"vcvars_arch": "x86_arm64", "msbuild_arch": "ARM64"},
 }
 
-header = [
-    cmd_set("INCLUDE", "{inc_dir}"),
-    cmd_set("INCLIB", "{lib_dir}"),
-    cmd_set("LIB", "{lib_dir}"),
-    cmd_append("PATH", "{bin_dir}"),
-]
-
 # dependencies, listed in order of compilation
 deps = {
     "libjpeg": {
@@ -401,23 +393,12 @@ def find_msvs():
         print("Visual Studio seems to be missing C compiler")
         return None
 
-    vs = {
-        "header": [],
-        # nmake selected by vcvarsall
-        "nmake": "nmake.exe",
-        "vs_dir": vspath,
-    }
-
     # vs2017
     msbuild = os.path.join(vspath, "MSBuild", "15.0", "Bin", "MSBuild.exe")
-    if os.path.isfile(msbuild):
-        vs["msbuild"] = f'"{msbuild}"'
-    else:
+    if not os.path.isfile(msbuild):
         # vs2019
         msbuild = os.path.join(vspath, "MSBuild", "Current", "Bin", "MSBuild.exe")
-        if os.path.isfile(msbuild):
-            vs["msbuild"] = f'"{msbuild}"'
-        else:
+        if not os.path.isfile(msbuild):
             print("Visual Studio MSBuild not found")
             return None
 
@@ -425,9 +406,13 @@ def find_msvs():
     if not os.path.isfile(vcvarsall):
         print("Visual Studio vcvarsall not found")
         return None
-    vs["header"].append(f'call "{vcvarsall}" {{vcvars_arch}}')
 
-    return vs
+    return {
+        "vs_dir": vspath,
+        "msbuild": f'"{msbuild}"',
+        "vcvarsall": f'"{vcvarsall}"',
+        "nmake": "nmake.exe",  # nmake selected by vcvarsall
+    }
 
 
 def extract_dep(url, filename):
@@ -497,6 +482,22 @@ def get_footer(dep):
     return lines
 
 
+def build_env():
+    lines = [
+        "if defined DISTUTILS_USE_SDK goto end",
+        cmd_set("INCLUDE", "{inc_dir}"),
+        cmd_set("INCLIB", "{lib_dir}"),
+        cmd_set("LIB", "{lib_dir}"),
+        cmd_append("PATH", "{bin_dir}"),
+        "call {vcvarsall} {vcvars_arch}",
+        cmd_set("DISTUTILS_USE_SDK", "1"),  # use same compiler to build Pillow
+        cmd_set("py_vcruntime_redist", "true"),  # always use /MD, never /MT
+        ":end",
+        "@echo on",
+    ]
+    write_script("build_env.cmd", lines)
+
+
 def build_dep(name):
     dep = deps[name]
     dir = dep["dir"]
@@ -534,11 +535,11 @@ def build_dep(name):
 
     banner = f"Building {name} ({dir})"
     lines = [
+        r'call "{build_dir}\build_env.cmd"',
         "@echo " + ("=" * 70),
         f"@echo ==== {banner:<60} ====",
         "@echo " + ("=" * 70),
-        "cd /D %s" % os.path.join(sources_dir, dir),
-        *prefs["header"],
+        cmd_cd(os.path.join(sources_dir, dir)),
         *dep.get("build", []),
         *get_footer(dep),
     ]
@@ -548,7 +549,7 @@ def build_dep(name):
 
 
 def build_dep_all():
-    lines = ["@echo on"]
+    lines = [r'call "{build_dir}\build_env.cmd"']
     for dep_name in deps:
         print()
         if dep_name in disabled:
@@ -562,29 +563,16 @@ def build_dep_all():
     write_script("build_dep_all.cmd", lines)
 
 
-def build_pillow():
-    lines = [
-        "@echo ---- Building Pillow (build_ext %*) ----",
-        cmd_cd("{pillow_dir}"),
-        *prefs["header"],
-        cmd_set("DISTUTILS_USE_SDK", "1"),  # use same compiler to build Pillow
-        cmd_set("py_vcruntime_redist", "true"),  # always use /MD, never /MT
-        r'"{python_dir}\{python_exe}" setup.py build_ext --vendor-raqm --vendor-fribidi %*',  # noqa: E501
-    ]
-
-    write_script("build_pillow.cmd", lines)
-
-
 if __name__ == "__main__":
     winbuild_dir = os.path.dirname(os.path.realpath(__file__))
     pillow_dir = os.path.realpath(os.path.join(winbuild_dir, ".."))
 
     parser = argparse.ArgumentParser(
         prog="winbuild\\build_prepare.py",
-        description="Download dependencies and generate build scripts for Pillow.",
+        description="Download and generate build scripts for Pillow dependencies.",
         epilog="""Arguments can also be supplied using the environment variables
-                  PILLOW_BUILD, PILLOW_DEPS, ARCHITECTURE, PYTHON, EXECUTABLE.
-                  See winbuild\\build.rst for more information.""",
+                  PILLOW_BUILD, PILLOW_DEPS, ARCHITECTURE. See winbuild\\build.rst
+                  for more information.""",
     )
     parser.add_argument(
         "-v", "--verbose", action="store_true", help="print generated scripts"
@@ -619,20 +607,6 @@ def build_pillow():
         ),
         help="build architecture (default: same as host Python)",
     )
-    parser.add_argument(
-        "--python",
-        dest="python_dir",
-        metavar="PYTHON",
-        default=os.environ.get("PYTHON"),
-        help="Python install directory (default: use host Python)",
-    )
-    parser.add_argument(
-        "--executable",
-        dest="python_exe",
-        metavar="EXECUTABLE",
-        default=os.environ.get("EXECUTABLE", "python.exe"),
-        help="Python executable (default: use host Python)",
-    )
     parser.add_argument(
         "--nmake",
         dest="cmake_generator",
@@ -657,11 +631,6 @@ def build_pillow():
     arch_prefs = architectures[args.architecture]
     print("Target architecture:", args.architecture)
 
-    if args.python_dir is None:
-        args.python_dir = os.path.dirname(os.path.realpath(sys.executable))
-        args.python_exe = os.path.basename(sys.executable)
-    print("Target Python:", os.path.join(args.python_dir, args.python_exe))
-
     msvs = find_msvs()
     if msvs is None:
         msg = "Visual Studio not found. Please install Visual Studio 2017 or newer."
@@ -699,9 +668,6 @@ def build_pillow():
         disabled += ["fribidi"]
 
     prefs = {
-        # Python paths / preferences
-        "python_dir": args.python_dir,
-        "python_exe": args.python_exe,
         "architecture": args.architecture,
         **arch_prefs,
         # Pillow paths
@@ -719,8 +685,6 @@ def build_pillow():
         "cmake": "cmake.exe",  # TODO find CMAKE automatically
         "cmake_generator": args.cmake_generator,
         # TODO find NASM automatically
-        # script header
-        "header": sum([header, msvs["header"], ["@echo on"]], []),
     }
 
     for k, v in deps.items():
@@ -729,7 +693,5 @@ def build_pillow():
     print()
 
     write_script(".gitignore", ["*"])
+    build_env()
     build_dep_all()
-    if args.verbose:
-        print()
-    build_pillow()


Test patch: