Repository: onnx/onnx

Issue:
pip install in an environment without setuptools fails
# Bug Report

### Is the issue related to model conversion?
<!-- If the ONNX checker reports issues with this model then this is most probably related to the converter used to convert the original framework model to ONNX. Please create this bug in the appropriate converter's GitHub repo (pytorch, tensorflow-onnx, sklearn-onnx, keras-onnx, onnxmltools) to get the best help. -->

No

### Describe the bug
<!-- Please describe the bug clearly and concisely -->

Installing from source in an environment w/o setup tools will fail. We should specify this requirement in pyproject so pip picks it up and install it automatically.

Reference: https://github.com/justinchuby/onnx-script/blob/main/pyproject.toml#L4-L5

### System information
<!--
- OS Platform and Distribution (*e.g. Linux Ubuntu 16.04*):
- ONNX version (*e.g. 1.7*):
- Python version:
- GCC/Compiler version (if compiling from source):
- CMake version:
- Protobuf version:
- Visual Studio version (if applicable):-->

ONNX 1.14



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -2,21 +2,23 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# NOTE: Put all metadata in pyproject.toml.
+# Set the environment variable `ONNX_PREVIEW_BUILD=1` to build the dev preview release.
+
+import contextlib
+import datetime
 import glob
+import logging
 import multiprocessing
 import os
-import pathlib
 import platform
 import shlex
+import shutil
 import subprocess
 import sys
-from collections import namedtuple
-from contextlib import contextmanager
-from datetime import date
-from distutils import log, sysconfig
-from distutils.spawn import find_executable
-from textwrap import dedent
-from typing import ClassVar, List
+import sysconfig
+import textwrap
+from typing import ClassVar
 
 import setuptools
 import setuptools.command.build_ext
@@ -25,19 +27,12 @@
 
 TOP_DIR = os.path.realpath(os.path.dirname(__file__))
 SRC_DIR = os.path.join(TOP_DIR, "onnx")
-TP_DIR = os.path.join(TOP_DIR, "third_party")
 CMAKE_BUILD_DIR = os.path.join(TOP_DIR, ".setuptools-cmake-build")
-PACKAGE_NAME = "onnx"
 
 WINDOWS = os.name == "nt"
 
-CMAKE = find_executable("cmake3") or find_executable("cmake")
-MAKE = find_executable("make")
-
-install_requires = []
-setup_requires = []
-tests_require = []
-extras_require = {}
+CMAKE = shutil.which("cmake3") or shutil.which("cmake")
+MAKE = shutil.which("make")
 
 ################################################################################
 # Global variables for controlling the build variant
@@ -58,6 +53,10 @@
 DEBUG = bool(os.getenv("DEBUG", "0") == "1")
 COVERAGE = bool(os.getenv("COVERAGE", "0") == "1")
 
+# Customize the wheel plat-name, usually needed for MacOS builds.
+# See usage in .github/workflows/release_mac.yml
+ONNX_WHEEL_PLATFORM_NAME = os.getenv("ONNX_WHEEL_PLATFORM_NAME")
+
 ################################################################################
 # Version
 ################################################################################
@@ -69,18 +68,15 @@
         .strip()
     )
 except (OSError, subprocess.CalledProcessError):
-    git_version = None
+    git_version = ""
 
 with open(os.path.join(TOP_DIR, "VERSION_NUMBER")) as version_file:
     VERSION_NUMBER = version_file.read().strip()
-    if "--weekly_build" in sys.argv:
-        today_number = date.today().strftime("%Y%m%d")
+    if os.getenv("ONNX_PREVIEW_BUILD") is not None:
+        # Create the dev build for weekly releases
+        today_number = datetime.date.today().strftime("%Y%m%d")
         VERSION_NUMBER += ".dev" + today_number
-        PACKAGE_NAME = "onnx-weekly"
-        sys.argv.remove("--weekly_build")
-    VersionInfo = namedtuple("VersionInfo", ["version", "git_version"])(
-        version=VERSION_NUMBER, git_version=git_version
-    )
+    VERSION_INFO = {"version": VERSION_NUMBER, "git_version": git_version}
 
 ################################################################################
 # Pre Check
@@ -93,7 +89,7 @@
 ################################################################################
 
 
-@contextmanager
+@contextlib.contextmanager
 def cd(path):
     if not os.path.isabs(path):
         raise RuntimeError(f"Can only cd to absolute path, got: {path}")
@@ -105,6 +101,16 @@ def cd(path):
         os.chdir(orig_path)
 
 
+def get_ext_suffix():
+    if sys.version_info < (3, 8) and sys.platform == "win32":
+        # Workaround for https://bugs.python.org/issue39825
+        # Reference: https://github.com/pytorch/pytorch/commit/4b96fc060b0cb810965b5c8c08bc862a69965667
+        import distutils
+
+        return distutils.sysconfig.get_config_var("EXT_SUFFIX")
+    return sysconfig.get_config_var("EXT_SUFFIX")
+
+
 ################################################################################
 # Customized commands
 ################################################################################
@@ -124,23 +130,20 @@ class CreateVersion(ONNXCommand):
     def run(self):
         with open(os.path.join(SRC_DIR, "version.py"), "w") as f:
             f.write(
-                dedent(
-                    """\
-            # This file is generated by setup.py. DO NOT EDIT!
+                textwrap.dedent(
+                    f"""\
+                    # This file is generated by setup.py. DO NOT EDIT!
 
 
-            version = "{version}"
-            git_version = "{git_version}"
-            """.format(
-                        **dict(VersionInfo._asdict())
-                    )
+                    version = "{VERSION_INFO['version']}"
+                    git_version = "{VERSION_INFO['git_version']}"
+                    """
                 )
             )
 
 
 class CmakeBuild(setuptools.Command):
-    """
-    Compiles everything when `python setup.py build` is run using cmake.
+    """Compiles everything when `python setup.py build` is run using cmake.
 
     Custom args can be passed to cmake by specifying the `CMAKE_ARGS`
     environment variable.
@@ -176,12 +179,12 @@ def run(self):
             # configure
             cmake_args = [
                 CMAKE,
-                f"-DPYTHON_INCLUDE_DIR={sysconfig.get_python_inc()}",
+                f"-DPYTHON_INCLUDE_DIR={sysconfig.get_path('include')}",
                 f"-DPYTHON_EXECUTABLE={sys.executable}",
                 "-DBUILD_ONNX_PYTHON=ON",
                 "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
                 f"-DONNX_NAMESPACE={ONNX_NAMESPACE}",
-                f"-DPY_EXT_SUFFIX={sysconfig.get_config_var('EXT_SUFFIX') or ''}",
+                f"-DPY_EXT_SUFFIX={get_ext_suffix() or ''}",
             ]
             if COVERAGE:
                 cmake_args.append("-DONNX_COVERAGE=ON")
@@ -225,10 +228,10 @@ def run(self):
                 extra_cmake_args = shlex.split(os.environ["CMAKE_ARGS"])
                 # prevent crossfire with downstream scripts
                 del os.environ["CMAKE_ARGS"]
-                log.info(f"Extra cmake args: {extra_cmake_args}")
+                logging.info("Extra cmake args: %s", extra_cmake_args)
                 cmake_args.extend(extra_cmake_args)
             cmake_args.append(TOP_DIR)
-            log.info(f"Using cmake args: {cmake_args}")
+            logging.info("Using cmake args: %s", cmake_args)
             if "-DONNX_DISABLE_EXCEPTIONS=ON" in cmake_args:
                 raise RuntimeError(
                     "-DONNX_DISABLE_EXCEPTIONS=ON option is only available for c++ builds. Python binding require exceptions to be enabled."
@@ -268,8 +271,7 @@ def run(self):
         for src in generated_python_files:
             dst = os.path.join(TOP_DIR, os.path.relpath(src, CMAKE_BUILD_DIR))
             self.copy_file(src, dst)
-        # TODO (https://github.com/pypa/setuptools/issues/3606)
-        # Review the command customisations to enable editable_mode
+        # Review the command customizations to enable editable_mode
         self.editable_mode = False
         return setuptools.command.build_py.build_py.run(self)
 
@@ -291,7 +293,7 @@ def build_extensions(self):
             filename = os.path.basename(self.get_ext_filename(fullname))
 
             lib_path = CMAKE_BUILD_DIR
-            if os.name == "nt":
+            if WINDOWS:
                 debug_lib_dir = os.path.join(lib_path, "Debug")
                 release_lib_dir = os.path.join(lib_path, "Release")
                 if os.path.exists(debug_lib_dir):
@@ -303,7 +305,7 @@ def build_extensions(self):
             self.copy_file(src, dst)
 
 
-CMDCLASS = {
+CMD_CLASS = {
     "create_version": CreateVersion,
     "cmake_build": CmakeBuild,
     "build_py": BuildPy,
@@ -315,98 +317,18 @@ def build_extensions(self):
 # Extensions
 ################################################################################
 
-ext_modules = [setuptools.Extension(name="onnx.onnx_cpp2py_export", sources=[])]
-
-################################################################################
-# Packages
-################################################################################
-
-# Add package directories here if you want to package them with the source
-# TODO try to remove unnecessary .cpp files
-include_dirs = [
-    "onnx.backend.test.data.*",
-    "onnx.common",
-    "onnx.defs.*",
-    "onnx.examples*",
-    "onnx.shape_inference",
-    "onnx.test.cpp",
-    "onnx.version_converter*",
-]
-
-packages = setuptools.find_packages() + setuptools.find_namespace_packages(
-    include=include_dirs
-)
+EXT_MODULES = [setuptools.Extension(name="onnx.onnx_cpp2py_export", sources=[])]
 
 
-def load_packages_from_requirements(requirements_file: str) -> List[str]:
-    """Load required packages from requirements-*.txt.
-
-    Arguments:
-        requirements_file {str} -- requirements file name (e.g. requirements.txt)
-    Returns:
-        List[str] -- list of required packages
-    """
-
-    requirements_path = os.path.join(os.getcwd(), requirements_file)
-    if not os.path.exists(requirements_path):
-        this = os.path.dirname(__file__)
-        requirements_path = os.path.join(this, requirements_file)
-    if not os.path.exists(requirements_path):
-        raise FileNotFoundError("Unable to find " + requirements_file)
-    requires_list = []
-    with open(requirements_path) as f:
-        requires_list = f.read().splitlines()
-    return requires_list
-
-
-install_requires = load_packages_from_requirements("requirements.txt")
-
-################################################################################
-# Test
-################################################################################
-
-setup_requires.append("pytest-runner")
-tests_require.append("pytest")
-tests_require.append("nbval")
-tests_require.append("tabulate")
-
-extras_require["lint"] = [
-    "lintrunner>=0.10.0",
-    "lintrunner-adapters>=0.3",
-]
-
-extras_require["reference"] = load_packages_from_requirements(
-    "requirements-reference.txt"
-)
-
 ################################################################################
 # Final
 ################################################################################
 
 setuptools.setup(
-    name=PACKAGE_NAME,
-    version=VersionInfo.version,
-    description="Open Neural Network Exchange",
-    long_description=pathlib.Path("README.md").read_text(),
-    long_description_content_type="text/markdown",
-    ext_modules=ext_modules,
-    cmdclass=CMDCLASS,
-    packages=packages,
-    license="Apache License v2.0",
-    include_package_data=True,
-    package_data={"onnx": ["py.typed", "*.pyi"]},
-    install_requires=install_requires,
-    setup_requires=setup_requires,
-    tests_require=tests_require,
-    extras_require=extras_require,
-    author="ONNX",
-    author_email="onnx-technical-discuss@lists.lfaidata.foundation",
-    url="https://github.com/onnx/onnx",
-    entry_points={
-        "console_scripts": [
-            "check-model = onnx.bin.checker:check_model",
-            "check-node = onnx.bin.checker:check_node",
-            "backend-test-tools = onnx.backend.test.cmd_tools:main",
-        ]
-    },
+    ext_modules=EXT_MODULES,
+    cmdclass=CMD_CLASS,
+    version=VERSION_INFO["version"],
+    options={"bdist_wheel": {"plat_name": ONNX_WHEEL_PLATFORM_NAME}}
+    if ONNX_WHEEL_PLATFORM_NAME is not None
+    else {},
 )


Test patch: