Repository: cupy/cupy

Issue:
BUG: CUDA_VERSION is set to 0 in sdist
I was investigating the bug reported in https://github.com/conda-forge/cupy-feedstock/issues/98, and it turns out the reason is on Conda-Forge we built CuPy from sdist, but the source has `CUDA_VERSION` inlined to 0:
```c++
// cupy_backends/cuda/api/driver.cpp

/* ... */

static PyObject *__pyx_pf_13cupy_backends_4cuda_3api_6driver_2get_build_version(CYTHON_UNUSED PyObject *__pyx_self) {
  PyObject *__pyx_r = NULL;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("get_build_version", 0);

  /* "cupy_backends/cuda/api/driver.pyx":141
 * 
 * def get_build_version():
 *     return CUDA_VERSION             # <<<<<<<<<<<<<<
 * 
 * 
 */
  __Pyx_XDECREF(__pyx_r);
  __Pyx_INCREF(__pyx_int_0);
  __pyx_r = __pyx_int_0;
  goto __pyx_L0;

  /* "cupy_backends/cuda/api/driver.pyx":140
 * ###############################################################################
 * 
 * def get_build_version():             # <<<<<<<<<<<<<<
 *     return CUDA_VERSION
 * 
 */

  /* function exit code */
  __pyx_L0:;
  __Pyx_XGIVEREF(__pyx_r);
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

/* ... */

static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) {
  if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error);
  __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error)
  return 0;
  __pyx_L1_error:;
  return -1;
}
```
I don't have a good solution or patch for now, as it seems to imply sdist should be generated per CUDA version. Perhaps I am overlooking some obvious fix?

cc: @jakirkham @kmaehashi 


Discussion:
Hmm... I don't come up with the good solution either other than installing the Cython during the installation of CuPy so that CPP code is generated in the user envirnoment.
I think the same can be said for `use_hip` compile time env. (sdist should be buildable on HIP but I guess currently not)
Yeah, in #4556 I am removing `use_hip` and making `HIP_VERSION` propagated together with `CUDA_VERSION`, but it wouldn't solve the problem I think. On Conda-Forge we have no problem switching to the git source (see https://github.com/conda-forge/cupy-feedstock/pull/100), so this issue is more for awareness 🙂
Discussed a bit in the team, adding Cython to `setup_requires` in `setup.py` and exclude cpp files from sdist sounds good to us. This way we can also resolve https://github.com/cupy/cupy/pull/4494 without dirty hacking.
It's great to solve two bugs at once 👍

Resolving patch:
diff --git a/cupy_setup_build.py b/cupy_setup_build.py
--- a/cupy_setup_build.py
+++ b/cupy_setup_build.py
@@ -13,13 +13,14 @@
 import pkg_resources
 import setuptools
 from setuptools.command import build_ext
-from setuptools.command import sdist
 
 from install import build
 from install.build import PLATFORM_LINUX
 from install.build import PLATFORM_WIN32
 
 
+# Cython requirements (minimum version and versions known to be broken).
+# Note: this must be in sync with setup_requires defined in setup.py.
 required_cython_version = pkg_resources.parse_version('0.28.0')
 ignore_cython_versions = [
 ]
@@ -366,14 +367,7 @@ def module_extension_name(file):
 def module_extension_sources(file, use_cython, no_cuda):
     pyx, others = ensure_module_file(file)
     base = path.join(*pyx.split('.'))
-    if use_cython:
-        pyx = base + '.pyx'
-        if not os.path.exists(pyx):
-            use_cython = False
-            print(
-                'NOTICE: Skipping cythonize as {} does not exist.'.format(pyx))
-    if not use_cython:
-        pyx = base + '.cpp'
+    pyx = base + ('.pyx' if use_cython else '.cpp')
 
     # If CUDA SDK is not available, remove CUDA C files from extension sources
     # and use stubs defined in header files.
@@ -805,18 +799,17 @@ def prepare_wheel_libs():
     return [os.path.relpath(x[1], 'cupy') for x in files_to_copy]
 
 
-try:
+def cythonize(extensions, arg_options):
+    # Delay importing Cython as it may be installed via setup_requires if
+    # the user does not have Cython installed.
     import Cython
     import Cython.Build
     cython_version = pkg_resources.parse_version(Cython.__version__)
-    cython_available = (
-        cython_version >= required_cython_version and
-        cython_version not in ignore_cython_versions)
-except ImportError:
-    cython_available = False
-
+    if (cython_version < required_cython_version or
+            cython_version in ignore_cython_versions):
+        raise AssertionError(
+            'Unsupported Cython version: {}'.format(cython_version))
 
-def cythonize(extensions, arg_options):
     directive_keys = ('linetrace', 'profile')
     directives = {key: arg_options[key] for key in directive_keys}
 
@@ -1094,18 +1087,6 @@ def compile(self, sources, **kwargs):
         return other_objects + cu_objects
 
 
-class sdist_with_cython(sdist.sdist):
-
-    """Custom `sdist` command with cyhonizing."""
-
-    def __init__(self, *args, **kwargs):
-        if not cython_available:
-            raise RuntimeError('Cython is required to make sdist.')
-        ext_modules = get_ext_modules(True)  # get .pyx modules
-        cythonize(ext_modules, cupy_setup_options)
-        sdist.sdist.__init__(self, *args, **kwargs)
-
-
 class custom_build_ext(build_ext.build_ext):
 
     """Custom `build_ext` command to include CUDA C source files."""
@@ -1129,9 +1110,8 @@ def _wrap_new_compiler(*args, **kwargs):
             # Intentionally causes DistutilsPlatformError in
             # ccompiler.new_compiler() function to hook.
             self.compiler = 'nvidia'
-        if cython_available:
-            ext_modules = get_ext_modules(True)  # get .pyx modules
-            cythonize(ext_modules, cupy_setup_options)
+        ext_modules = get_ext_modules(True)  # get .pyx modules
+        cythonize(ext_modules, cupy_setup_options)
         check_extensions(self.extensions)
         build_ext.build_ext.run(self)
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -24,9 +24,13 @@
 
 
 requirements = {
+    # setup_requires remains here for pip v18 or earlier.
+    # Keep in sync with pyproject.yaml.
     'setup': [
+        'Cython>=0.28.0',
         'fastrlock>=0.5',
     ],
+
     'install': [
         'numpy>=1.17',
         'fastrlock>=0.5',
@@ -116,7 +120,6 @@ def reduce_requirements(key):
 long_description = cupy_setup_build.get_long_description()
 ext_modules = cupy_setup_build.get_ext_modules()
 build_ext = cupy_setup_build.custom_build_ext
-sdist = cupy_setup_build.sdist_with_cython
 
 here = os.path.abspath(os.path.dirname(__file__))
 # Get __version__ variable
@@ -166,6 +169,5 @@ def reduce_requirements(key):
     tests_require=tests_require,
     extras_require=extras_require,
     ext_modules=ext_modules,
-    cmdclass={'build_ext': build_ext,
-              'sdist': sdist},
+    cmdclass={'build_ext': build_ext},
 )


Test patch: