Repository: cupy/cupy

Issue:
Support cuDNN 8.8
## [Tasks](https://github.com/cupy/cupy/wiki/Actions-Needed-for-Dependency-Update)

- [x] Read Release Notes ([cuTENSOR](https://docs.nvidia.com/cuda/cutensor/index.html#changelog) / [cuSPARSELt](https://docs.nvidia.com/cuda/cusparselt/release_notes.html) / [cuDNN](https://docs.nvidia.com/deeplearning/cudnn/release-notes/index.html) / [NCCL](https://docs.nvidia.com/deeplearning/sdk/nccl-release-notes/index.html)).
- [x] Install cuDNN 8.8 to FlexCI Windows base image
- [x] Fix code and CI to support the new version. (#7472)
  - **FlexCI**: Update `.pfnci/schema.yaml` and `.pfnci/matirx.yaml`.
  - **Library Installer**: Update `cupyx/tools/install_library.py`.
- [x] Backport the above PR. (#7475)
- [x] Fix documentation. (#7478)
  - Update Requirements section in the [Installation Guide](https://docs.cupy.dev/en/latest/install.html)
- [x] Backport the above PR. (#7486)
- [x] Implement new features using the version, if any.

cupy-release-tools:
- [x] https://github.com/cupy/cupy-release-tools/pull/335
- [x] backport https://github.com/cupy/cupy-release-tools/pull/336


Discussion:

Resolving patch:
diff --git a/cupyx/tools/install_library.py b/cupyx/tools/install_library.py
--- a/cupyx/tools/install_library.py
+++ b/cupyx/tools/install_library.py
@@ -26,17 +26,15 @@
 library_records = {}
 
 
-def _make_cudnn_url(public_version, cuda_version, filename):
-    # https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.2/local_installers/11.5/cudnn-linux-x86_64-8.3.2.44_cuda11.5-archive.tar.xz
+def _make_cudnn_url(platform, filename):
+    # https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.8.1.3_cuda12-archive.tar.xz
     return (
-        'https://developer.download.nvidia.com/compute/redist/cudnn' +
-        '/v{}/local_installers/{}/{}'.format(
-            public_version, cuda_version, filename))
+        'https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/' +
+        f'{platform}/{filename}')
 
 
 def __make_cudnn_record(
-        cuda_version, public_version, archive_cuda_version, filename_linux,
-        filename_windows):
+        cuda_version, public_version, filename_linux, filename_windows):
     major_version = public_version.split('.')[0]
     # Dependency order is documented at:
     # https://docs.nvidia.com/deeplearning/cudnn/api/index.html
@@ -48,14 +46,12 @@ def __make_cudnn_record(
         'cudnn': public_version,
         'assets': {
             'Linux': {
-                'url': _make_cudnn_url(
-                    public_version, archive_cuda_version, filename_linux),
+                'url': _make_cudnn_url('linux-x86_64', filename_linux),
                 'filenames': [f'libcudnn{suffix}.so.{public_version}'
                               for suffix in suffix_list]
             },
             'Windows': {
-                'url': _make_cudnn_url(
-                    public_version, archive_cuda_version, filename_windows),
+                'url': _make_cudnn_url('windows-x86_64', filename_windows),
                 'filenames': [f'cudnn{suffix}64_{major_version}.dll'
                               for suffix in suffix_list]
             },
@@ -64,20 +60,21 @@ def __make_cudnn_record(
 
 
 def _make_cudnn_record(cuda_version):
+    cuda_major = int(cuda_version.split('.')[0])
+    assert cuda_major in (11, 12)
     return __make_cudnn_record(
-        cuda_version, '8.7.0', '11.8',
-        'cudnn-linux-x86_64-8.7.0.84_cuda11-archive.tar.xz',
-        'cudnn-windows-x86_64-8.7.0.84_cuda11-archive.zip')
+        cuda_version, '8.8.1',
+        f'cudnn-linux-x86_64-8.8.1.3_cuda{cuda_major}-archive.tar.xz',
+        f'cudnn-windows-x86_64-8.8.1.3_cuda{cuda_major}-archive.zip')
 
 
 # Latest cuDNN versions: https://developer.nvidia.com/rdp/cudnn-download
-# TODO(kmaehashi): cuDNN for CUDA 12 not yet available
-# _cudnn_records.append(_make_cudnn_record('12.x'))
+_cudnn_records.append(_make_cudnn_record('12.x'))
 _cudnn_records.append(_make_cudnn_record('11.x'))  # CUDA 11.2+
 _cudnn_records.append(_make_cudnn_record('11.1'))
 _cudnn_records.append(_make_cudnn_record('11.0'))
 _cudnn_records.append(__make_cudnn_record(
-    '10.2', '8.7.0', '10.2',
+    '10.2', '8.7.0',
     'cudnn-linux-x86_64-8.7.0.84_cuda10-archive.tar.xz',
     'cudnn-windows-x86_64-8.7.0.84_cuda10-archive.zip'))
 library_records['cudnn'] = _cudnn_records


Test patch:
diff --git a/.pfnci/linux/tests/benchmark.Dockerfile b/.pfnci/linux/tests/benchmark.Dockerfile
--- a/.pfnci/linux/tests/benchmark.Dockerfile
+++ b/.pfnci/linux/tests/benchmark.Dockerfile
@@ -14,7 +14,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        && \
     apt-get -qqy install ccache git curl && \
     apt-get -qqy --allow-change-held-packages \
-            --allow-downgrades install 'libnccl2=2.16.*+cuda11.8' 'libnccl-dev=2.16.*+cuda11.8' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.7.*+cuda11.8' 'libcudnn8-dev=8.7.*+cuda11.8'
+            --allow-downgrades install 'libnccl2=2.16.*+cuda11.8' 'libnccl-dev=2.16.*+cuda11.8' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.8.*+cuda11.8' 'libcudnn8-dev=8.8.*+cuda11.8'
 
 ENV PATH "/usr/lib/ccache:${PATH}"
 
diff --git a/.pfnci/linux/tests/benchmark.head.Dockerfile b/.pfnci/linux/tests/benchmark.head.Dockerfile
--- a/.pfnci/linux/tests/benchmark.head.Dockerfile
+++ b/.pfnci/linux/tests/benchmark.head.Dockerfile
@@ -14,7 +14,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        && \
     apt-get -qqy install ccache git curl && \
     apt-get -qqy --allow-change-held-packages \
-            --allow-downgrades install 'libnccl2=2.16.*+cuda11.8' 'libnccl-dev=2.16.*+cuda11.8' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.7.*+cuda11.8' 'libcudnn8-dev=8.7.*+cuda11.8'
+            --allow-downgrades install 'libnccl2=2.16.*+cuda11.8' 'libnccl-dev=2.16.*+cuda11.8' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.8.*+cuda11.8' 'libcudnn8-dev=8.8.*+cuda11.8'
 
 ENV PATH "/usr/lib/ccache:${PATH}"
 
diff --git a/.pfnci/linux/tests/cuda117.Dockerfile b/.pfnci/linux/tests/cuda117.Dockerfile
--- a/.pfnci/linux/tests/cuda117.Dockerfile
+++ b/.pfnci/linux/tests/cuda117.Dockerfile
@@ -14,7 +14,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        && \
     apt-get -qqy install ccache git curl && \
     apt-get -qqy --allow-change-held-packages \
-            --allow-downgrades install 'libnccl2=2.13.*+cuda11.7' 'libnccl-dev=2.13.*+cuda11.7' 'libcutensor1=1.5.*' 'libcutensor-dev=1.5.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.6.*+cuda11.8' 'libcudnn8-dev=8.6.*+cuda11.8'
+            --allow-downgrades install 'libnccl2=2.13.*+cuda11.7' 'libnccl-dev=2.13.*+cuda11.7' 'libcutensor1=1.5.*' 'libcutensor-dev=1.5.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.7.*+cuda11.8' 'libcudnn8-dev=8.7.*+cuda11.8'
 
 ENV PATH "/usr/lib/ccache:${PATH}"
 
diff --git a/.pfnci/linux/tests/cuda117.multi.Dockerfile b/.pfnci/linux/tests/cuda117.multi.Dockerfile
--- a/.pfnci/linux/tests/cuda117.multi.Dockerfile
+++ b/.pfnci/linux/tests/cuda117.multi.Dockerfile
@@ -14,7 +14,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        && \
     apt-get -qqy install ccache git curl && \
     apt-get -qqy --allow-change-held-packages \
-            --allow-downgrades install 'libnccl2=2.13.*+cuda11.7' 'libnccl-dev=2.13.*+cuda11.7' 'libcutensor1=1.5.*' 'libcutensor-dev=1.5.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.6.*+cuda11.8' 'libcudnn8-dev=8.6.*+cuda11.8'
+            --allow-downgrades install 'libnccl2=2.13.*+cuda11.7' 'libnccl-dev=2.13.*+cuda11.7' 'libcutensor1=1.5.*' 'libcutensor-dev=1.5.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.7.*+cuda11.8' 'libcudnn8-dev=8.7.*+cuda11.8'
 
 ENV PATH "/usr/lib/ccache:${PATH}"
 
diff --git a/.pfnci/linux/tests/cuda118.Dockerfile b/.pfnci/linux/tests/cuda118.Dockerfile
--- a/.pfnci/linux/tests/cuda118.Dockerfile
+++ b/.pfnci/linux/tests/cuda118.Dockerfile
@@ -14,7 +14,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        && \
     apt-get -qqy install ccache git curl && \
     apt-get -qqy --allow-change-held-packages \
-            --allow-downgrades install 'libnccl2=2.16.*+cuda11.8' 'libnccl-dev=2.16.*+cuda11.8' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.7.*+cuda11.8' 'libcudnn8-dev=8.7.*+cuda11.8'
+            --allow-downgrades install 'libnccl2=2.16.*+cuda11.8' 'libnccl-dev=2.16.*+cuda11.8' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.8.*+cuda11.8' 'libcudnn8-dev=8.8.*+cuda11.8'
 
 ENV PATH "/usr/lib/ccache:${PATH}"
 
diff --git a/.pfnci/linux/tests/cuda118.multi.Dockerfile b/.pfnci/linux/tests/cuda118.multi.Dockerfile
--- a/.pfnci/linux/tests/cuda118.multi.Dockerfile
+++ b/.pfnci/linux/tests/cuda118.multi.Dockerfile
@@ -14,7 +14,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        && \
     apt-get -qqy install ccache git curl && \
     apt-get -qqy --allow-change-held-packages \
-            --allow-downgrades install 'libnccl2=2.16.*+cuda11.8' 'libnccl-dev=2.16.*+cuda11.8' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.7.*+cuda11.8' 'libcudnn8-dev=8.7.*+cuda11.8'
+            --allow-downgrades install 'libnccl2=2.16.*+cuda11.8' 'libnccl-dev=2.16.*+cuda11.8' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.8.*+cuda11.8' 'libcudnn8-dev=8.8.*+cuda11.8'
 
 ENV PATH "/usr/lib/ccache:${PATH}"
 
diff --git a/.pfnci/linux/tests/cuda120.Dockerfile b/.pfnci/linux/tests/cuda120.Dockerfile
--- a/.pfnci/linux/tests/cuda120.Dockerfile
+++ b/.pfnci/linux/tests/cuda120.Dockerfile
@@ -14,7 +14,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        && \
     apt-get -qqy install ccache git curl && \
     apt-get -qqy --allow-change-held-packages \
-            --allow-downgrades install 'libnccl2=2.16.*+cuda12.0' 'libnccl-dev=2.16.*+cuda12.0' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*'
+            --allow-downgrades install 'libnccl2=2.16.*+cuda12.0' 'libnccl-dev=2.16.*+cuda12.0' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.8.*+cuda12.0' 'libcudnn8-dev=8.8.*+cuda12.0'
 
 ENV PATH "/usr/lib/ccache:${PATH}"
 
diff --git a/.pfnci/linux/tests/cuda120.multi.Dockerfile b/.pfnci/linux/tests/cuda120.multi.Dockerfile
--- a/.pfnci/linux/tests/cuda120.multi.Dockerfile
+++ b/.pfnci/linux/tests/cuda120.multi.Dockerfile
@@ -14,7 +14,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        && \
     apt-get -qqy install ccache git curl && \
     apt-get -qqy --allow-change-held-packages \
-            --allow-downgrades install 'libnccl2=2.16.*+cuda12.0' 'libnccl-dev=2.16.*+cuda12.0' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*'
+            --allow-downgrades install 'libnccl2=2.16.*+cuda12.0' 'libnccl-dev=2.16.*+cuda12.0' 'libcutensor1=1.6.*' 'libcutensor-dev=1.6.*' 'libcusparselt0=0.2.0.*' 'libcusparselt-dev=0.2.0.*' 'libcudnn8=8.8.*+cuda12.0' 'libcudnn8-dev=8.8.*+cuda12.0'
 
 ENV PATH "/usr/lib/ccache:${PATH}"
 
diff --git a/.pfnci/windows/test.ps1 b/.pfnci/windows/test.ps1
--- a/.pfnci/windows/test.ps1
+++ b/.pfnci/windows/test.ps1
@@ -56,7 +56,11 @@ function Main {
     # Setup environment
     echo "Using CUDA $cuda and Python $python"
     ActivateCUDA $cuda
-    ActivateCuDNN "8.6" $cuda
+    if ($cuda -eq "10.2") {
+        ActivateCuDNN "8.6" $cuda
+    } else {
+        ActivateCuDNN "8.8" $cuda
+    }
     ActivateNVTX1
     ActivatePython $python