Repository: rapidsai/dask-cuda (language: python)

Issue:
Support Pandas version 2
Pandas 2.0.0 was released in April 2023. We should spend some effort to make this project compatible with the 2.y versions. Pandas 3 also has a dev release out, so maybe we can try for that as well now.
Forward-merge branch-24.02 to branch-24.04
Forward-merge triggered by push to `branch-24.02` that creates a PR to keep `branch-24.04` up-to-date. If this PR is unable to be immediately merged due to conflicts, it will remain open for the team to manually merge.
Support CUDA 12.2
## Description

* switches to CUDA 12.2.2 for building conda packages and wheels
* adds new tests running against CUDA 12.2.2

This is part of ongoing work to build and test wheels against CUDA 12.2.2 across all of RAPIDS. For more details see:

* https://github.com/rapidsai/build-planning/issues/7
* https://github.com/rapidsai/shared-workflows/pull/166

### Notes for Reviewers

Planning a second round of PRs to revert these references back to a proper `branch-24.{nn}` release branch of `shared-workflows` once https://github.com/rapidsai/shared-workflows/pull/166 is merged.


Discussion / hints:

Fix patch:
diff --git a/.github/ops-bot.yaml b/.github/ops-bot.yaml
index 9a0b41550..2ed5231ae 100644
--- a/.github/ops-bot.yaml
+++ b/.github/ops-bot.yaml
@@ -6,3 +6,4 @@ branch_checker: true
 label_checker: true
 release_drafter: true
 recently_updated: true
+forward_merger: true
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 8679cd61b..5e63e62d4 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -28,7 +28,7 @@ concurrency:
 jobs:
   conda-python-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -38,7 +38,7 @@ jobs:
     if: github.ref_type == 'branch'
     needs: [conda-python-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04
     with:
       arch: "amd64"
       branch: ${{ inputs.branch }}
@@ -51,7 +51,7 @@ jobs:
   upload-conda:
     needs: [conda-python-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.04
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -59,7 +59,7 @@ jobs:
       sha: ${{ inputs.sha }}
   wheel-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -67,11 +67,12 @@ jobs:
       date: ${{ inputs.date }}
       script: ci/build_wheel.sh
       # Package is pure Python and only ever requires one build.
-      matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and .CUDA_VER == "12.0.1"))
+      # This selects "ARCH=amd64 + the latest supported Python + CUDA".
+      matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.]
   wheel-publish:
     needs: wheel-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.04
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
index 8d701e0ae..c63386567 100644
--- a/.github/workflows/pr.yaml
+++ b/.github/workflows/pr.yaml
@@ -18,26 +18,26 @@ jobs:
       - docs-build
       - wheel-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.04
   checks:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.04
   conda-python-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04
     with:
       build_type: pull-request
   conda-python-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04
     with:
       build_type: pull-request
   docs-build:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
@@ -46,9 +46,10 @@ jobs:
       run_script: "ci/build_docs.sh"
   wheel-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04
     with:
       build_type: pull-request
       # Package is pure Python and only ever requires one build.
-      matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and .CUDA_VER == "12.0.1"))
+      # This selects "ARCH=amd64 + the latest supported Python + CUDA".
+      matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.]
       script: "ci/build_wheel.sh"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 126cf46c2..9ffd9105b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,25 @@
+# dask-cuda 24.04.00 (10 Apr 2024)
+
+## 🐛 Bug Fixes
+
+- handle more RAPIDS version formats in update-version.sh ([#1307](https://github.com/rapidsai/dask-cuda/pull/1307)) [@jameslamb](https://github.com/jameslamb)
+
+## 🚀 New Features
+
+- Allow using pandas 2 ([#1308](https://github.com/rapidsai/dask-cuda/pull/1308)) [@vyasr](https://github.com/vyasr)
+- Support CUDA 12.2 ([#1302](https://github.com/rapidsai/dask-cuda/pull/1302)) [@jameslamb](https://github.com/jameslamb)
+
+## 🛠️ Improvements
+
+- Use `conda env create --yes` instead of `--force` ([#1326](https://github.com/rapidsai/dask-cuda/pull/1326)) [@bdice](https://github.com/bdice)
+- Add upper bound to prevent usage of NumPy 2 ([#1320](https://github.com/rapidsai/dask-cuda/pull/1320)) [@bdice](https://github.com/bdice)
+- Generalize GHA selectors for pure Python testing ([#1318](https://github.com/rapidsai/dask-cuda/pull/1318)) [@jakirkham](https://github.com/jakirkham)
+- Requre NumPy 1.23+ ([#1316](https://github.com/rapidsai/dask-cuda/pull/1316)) [@jakirkham](https://github.com/jakirkham)
+- Add support for Python 3.11 ([#1315](https://github.com/rapidsai/dask-cuda/pull/1315)) [@jameslamb](https://github.com/jameslamb)
+- target branch-24.04 for GitHub Actions workflows ([#1314](https://github.com/rapidsai/dask-cuda/pull/1314)) [@jameslamb](https://github.com/jameslamb)
+- Filter dd deprecation ([#1312](https://github.com/rapidsai/dask-cuda/pull/1312)) [@rjzamora](https://github.com/rjzamora)
+- Update ops-bot.yaml ([#1310](https://github.com/rapidsai/dask-cuda/pull/1310)) [@AyodeAwe](https://github.com/AyodeAwe)
+
 # dask-cuda 24.02.00 (12 Feb 2024)
 
 ## 🚨 Breaking Changes
diff --git a/VERSION b/VERSION
index 3c6c5e2b7..4a2fe8aa5 100644
--- a/VERSION
+++ b/VERSION
@@ -1,1 +1,1 @@
-24.02.00
+24.04.00
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
index 2abfce6bd..8376e254a 100755
--- a/ci/build_docs.sh
+++ b/ci/build_docs.sh
@@ -10,7 +10,7 @@ rapids-dependency-file-generator \
     --file_key docs \
     --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
-rapids-mamba-retry env create --force -f env.yaml -n docs
+rapids-mamba-retry env create --yes -f env.yaml -n docs
 conda activate docs
 
 rapids-print-env
@@ -23,7 +23,7 @@ rapids-mamba-retry install \
     --channel "${PYTHON_CHANNEL}" \
     dask-cuda
 
-export RAPIDS_VERSION_NUMBER="24.02"
+export RAPIDS_VERSION_NUMBER="24.04"
 export RAPIDS_DOCS_DIR="$(mktemp -d)"
 
 rapids-logger "Build Python docs"
diff --git a/ci/check_style.sh b/ci/check_style.sh
index be3ac3f4b..9bc26fe71 100755
--- a/ci/check_style.sh
+++ b/ci/check_style.sh
@@ -11,7 +11,7 @@ rapids-dependency-file-generator \
   --file_key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
-rapids-mamba-retry env create --force -f env.yaml -n checks
+rapids-mamba-retry env create --yes -f env.yaml -n checks
 conda activate checks
 
 # Run pre-commit checks
diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
index b2fc490f1..9f40318b2 100755
--- a/ci/release/update-version.sh
+++ b/ci/release/update-version.sh
@@ -35,13 +35,21 @@ function sed_runner() {
 # Centralized version file update
 echo "${NEXT_FULL_TAG}" | tr -d '"' > VERSION
 
-# Bump cudf and dask-cudf testing dependencies
-sed_runner "s/cudf==.*/cudf==${NEXT_SHORT_TAG_PEP440}.*/g" dependencies.yaml
-sed_runner "s/dask-cudf==.*/dask-cudf==${NEXT_SHORT_TAG_PEP440}.*/g" dependencies.yaml
-sed_runner "s/kvikio==.*/kvikio==${NEXT_SHORT_TAG_PEP440}.*/g" dependencies.yaml
+# Bump testing dependencies
 sed_runner "s/ucx-py==.*/ucx-py==${NEXT_UCXPY_VERSION}.*/g" dependencies.yaml
 sed_runner "s/ucxx==.*/ucxx==${NEXT_UCXPY_VERSION}.*/g" dependencies.yaml
-sed_runner "s/rapids-dask-dependency==.*/rapids-dask-dependency==${NEXT_SHORT_TAG_PEP440}.*/g" dependencies.yaml
+
+DEPENDENCIES=(
+  cudf
+  dask-cudf
+  kvikio
+  rapids-dask-dependency
+)
+for FILE in dependencies.yaml conda/environments/*.yaml; do
+  for DEP in "${DEPENDENCIES[@]}"; do
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}"
+  done
+done
 
 # CI files
 for FILE in .github/workflows/*.yaml; do
diff --git a/conda/environments/all_cuda-114_arch-x86_64.yaml b/conda/environments/all_cuda-114_arch-x86_64.yaml
index 4b76616ab..c264b8352 100644
--- a/conda/environments/all_cuda-114_arch-x86_64.yaml
+++ b/conda/environments/all_cuda-114_arch-x86_64.yaml
@@ -10,27 +10,27 @@ dependencies:
 - click >=8.1
 - cuda-version=11.4
 - cudatoolkit
-- cudf==24.2.*
-- dask-cudf==24.2.*
-- distributed-ucxx==0.36.*
-- kvikio==24.2.*
+- cudf==24.4.*
+- dask-cudf==24.4.*
+- distributed-ucxx==0.37.*
+- kvikio==24.4.*
 - numactl-devel-cos7-x86_64
 - numba>=0.57
-- numpy>=1.21
+- numpy>=1.23,<2.0a0
 - numpydoc>=1.1.0
-- pandas>=1.3,<1.6.0.dev0
+- pandas>=1.3
 - pre-commit
 - pynvml>=11.0.0,<11.5
 - pytest
 - pytest-cov
-- python>=3.9,<3.11
-- rapids-dask-dependency==24.2.*
+- python>=3.9,<3.12
+- rapids-dask-dependency==24.4.*
 - setuptools>=64.0.0
 - sphinx
 - sphinx-click>=2.7.1
 - sphinx-rtd-theme>=0.5.1
 - ucx-proc=*=gpu
-- ucx-py==0.36.*
-- ucxx==0.36.*
+- ucx-py==0.37.*
+- ucxx==0.37.*
 - zict>=2.0.0
 name: all_cuda-114_arch-x86_64
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
index bb23025eb..882d0b07a 100644
--- a/conda/environments/all_cuda-118_arch-x86_64.yaml
+++ b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -10,27 +10,27 @@ dependencies:
 - click >=8.1
 - cuda-version=11.8
 - cudatoolkit
-- cudf==24.2.*
-- dask-cudf==24.2.*
-- distributed-ucxx==0.36.*
-- kvikio==24.2.*
+- cudf==24.4.*
+- dask-cudf==24.4.*
+- distributed-ucxx==0.37.*
+- kvikio==24.4.*
 - numactl-devel-cos7-x86_64
 - numba>=0.57
-- numpy>=1.21
+- numpy>=1.23,<2.0a0
 - numpydoc>=1.1.0
-- pandas>=1.3,<1.6.0.dev0
+- pandas>=1.3
 - pre-commit
 - pynvml>=11.0.0,<11.5
 - pytest
 - pytest-cov
-- python>=3.9,<3.11
-- rapids-dask-dependency==24.2.*
+- python>=3.9,<3.12
+- rapids-dask-dependency==24.4.*
 - setuptools>=64.0.0
 - sphinx
 - sphinx-click>=2.7.1
 - sphinx-rtd-theme>=0.5.1
 - ucx-proc=*=gpu
-- ucx-py==0.36.*
-- ucxx==0.36.*
+- ucx-py==0.37.*
+- ucxx==0.37.*
 - zict>=2.0.0
 name: all_cuda-118_arch-x86_64
diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml
similarity index 67%
rename from conda/environments/all_cuda-120_arch-x86_64.yaml
rename to conda/environments/all_cuda-122_arch-x86_64.yaml
index a0dec45da..afa093289 100644
--- a/conda/environments/all_cuda-120_arch-x86_64.yaml
+++ b/conda/environments/all_cuda-122_arch-x86_64.yaml
@@ -10,28 +10,28 @@ dependencies:
 - click >=8.1
 - cuda-nvcc-impl
 - cuda-nvrtc
-- cuda-version=12.0
-- cudf==24.2.*
-- dask-cudf==24.2.*
-- distributed-ucxx==0.36.*
-- kvikio==24.2.*
+- cuda-version=12.2
+- cudf==24.4.*
+- dask-cudf==24.4.*
+- distributed-ucxx==0.37.*
+- kvikio==24.4.*
 - numactl-devel-cos7-x86_64
 - numba>=0.57
-- numpy>=1.21
+- numpy>=1.23,<2.0a0
 - numpydoc>=1.1.0
-- pandas>=1.3,<1.6.0.dev0
+- pandas>=1.3
 - pre-commit
 - pynvml>=11.0.0,<11.5
 - pytest
 - pytest-cov
-- python>=3.9,<3.11
-- rapids-dask-dependency==24.2.*
+- python>=3.9,<3.12
+- rapids-dask-dependency==24.4.*
 - setuptools>=64.0.0
 - sphinx
 - sphinx-click>=2.7.1
 - sphinx-rtd-theme>=0.5.1
 - ucx-proc=*=gpu
-- ucx-py==0.36.*
-- ucxx==0.36.*
+- ucx-py==0.37.*
+- ucxx==0.37.*
 - zict>=2.0.0
-name: all_cuda-120_arch-x86_64
+name: all_cuda-122_arch-x86_64
diff --git a/dask_cuda/__init__.py b/dask_cuda/__init__.py
index dbbb1f7fb..30f987ac4 100644
--- a/dask_cuda/__init__.py
+++ b/dask_cuda/__init__.py
@@ -3,7 +3,6 @@
 if sys.platform != "linux":
     raise ImportError("Only Linux is supported by Dask-CUDA at this time")
 
-
 import dask
 import dask.utils
 import dask.dataframe.core
diff --git a/dependencies.yaml b/dependencies.yaml
index eb7148615..33a931e55 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -3,7 +3,7 @@ files:
   all:
     output: conda
     matrix:
-      cuda: ["11.4", "11.8", "12.0"]
+      cuda: ["11.4", "11.8", "12.2"]
       arch: [x86_64]
     includes:
       - build_python
@@ -98,6 +98,10 @@ dependencies:
               cuda: "12.0"
             packages:
               - cuda-version=12.0
+          - matrix:
+              cuda: "12.2"
+            packages:
+              - cuda-version=12.2
   cuda:
     specific:
       - output_types: conda
@@ -136,35 +140,39 @@ dependencies:
               py: "3.10"
             packages:
               - python=3.10
+          - matrix:
+              py: "3.11"
+            packages:
+              - python=3.11
           - matrix:
             packages:
-              - python>=3.9,<3.11
+              - python>=3.9,<3.12
   run_python:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
           - click >=8.1
           - numba>=0.57
-          - numpy>=1.21
-          - pandas>=1.3,<1.6.0.dev0
+          - numpy>=1.23,<2.0a0
+          - pandas>=1.3
           - pynvml>=11.0.0,<11.5
-          - rapids-dask-dependency==24.2.*
+          - rapids-dask-dependency==24.4.*
           - zict>=2.0.0
   test_python:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - cudf==24.2.*
-          - dask-cudf==24.2.*
-          - kvikio==24.2.*
+          - cudf==24.4.*
+          - dask-cudf==24.4.*
+          - kvikio==24.4.*
           - pytest
           - pytest-cov
-          - ucx-py==0.36.*
+          - ucx-py==0.37.*
       - output_types: [conda]
         packages:
-          - distributed-ucxx==0.36.*
+          - distributed-ucxx==0.37.*
           - ucx-proc=*=gpu
-          - ucxx==0.36.*
+          - ucxx==0.37.*
     specific:
       - output_types: conda
         matrices:
diff --git a/pyproject.toml b/pyproject.toml
index 6668e3f93..849573179 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,10 +18,10 @@ requires-python = ">=3.9"
 dependencies = [
     "click >=8.1",
     "numba>=0.57",
-    "numpy>=1.21",
-    "pandas>=1.3,<1.6.0.dev0",
+    "numpy>=1.23,<2.0a0",
+    "pandas>=1.3",
     "pynvml>=11.0.0,<11.5",
-    "rapids-dask-dependency==24.2.*",
+    "rapids-dask-dependency==24.4.*",
     "zict>=2.0.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
@@ -32,6 +32,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
 ]
 
 [project.scripts]
@@ -49,12 +50,12 @@ docs = [
     "sphinx-rtd-theme>=0.5.1",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 test = [
-    "cudf==24.2.*",
-    "dask-cudf==24.2.*",
-    "kvikio==24.2.*",
+    "cudf==24.4.*",
+    "dask-cudf==24.4.*",
+    "kvikio==24.4.*",
     "pytest",
     "pytest-cov",
-    "ucx-py==0.36.*",
+    "ucx-py==0.37.*",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
@@ -123,6 +124,10 @@ filterwarnings = [
     "error::FutureWarning",
     # remove after https://github.com/rapidsai/dask-cuda/issues/1087 is closed
     "ignore:There is no current event loop:DeprecationWarning:tornado",
+    # This warning must be filtered until dask-expr support
+    # is enabled in both dask-cudf and dask-cuda.
+    # See: https://github.com/rapidsai/dask-cuda/issues/1311
+    "ignore:Dask DataFrame implementation is deprecated:DeprecationWarning",
 ]
 
 [tool.setuptools]


Test patch:
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 9bf068690..011517e1a 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -16,7 +16,7 @@ on:
 jobs:
   conda-python-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
diff --git a/ci/test_python.sh b/ci/test_python.sh
index f700c935b..fab53af39 100755
--- a/ci/test_python.sh
+++ b/ci/test_python.sh
@@ -11,7 +11,7 @@ rapids-dependency-file-generator \
   --file_key test_python \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
-rapids-mamba-retry env create --force -f env.yaml -n test
+rapids-mamba-retry env create --yes -f env.yaml -n test
 
 # Temporarily allow unbound variables for conda activation.
 set +u