Repository: coiled/benchmarks (language: python)

Issue:
Migrate AB runs (and database) to 3.10
https://github.com/coiled/dask-engineering/issues/256


Discussion / hints:

Fix patch:
diff --git a/AB_environments/AB_baseline.conda.yaml b/AB_environments/AB_baseline.conda.yaml
index f0bd820acf..2aeb5ce2cc 100644
--- a/AB_environments/AB_baseline.conda.yaml
+++ b/AB_environments/AB_baseline.conda.yaml
@@ -5,42 +5,43 @@
 channels:
   - conda-forge
 dependencies:
-  - python =3.9  # Single '=' means latest patch version available
+  - python =3.10  # Single '=' means latest patch version available
   # Copy-paste from ci/environment.yml
   - pip
   - coiled >=0.2.54
-  - numpy ==1.26.2
-  - pandas ==2.1.4
-  - dask ==2023.12.0
-  - distributed ==2023.12.0
-  - dask-expr ==0.2.8
+  - numpy ==1.26.4
+  - pandas ==2.2.2
+  - dask ==2024-07.1
+  - distributed ==2024-07.1
+  - dask-expr ==1.1.9
   - dask-labextension ==7.0.0
-  - dask-ml ==2023.3.24
-  - fsspec ==2023.12.1
-  - s3fs ==2023.12.1
-  - gcsfs ==2023.12.1
-  - pyarrow ==15.0.0
-  - jupyterlab ==4.0.9
-  - lz4 ==4.3.2
-  - ipywidgets ==8.1.1
-  - numba ==0.58.1
-  - scikit-learn ==1.3.2
+  - dask-ml ==2024.4.4
+  - fsspec ==2024.6.1
+  - s3fs ==2024.6.1
+  - gcsfs ==2024.6.1
+  - pyarrow ==17.0.0
+  - jupyterlab ==4.2.4
+  - lz4 ==4.3.3
+  - ipywidgets ==8.1.3
+  - numba ==0.60.0
+  - scikit-learn ==1.5.1
   - ipycytoscape ==1.3.3
   - click ==8.1.7
-  - xarray ==2023.12.0
-  - zarr ==2.16.1
-  - cftime ==1.6.3
-  - msgpack-python ==1.0.7
+  - xarray ==2024.07.0
+  - zarr ==2.18.2
+  - cftime ==1.6.4
+  - msgpack-python
   - cloudpickle ==3.0.0
-  - tornado ==6.3.3
-  - toolz ==0.12.0
+  - tornado ==6.4.1
+  - toolz ==0.12.1
   - zict ==3.0.0
-  - xgboost ==1.7.6
-  - optuna ==3.5.0
-  - scipy ==1.11.4
-  - sqlalchemy ==2.0.23
-  - pynvml ==11.5.0
-  - bokeh ==3.3.2
+  - xgboost ==2.1.1
+  - optuna ==3.6.1
+  - optuna-integration ==3.6.0
+  - scipy ==1.14.0
+  - sqlalchemy ==2.0.31
+  - pynvml ==11.5.3
+  - bokeh ==3.5.1
   - gilknocker ==0.4.1
   - openssl >1.1.0g
   # End copy-paste
diff --git a/AB_environments/AB_sample.conda.yaml b/AB_environments/AB_sample.conda.yaml
index 0d885703d5..77682e5c1c 100644
--- a/AB_environments/AB_sample.conda.yaml
+++ b/AB_environments/AB_sample.conda.yaml
@@ -7,45 +7,47 @@
 # - AB_<name>.cluster.yaml
 
 # You should always start from a copy-paste from AB_baseline.conda.yaml
+
 channels:
   - conda-forge
 dependencies:
-  - python =3.9  # Single '=' means latest patch version available
-  # Copy-paste from AB_baseline.conda.yaml
+  - python =3.10  # Single '=' means latest patch version available
+  # Copy-paste from ci/environment.yml
   - pip
   - coiled >=0.2.54
-  - numpy ==1.26.2
-  - pandas ==2.1.4
-  - dask ==2023.12.0
-  - distributed ==2023.12.0
-  - dask-expr ==0.2.8
+  - numpy ==1.26.4
+  - pandas ==2.2.2
+  - dask ==2024-07.1
+  - distributed ==2024-07.1
+  - dask-expr ==1.1.9
   - dask-labextension ==7.0.0
-  - dask-ml ==2023.3.24
-  - fsspec ==2023.12.1
-  - s3fs ==2023.12.1
-  - gcsfs ==2023.12.1
-  - pyarrow ==15.0.0
-  - jupyterlab ==4.0.9
-  - lz4 ==4.3.2
-  - ipywidgets ==8.1.1
-  - numba ==0.58.1
-  - scikit-learn ==1.3.2
+  - dask-ml ==2024.4.4
+  - fsspec ==2024.6.1
+  - s3fs ==2024.6.1
+  - gcsfs ==2024.6.1
+  - pyarrow ==17.0.0
+  - jupyterlab ==4.2.4
+  - lz4 ==4.3.3
+  - ipywidgets ==8.1.3
+  - numba ==0.60.0
+  - scikit-learn ==1.5.1
   - ipycytoscape ==1.3.3
   - click ==8.1.7
-  - xarray ==2023.12.0
-  - zarr ==2.16.1
-  - cftime ==1.6.3
-  - msgpack-python ==1.0.7
+  - xarray ==2024.07.0
+  - zarr ==2.18.2
+  - cftime ==1.6.4
+  - msgpack-python
   - cloudpickle ==3.0.0
-  - tornado ==6.3.3
-  - toolz ==0.12.0
+  - tornado ==6.4.1
+  - toolz ==0.12.1
   - zict ==3.0.0
-  - xgboost ==1.7.6
-  - optuna ==3.5.0
-  - scipy ==1.11.4
-  - sqlalchemy ==2.0.23
-  - pynvml ==11.5.0
-  - bokeh ==3.3.2
+  - xgboost ==2.1.1
+  - optuna ==3.6.1
+  - optuna-integration ==3.6.0
+  - scipy ==1.14.0
+  - sqlalchemy ==2.0.31
+  - pynvml ==11.5.3
+  - bokeh ==3.5.1
   - gilknocker ==0.4.1
   - openssl >1.1.0g
   # End copy-paste
diff --git a/ci/environment.yml b/ci/environment.yml
index 387b2189e9..8fafe24f32 100644
--- a/ci/environment.yml
+++ b/ci/environment.yml
@@ -8,41 +8,42 @@ dependencies:
 # - AB_environments/AB_sample.conda.yaml
 ########################################################
 
-  - python >=3.9
+  - python >=3.10
   - pip
   - coiled >=0.2.54
-  - numpy ==1.26.2
-  - pandas ==2.2.1
-  - dask ==2023.12.0
-  - distributed ==2023.12.0
-  - dask-expr ==0.2.8
+  - numpy ==1.26.4
+  - pandas ==2.2.2
+  - dask ==2024-07.1
+  - distributed ==2024-07.1
+  - dask-expr ==1.1.9
   - dask-labextension ==7.0.0
-  - dask-ml ==2023.3.24
-  - fsspec ==2023.12.1
-  - s3fs ==2023.12.1
-  - gcsfs ==2023.12.1
+  - dask-ml ==2024.4.4
+  - fsspec ==2024.6.1
+  - s3fs ==2024.6.1
+  - gcsfs ==2024.6.1
   - pyarrow ==15.0.0
-  - jupyterlab ==4.0.9
-  - lz4 ==4.3.2
-  - ipywidgets ==8.1.1
-  - numba ==0.58.1
-  - scikit-learn ==1.3.2
+  - jupyterlab ==4.2.4
+  - lz4 ==4.3.3
+  - ipywidgets ==8.1.3
+  - numba ==0.60.0
+  - scikit-learn ==1.5.1
   - ipycytoscape ==1.3.3
   - click ==8.1.7
-  - xarray ==2023.12.0
-  - zarr ==2.16.1
-  - cftime ==1.6.3
-  - msgpack-python ==1.0.7
+  - xarray ==2024.07.0
+  - zarr ==2.18.2
+  - cftime ==1.6.4
+  - msgpack-python
   - cloudpickle ==3.0.0
-  - tornado ==6.3.3
-  - toolz ==0.12.0
+  - tornado ==6.4.1
+  - toolz ==0.12.1
   - zict ==3.0.0
-  - xgboost ==1.7.6
-  - optuna ==3.5.0
-  - scipy ==1.11.4
-  - sqlalchemy ==2.0.23
-  - pynvml ==11.5.0
-  - bokeh ==3.3.2
+  - xgboost ==2.1.1
+  - optuna ==3.6.1
+  - optuna-integration ==3.6.0
+  - scipy ==1.14.0
+  - sqlalchemy ==2.0.31
+  - pynvml ==11.5.3
+  - bokeh ==3.5.1
   - gilknocker ==0.4.1
   - openssl >1.1.0g
 


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index b5870c1d4b..f23283b152 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -32,7 +32,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python_version: ["3.9"]
+        python_version: ["3.10"]
         pytest_args: [tests --ignore=tests/tpch]
         extra-env: [""]
         name_prefix: [tests]
@@ -40,7 +40,7 @@ jobs:
           # Run stability tests on the lowest and highest versions of Python only
           # These are temporarily redundant with the current global python_version
           # - pytest_args: tests/stability
-          #   python_version: "3.9"
+          #   python_version: "3.10"
           #   os: ubuntu-latest
           #   name_prefix: stability
           - pytest_args: tests/stability
@@ -49,20 +49,20 @@ jobs:
             name_prefix: stability
           # Run stability tests on Python Windows and MacOS (latest py39 only)
           - pytest_args: tests/stability
-            python_version: "3.9"
+            python_version: "3.10"
             os: windows-latest
             name_prefix: stability
           - pytest_args: tests/stability
-            python_version: "3.9"
+            python_version: "3.10"
             os: macos-latest
             name_prefix: stability
           - pytest_args: tests/workflows/test_snowflake.py
-            python_version: "3.9"
+            python_version: "3.10"
             os: ubuntu-latest
             name_prefix: snowflake
             extra-env: ci/environment-snowflake.yml
           - pytest_args: tests/tpch
-            python_version: "3.9"
+            python_version: "3.10"
             os: ubuntu-latest
             extra-env: ci/environment-tpch-nondask.yml
             name_prefix: tpch
diff --git a/tests/workflows/test_xgboost_optuna.py b/tests/workflows/test_xgboost_optuna.py
index 8d28fbe774..57382ecda2 100644
--- a/tests/workflows/test_xgboost_optuna.py
+++ b/tests/workflows/test_xgboost_optuna.py
@@ -7,8 +7,8 @@
 xgb = pytest.importorskip("xgboost")
 pytest.importorskip("sklearn")
 
-from optuna.integration.dask import DaskStorage  # noqa: E402
 from optuna.samplers import RandomSampler  # noqa: E402
+from optuna_integration import DaskStorage  # noqa: E402
 from sklearn.datasets import fetch_covtype  # noqa: E402
 from sklearn.model_selection import KFold, cross_val_score  # noqa: E402
 from sklearn.preprocessing import LabelEncoder  # noqa: E402