Repository: dask/dask
Pull request #10544 (2023-10-04 12:18:33)

Support for Python 3.12
- Blocked by https://github.com/dask/distributed/pull/8223

graingert: requires https://github.com/dask/distributed/pull/8223/files#diff-3b1a0d09b95e5e870b6962e81a2682ba3805afb0a134c642259b8162dd94e2acR220 to pass
crusaderky: @graingert there seems to be a genuine failure
crusaderky: @graingert also missing changes to pyproject.toml
graingert: `FAILED dask/array/tests/test_array_core.py::test_setitem_hardmask - AssertionError: {('array-ca2bfc119ac07031ee36eb791623951f',): 2}
assert not {('array-ca2bfc119ac07031ee36eb791623951f',): 2}` seems to fail every time on windows

Patch:
diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
index 6f5620dc7..062f3abf9 100644
--- a/.github/workflows/additional.yml
+++ b/.github/workflows/additional.yml
@@ -13,7 +13,7 @@ jobs:
     timeout-minutes: 90
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.1.1
 
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2.2.0
@@ -22,8 +22,8 @@ jobs:
           miniforge-version: latest
           use-mamba: true
           channel-priority: strict
-          python-version: "3.10"
-          environment-file: continuous_integration/environment-3.10.yaml
+          python-version: "3.11"
+          environment-file: continuous_integration/environment-3.11.yaml
           activate-environment: test-environment
           auto-activate-base: false
 
@@ -39,10 +39,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.1.1
 
       - name: Setup Conda
         uses: conda-incubator/setup-miniconda@v2.2.0
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
index 9fac40abd..da7b249e4 100644
--- a/.github/workflows/conda.yml
+++ b/.github/workflows/conda.yml
@@ -25,7 +25,7 @@ jobs:
     name: Build (and upload)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.0
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0
       - name: Set up Python
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index fa8e8a7c5..5e2a69e24 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -11,7 +11,7 @@ jobs:
     name: pre-commit hooks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.0
+      - uses: actions/checkout@v4.1.1
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
diff --git a/.github/workflows/update-gpuci.yml b/.github/workflows/update-gpuci.yml
index 686efabc1..6ce4eb4e7 100644
--- a/.github/workflows/update-gpuci.yml
+++ b/.github/workflows/update-gpuci.yml
@@ -11,7 +11,7 @@ jobs:
     if: github.repository == 'dask/dask'
 
     steps:
-      - uses: actions/checkout@v4.1.0
+      - uses: actions/checkout@v4.1.1
 
       - name: Parse current axis YAML
         id: rapids_current
diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml
index caadee1a5..e9c129670 100644
--- a/.github/workflows/upstream.yml
+++ b/.github/workflows/upstream.yml
@@ -20,7 +20,7 @@ jobs:
     outputs:
       test-upstream: ${{ steps.detect-trigger.outputs.trigger-found }}
     steps:
-      - uses: actions/checkout@v4.1.0
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 2
       - uses: xarray-contrib/ci-trigger@v1
@@ -47,7 +47,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.1.1
 
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2.2.0
@@ -56,8 +56,8 @@ jobs:
           miniforge-version: latest
           use-mamba: true
           channel-priority: strict
-          python-version: "3.10"
-          environment-file: continuous_integration/environment-3.10.yaml
+          python-version: "3.11"
+          environment-file: continuous_integration/environment-3.11.yaml
           activate-environment: test-environment
           auto-activate-base: false
 
diff --git a/continuous_integration/environment-3.12.yaml b/continuous_integration/environment-3.12.yaml
new file mode 100644
index 000000000..cae467f38
--- /dev/null
+++ b/continuous_integration/environment-3.12.yaml
@@ -0,0 +1,79 @@
+# This job includes coverage
+name: test-environment
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  # required dependencies
+  - python=3.12
+  - packaging
+  - pyyaml
+  - click
+  - cloudpickle
+  - partd
+  - fsspec
+  - importlib_metadata
+  - toolz
+  # test dependencies
+  - pre-commit
+  - pytest
+  - pytest-cov
+  - pytest-rerunfailures
+  - pytest-timeout
+  - pytest-xdist
+  - moto
+  # Optional dependencies
+  - mimesis
+  - numpy
+  - pandas
+  - flask
+  - fastparquet>=0.8.0
+  - h5py
+  - pytables
+  - zarr
+  # `tiledb-py=0.17.5` lead to strange seg faults in CI, However 0.18 is needed for 3.11
+  # https://github.com/dask/dask/pull/9569
+  # - tiledb-py # crashes on Python 3.11
+  # - pyspark
+  # - tiledb>=2.5.0 # crashes on Python 3.11
+  - xarray
+  - sqlalchemy>=1.4.16
+  - pyarrow>=13
+  - coverage
+  - jsonschema
+  # # other -- IO
+  - boto3
+  - botocore
+  - bokeh
+  - httpretty
+  - aiohttp
+  - s3fs
+  # Need a new `crick` release with support for `numpy=1.24+`
+  # https://github.com/dask/crick/issues/25
+  # - crick
+  - cytoolz
+  - distributed
+  - ipython
+  - ipycytoscape
+  # until https://github.com/jupyter-widgets/ipywidgets/issues/3731 is fixed
+  - ipywidgets<8.0.5
+  - ipykernel<6.22.0
+  - lz4
+  # - numba no support for 3.12
+  - psutil
+  - requests
+  - scikit-image
+  - scikit-learn
+  - scipy
+  - python-snappy
+  - sparse
+  - cachey
+  - python-graphviz
+  # Not available on macOS for Python 3.12 yet
+  # - python-cityhash
+  - python-xxhash
+  - mmh3
+  - jinja2
+  - pip
+  - pip:
+    - git+https://github.com/dask/distributed
diff --git a/continuous_integration/scripts/install.sh b/continuous_integration/scripts/install.sh
index 31e568325..84a284630 100644
--- a/continuous_integration/scripts/install.sh
+++ b/continuous_integration/scripts/install.sh
@@ -31,7 +31,7 @@ if [[ ${UPSTREAM_DEV} ]]; then
         git+https://github.com/dask/distributed \
         git+https://github.com/dask/fastparquet \
         git+https://github.com/zarr-developers/zarr-python \
-        git+https://github.com/PyTables/PyTables@ab565af10250922f2778204fd62b42ee12087d12  # numpy 2 support
+        git+https://github.com/PyTables/PyTables  # numpy 2 support
 
     # FIXME https://github.com/mamba-org/mamba/issues/412
     # mamba uninstall --force ...
@@ -54,7 +54,7 @@ if [[ ${UPSTREAM_DEV} ]]; then
 
     # FIXME https://github.com/mamba-org/mamba/issues/412
     # mamba uninstall --force ...
-    conda uninstall --force crick tiledb tiledb-py
+    # conda uninstall --force crick tiledb tiledb-py
 
 
 fi
diff --git a/dask/array/core.py b/dask/array/core.py
index e03b9cd9a..35a6a3bc9 100644
--- a/dask/array/core.py
+++ b/dask/array/core.py
@@ -3097,7 +3097,10 @@ def normalize_chunks(chunks, shape=None, limit=None, dtype=None, previous_chunks
     if shape is not None:
         chunks = tuple(c if c not in {None, -1} else s for c, s in zip(chunks, shape))
 
+    allints = None
     if chunks and shape is not None:
+        # allints: did we start with chunks as a simple tuple of ints?
+        allints = all(isinstance(c, int) for c in chunks)
         chunks = sum(
             (
                 blockdims_from_blockshape((s,), (c,))
@@ -3128,7 +3131,9 @@ def normalize_chunks(chunks, shape=None, limit=None, dtype=None, previous_chunks
                 "Chunks do not add up to shape. "
                 "Got chunks=%s, shape=%s" % (chunks, shape)
             )
-
+    if allints or isinstance(sum(sum(_) for _ in chunks), int):
+        # Fastpath for when we already know chunks contains only integers
+        return tuple(tuple(ch) for ch in chunks)
     return tuple(
         tuple(int(x) if not math.isnan(x) else np.nan for x in c) for c in chunks
     )
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
index 88ebea8f3..360cc592d 100644
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -849,9 +849,9 @@ Dask Name: {name}, {layers}"""
         Dask DataFrame Structure:
                          name     id        x        y
         npartitions=1
-        2021-01-01     object  int64  float64  float64
+        2021-01-01     string  int64  float64  float64
         2021-01-02        ...    ...      ...      ...
-        Dask Name: get-partition, 2 graph layers
+        Dask Name: get-partition, 3 graph layers
 
         See Also
         --------
diff --git a/dask/dataframe/io/parquet/core.py b/dask/dataframe/io/parquet/core.py
index 584279e73..d26a266d6 100644
--- a/dask/dataframe/io/parquet/core.py
+++ b/dask/dataframe/io/parquet/core.py
@@ -315,7 +315,7 @@ def read_parquet(
         set the default value of ``split_row_groups`` (using row-group metadata
         from a single file), and will be ignored if ``split_row_groups`` is not
         set to 'infer' or 'adaptive'. Default may be engine-dependant, but is
-        128 MiB for the 'pyarrow' and 'fastparquet' engines.
+        256 MiB for the 'pyarrow' and 'fastparquet' engines.
     aggregate_files : bool or str, default None
         WARNING: Passing a string argument to ``aggregate_files`` will result
         in experimental behavior. This behavior may change in the future.
diff --git a/dask/dataframe/io/parquet/utils.py b/dask/dataframe/io/parquet/utils.py
index 1344902af..5e714322d 100644
--- a/dask/dataframe/io/parquet/utils.py
+++ b/dask/dataframe/io/parquet/utils.py
@@ -174,7 +174,7 @@ class Engine:
 
     @classmethod
     def default_blocksize(cls):
-        return "128 MiB"
+        return "256 MiB"
 
     @classmethod
     def read_partition(
@@ -907,7 +907,7 @@ def _infer_split_row_groups(row_group_sizes, blocksize, aggregate_files=False):
     # Use blocksize to choose an appropriate split_row_groups value
     if row_group_sizes:
         blocksize = parse_bytes(blocksize)
-        if aggregate_files or np.sum(row_group_sizes) > blocksize:
+        if aggregate_files or np.sum(row_group_sizes) > 2 * blocksize:
             # If we are aggregating files, or the file is larger
             # than `blocksize`, set split_row_groups to "adaptive"
             return "adaptive"
diff --git a/dask/dataframe/io/sql.py b/dask/dataframe/io/sql.py
index b1f59a4e1..de4b33290 100644
--- a/dask/dataframe/io/sql.py
+++ b/dask/dataframe/io/sql.py
@@ -539,12 +539,13 @@ def to_sql(
     Dask Name: from_pandas, 2 tasks
 
     >>> from dask.utils import tmpfile
-    >>> from sqlalchemy import create_engine
+    >>> from sqlalchemy import create_engine, text
     >>> with tmpfile() as f:
     ...     db = 'sqlite:///%s' %f
     ...     ddf.to_sql('test', db)
     ...     engine = create_engine(db, echo=False)
-    ...     result = engine.execute("SELECT * FROM test").fetchall()
+    ...     with engine.connect() as conn:
+    ...         result = conn.execute(text("SELECT * FROM test")).fetchall()
     >>> result
     [(0, 0, '00'), (1, 1, '11'), (2, 2, '22'), (3, 3, '33')]
     """
diff --git a/dask/dataframe/multi.py b/dask/dataframe/multi.py
index ade477ecd..82d43f315 100644
--- a/dask/dataframe/multi.py
+++ b/dask/dataframe/multi.py
@@ -1255,7 +1255,7 @@ def concat(
     ...     dd.from_pandas(pd.Series(['a', 'b'], dtype='category'), 1),
     ...     dd.from_pandas(pd.Series(['a', 'c'], dtype='category'), 1),
     ... ], interleave_partitions=True).dtype
-    CategoricalDtype(categories=['a', 'b', 'c'], ordered=False)
+    CategoricalDtype(categories=['a', 'b', 'c'], ordered=False, categories_dtype=object)
     """
 
     if not isinstance(dfs, list):
diff --git a/dask/dataframe/reshape.py b/dask/dataframe/reshape.py
index cb3868a54..58e067570 100644
--- a/dask/dataframe/reshape.py
+++ b/dask/dataframe/reshape.py
@@ -95,16 +95,16 @@ def get_dummies(
     Dask DataFrame Structure:
                        a      b      c
     npartitions=2
-    0              uint8  uint8  uint8
+    0              bool  bool  bool
     2                ...    ...    ...
     3                ...    ...    ...
     Dask Name: get_dummies, 2 graph layers
     >>> dd.get_dummies(s).compute()  # doctest: +ELLIPSIS
-       a  b  c
-    0  1  0  0
-    1  0  1  0
-    2  0  0  1
-    3  1  0  0
+           a      b      c
+    0   True  False  False
+    1  False   True  False
+    2  False  False   True
+    3   True  False  False
 
     See Also
     --------
diff --git a/dask/sizeof.py b/dask/sizeof.py
index bbd80b3a1..5a98d9465 100644
--- a/dask/sizeof.py
+++ b/dask/sizeof.py
@@ -65,11 +65,17 @@ class SimpleSizeof:
 
     Examples
     --------
+    >>> def _get_gc_overhead():
+    ...     class _CustomObject:
+    ...         def __sizeof__(self):
+    ...             return 0
+    ...
+    ...     return sys.getsizeof(_CustomObject())
 
     >>> class TheAnswer(SimpleSizeof):
-    ...         def __sizeof__(self):
-    ...             # Sizeof always add overhead of an object for GC
-    ...             return 42 - sizeof(object())
+    ...     def __sizeof__(self):
+    ...         # Sizeof always add overhead of an object for GC
+    ...         return 42 - _get_gc_overhead()
 
     >>> sizeof(TheAnswer())
     42
diff --git a/docs/source/dataframe-parquet.rst b/docs/source/dataframe-parquet.rst
index 813050ba9..7d92d5029 100644
--- a/docs/source/dataframe-parquet.rst
+++ b/docs/source/dataframe-parquet.rst
@@ -110,7 +110,7 @@ Partition Size
 By default, Dask will use metadata from the first parquet file in the dataset
 to infer whether or not it is safe load each file individually as a partition
 in the Dask dataframe. If the uncompressed byte size of the parquet data
-exceeds ``blocksize`` (which is 128 MiB by default), then each partition will
+exceeds ``blocksize`` (which is 256 MiB by default), then each partition will
 correspond to a range of parquet row-groups instead of the entire file.
 
 For best performance, use files that can be individually mapped to good
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index b9085c74d..a680895ff 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -108,7 +108,7 @@ pip or conda_
 
 ``conda``::
 
-  conda env create -n dask-dev -f continuous_integration/environment-3.10.yaml
+  conda env create -n dask-dev -f continuous_integration/environment-3.11.yaml
   conda activate dask-dev
   python -m pip install --no-deps -e .
 
@@ -254,7 +254,7 @@ after the line.
 
 .. _numpydoc: https://numpydoc.readthedocs.io/en/latest/format.html#docstring-standard
 
-Docstrings are tested under Python 3.10 on GitHub Actions. You can test
+Docstrings are tested under Python 3.11 on GitHub Actions. You can test
 docstrings with pytest as follows::
 
    py.test dask --doctest-modules
diff --git a/pyproject.toml b/pyproject.toml
index a6bb32fa7..a576efaa8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering",
     "Topic :: System :: Distributed Computing",
 ]


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 81dbae73a..a05016600 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -24,13 +24,15 @@ jobs:
       fail-fast: false
       matrix:
         os: ["windows-latest", "ubuntu-latest", "macos-latest"]
-        environment: ["3.9", "3.10", "3.11"]
+        environment: ["3.9", "3.10", "3.11", "3.12"]
         extra: [null]
         exclude:
           - os: "macos-latest"
             environment: "3.9"
           - os: "macos-latest"
             environment: "3.10"
+          - os: "macos-latest"
+            environment: "3.11"
         include:
           # Minimum dependencies
           - os: "ubuntu-latest"
@@ -45,7 +47,7 @@ jobs:
             environment: "mindeps-optional"
           # Pyarrow strings turned on
           - os: "ubuntu-latest"
-            environment: "3.11"
+            environment: "3.12"
             extra: "pyarrow"
 
     env:
@@ -55,7 +57,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0  # Needed by codecov.io
       - name: Setup Java
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
index d426390f2..4c882bf07 100644
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -14,6 +14,7 @@ np = pytest.importorskip("numpy")
 import math
 import operator
 import os
+import sys
 import time
 import warnings
 from functools import reduce
@@ -21,11 +22,13 @@ from io import StringIO
 from operator import add, sub
 from threading import Lock
 
+from packaging.version import Version
 from tlz import concat, merge
 from tlz.curried import identity
 
 import dask
 import dask.array as da
+from dask._compatibility import PY_VERSION
 from dask.array.chunk import getitem
 from dask.array.core import (
     Array,
@@ -3910,6 +3913,10 @@ def test_setitem_1d():
         dx[index] = 1
 
 
+@pytest.mark.xfail(
+    sys.platform == "win32" and PY_VERSION >= Version("3.12.0"),
+    reason="https://github.com/dask/dask/issues/10604",
+)
 def test_setitem_hardmask():
     x = np.ma.array([1, 2, 3, 4], dtype=int)
     x.harden_mask()
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
index b5d57147d..ddd6b4ad8 100644
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -2443,7 +2443,7 @@ def test_repartition_noop(type_ctor):
         (
             "M",
             "MS",
-            (UserWarning, "'M' will be deprecated, please use 'ME' instead")
+            (FutureWarning, "'M' will be deprecated, please use 'ME' instead")
             if PANDAS_GE_220
             else None,
         ),
@@ -2452,7 +2452,7 @@ def test_repartition_noop(type_ctor):
         (
             "2M",
             "2MS",
-            (UserWarning, "'M' will be deprecated, please use 'ME' instead")
+            (FutureWarning, "'M' will be deprecated, please use 'ME' instead")
             if PANDAS_GE_220
             else None,
         ),
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
index cd32eec5c..38b3eed78 100644
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -5,9 +5,11 @@ import warnings
 import numpy as np
 import pandas as pd
 import pytest
+from packaging.version import Version
 from pandas.api.types import is_object_dtype
 
 import dask.dataframe as dd
+from dask._compatibility import PY_VERSION
 from dask.base import compute_as_if_collection
 from dask.dataframe._compat import (
     PANDAS_GE_140,
@@ -2135,7 +2137,7 @@ def test_concat5():
             False,
             True,
             marks=pytest.mark.xfail(
-                PANDAS_GE_220,
+                PANDAS_GE_220 or PY_VERSION >= Version("3.12.0"),
                 reason="fails on pandas dev: https://github.com/dask/dask/issues/10558",
                 raises=AssertionError,
                 strict=False,
@@ -2148,7 +2150,7 @@ def test_concat5():
             False,
             True,
             marks=pytest.mark.xfail(
-                PANDAS_GE_220,
+                PANDAS_GE_220 or PY_VERSION >= Version("3.12.0"),
                 reason="fails on pandas dev: https://github.com/dask/dask/issues/10558",
                 raises=AssertionError,
                 strict=False,
diff --git a/dask/tests/test_docs.py b/dask/tests/test_docs.py
index bf5e0c5ec..4869dca24 100644
--- a/dask/tests/test_docs.py
+++ b/dask/tests/test_docs.py
@@ -15,7 +15,7 @@ def test_development_guidelines_matches_ci():
     development_doc_file = root_dir / "docs" / "source" / "develop.rst"
     additional_ci_file = root_dir / ".github" / "workflows" / "additional.yml"
     upstream_ci_file = root_dir / ".github" / "workflows" / "upstream.yml"
-    latest_env = "environment-3.10.yaml"
+    latest_env = "environment-3.11.yaml"
 
     for filename in [development_doc_file, additional_ci_file, upstream_ci_file]:
         with open(filename, encoding="utf8") as f: