Repository: dask/dask

Issue:
Python 3.11 support 
Python 3.11 has been released 🎉 Historically it has taken some time for our dependencies to add support for new Python versions and for those changes to show up in releases. Opening this issue for tracking purposes 

cc @charlesbluca in case this is something you might be interested in 


Discussion:
Right now, the current blockers to solving a 3.11 environment are:

- `tiledb-py` constraint set in https://github.com/dask/dask/pull/9569
    - it suffices to set `tiledb-py!=0.17.5`, but not sure if we've tested CI with newer versions
- `numba` builds for 3.11 (migration being tracked in https://github.com/numba/numba/issues/8304)
- `numcodecs` builds for 3.11 (required by `zarr`, being tracked in https://github.com/zarr-developers/numcodecs/issues/377)
- `libprotobuf` pin set in https://github.com/dask/dask/pull/9465

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -83,6 +83,7 @@
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Scientific/Engineering",
         "Topic :: System :: Distributed Computing",
     ],


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -23,7 +23,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["windows-latest", "ubuntu-latest", "macos-latest"]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         exclude:
             - os: "macos-latest"
               python-version: "3.8"
diff --git a/dask/dataframe/io/tests/test_hdf.py b/dask/dataframe/io/tests/test_hdf.py
--- a/dask/dataframe/io/tests/test_hdf.py
+++ b/dask/dataframe/io/tests/test_hdf.py
@@ -5,9 +5,11 @@
 import numpy as np
 import pandas as pd
 import pytest
+from packaging.version import Version
 
 import dask
 import dask.dataframe as dd
+from dask.compatibility import _PY_VERSION
 from dask.dataframe._compat import tm
 from dask.dataframe.optimize import optimize_dataframe_getitem
 from dask.dataframe.utils import assert_eq
@@ -46,6 +48,10 @@ def test_to_hdf():
         tm.assert_frame_equal(df, out[:])
 
 
+@pytest.mark.skipif(
+    _PY_VERSION >= Version("3.11"),
+    reason="segfaults due to https://github.com/PyTables/PyTables/issues/977",
+)
 def test_to_hdf_multiple_nodes():
     pytest.importorskip("tables")
     df = pd.DataFrame(
@@ -388,6 +394,10 @@ def test_to_hdf_link_optimizations():
         assert dependency_depth(d.dask) == 2 + a.npartitions
 
 
+@pytest.mark.skipif(
+    _PY_VERSION >= Version("3.11"),
+    reason="segfaults due to https://github.com/PyTables/PyTables/issues/977",
+)
 @pytest.mark.slow
 def test_to_hdf_lock_delays():
     pytest.importorskip("tables")
@@ -478,6 +488,10 @@ def test_to_hdf_exceptions():
                 a.to_hdf(hdf, "/data_*_*")
 
 
+@pytest.mark.skipif(
+    _PY_VERSION >= Version("3.11"),
+    reason="segfaults due to https://github.com/PyTables/PyTables/issues/977",
+)
 @pytest.mark.parametrize("scheduler", ["sync", "threads", "processes"])
 @pytest.mark.parametrize("npartitions", [1, 4, 10])
 def test_to_hdf_schedulers(scheduler, npartitions):
@@ -679,6 +693,10 @@ def test_read_hdf_multiply_open():
             dd.read_hdf(fn, "/data", chunksize=2, mode="r")
 
 
+@pytest.mark.skipif(
+    _PY_VERSION >= Version("3.11"),
+    reason="segfaults due to https://github.com/PyTables/PyTables/issues/977",
+)
 def test_read_hdf_multiple():
     pytest.importorskip("tables")
     df = pd.DataFrame(
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -661,6 +661,7 @@ def write_partition(df, i):
         assert_eq(df, ddf2, check_index=False)
 
 
+@PYARROW_MARK
 @pytest.mark.xfail(
     not PANDAS_GT_130,
     reason=(
@@ -3005,6 +3006,7 @@ def test_chunksize_aggregate_files(tmpdir, write_engine, read_engine, aggregate_
     assert_eq(df1[["c", "d"]], df2[["c", "d"]], check_index=False)
 
 
+@PYARROW_MARK
 @pytest.mark.parametrize("metadata", [True, False])
 @pytest.mark.parametrize("chunksize", [None, 1024, 4096, "1MiB"])
 def test_chunksize(tmpdir, chunksize, engine, metadata):
@@ -3998,6 +4000,7 @@ def test_metadata_task_size(tmpdir, engine, write_metadata_file, metadata_task_s
     assert_eq(ddf2b, ddf2c)
 
 
+@PYARROW_MARK
 @pytest.mark.parametrize("partition_on", ("b", None))
 def test_extra_file(tmpdir, engine, partition_on):
     # Check that read_parquet can handle spark output