Repository: dask/dask
Pull request #10638 (2023-11-21 11:52:27)

Make meta check simpler in dispatch
Not really happy with this, but hasattr messes with memory usage on pandas object since it populates some internal stuff

We can get rid of the pandas checks if we are ok with 
```
s = pd.Series([1, 2, 3, 4], index=["a", "b", "c", "d"])
ds = dd.from_pandas(s, npartitions=2)

expected = s.memory_usage(deep=True)
result = ds.memory_usage(deep=True)
assert_eq(expected, result)
```

having small difference


Patch:
diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
index 062f3abf9..e2d4438f4 100644
--- a/.github/workflows/additional.yml
+++ b/.github/workflows/additional.yml
@@ -16,7 +16,7 @@ jobs:
         uses: actions/checkout@v4.1.1
 
       - name: Setup Conda Environment
-        uses: conda-incubator/setup-miniconda@v2.2.0
+        uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           miniforge-variant: Mambaforge
           miniforge-version: latest
@@ -45,7 +45,7 @@ jobs:
         uses: actions/checkout@v4.1.1
 
       - name: Setup Conda
-        uses: conda-incubator/setup-miniconda@v2.2.0
+        uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           miniforge-variant: Mambaforge
           miniforge-version: latest
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
index da7b249e4..e8430c774 100644
--- a/.github/workflows/conda.yml
+++ b/.github/workflows/conda.yml
@@ -29,7 +29,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: conda-incubator/setup-miniconda@v2.2.0
+        uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
diff --git a/.github/workflows/label-prs.yml b/.github/workflows/label-prs.yml
index ec39e68a3..b27f957d9 100644
--- a/.github/workflows/label-prs.yml
+++ b/.github/workflows/label-prs.yml
@@ -6,7 +6,7 @@ jobs:
   label:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@main
+    - uses: actions/labeler@v4.3.0
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"
         sync-labels: false
diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml
index e9c129670..fd53cfeb8 100644
--- a/.github/workflows/upstream.yml
+++ b/.github/workflows/upstream.yml
@@ -50,7 +50,7 @@ jobs:
         uses: actions/checkout@v4.1.1
 
       - name: Setup Conda Environment
-        uses: conda-incubator/setup-miniconda@v2.2.0
+        uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           miniforge-variant: Mambaforge
           miniforge-version: latest
@@ -81,7 +81,7 @@ jobs:
           && github.event_name != 'pull_request'
           && github.repository == 'dask/dask'
           && steps.run_tests.outcome == 'failure'
-        uses: xarray-contrib/issue-from-pytest-log@v1.2.6
+        uses: xarray-contrib/issue-from-pytest-log@v1.2.7
         with:
           log-path: output-log.jsonl
           issue-title: ⚠️ Upstream CI failed ⚠️
diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml
index dd7525507..32c44da26 100644
--- a/continuous_integration/gpuci/axis.yaml
+++ b/continuous_integration/gpuci/axis.yaml
@@ -3,12 +3,12 @@ PYTHON_VER:
 - "3.10"
 
 CUDA_VER:
-- "11.5.2"
+- "11.8.0"
 
 LINUX_VER:
 - ubuntu20.04
 
 RAPIDS_VER:
-- "23.12"
+- "24.02"
 
 excludes:
diff --git a/dask/base.py b/dask/base.py
index 044beb7ac..c015c4b70 100644
--- a/dask/base.py
+++ b/dask/base.py
@@ -1194,6 +1194,14 @@ def register_pandas():
     def normalize_period_dtype(dtype):
         return normalize_token(dtype.name)
 
+    @normalize_token.register(type(pd.NA))
+    def normalize_na(na):
+        return pd.NA
+
+    @normalize_token.register(pd.offsets.BaseOffset)
+    def normalize_offset(offset):
+        return [offset.n, offset.name]
+
 
 @normalize_token.register_lazy("numpy")
 def register_numpy():
diff --git a/dask/dataframe/dispatch.py b/dask/dataframe/dispatch.py
index 9c70302de..5eca20859 100644
--- a/dask/dataframe/dispatch.py
+++ b/dask/dataframe/dispatch.py
@@ -8,8 +8,7 @@ from __future__ import annotations
 
 import pandas as pd
 
-import dask.array as da
-import dask.dataframe as dd
+from dask import is_dask_collection
 from dask.utils import Dispatch
 
 make_meta_dispatch = Dispatch("make_meta_dispatch")
@@ -119,17 +118,9 @@ def make_meta(x, index=None, parent_meta=None):
     A valid meta-data
     """
 
-    if isinstance(
-        x,
-        (
-            dd._Frame,
-            dd.core.Scalar,
-            dd.groupby._GroupBy,
-            dd.accessor.Accessor,
-            da.Array,
-        ),
-    ):
-        return x._meta
+    if not isinstance(x, (pd.Series, pd.DataFrame, pd.Index)) and hasattr(x, "_meta"):
+        if is_dask_collection(x):
+            return x._meta
 
     try:
         return make_meta_dispatch(x, index=index)
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 438fa81cb..4a104fa6d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,67 @@
 Changelog
 =========
 
+.. _v2023.12.0:
+
+2023.12.0
+---------
+
+Released on December 1, 2023
+
+Highlights
+^^^^^^^^^^
+
+PipInstall restart and environment variables
+""""""""""""""""""""""""""""""""""""""""""""
+
+The ``distributed.PipInstall`` plugin now has more robust restart logic and also supports
+`environment variables <https://pip.pypa.io/en/stable/reference/requirements-file-format/#using-environment-variables>`_.
+
+See :pr-distributed:`8374`, :pr-distributed:`8357`, and :pr-distributed:`8343` from `Hendrik Makait`_ for details.
+
+
+Bokeh 3.3.0 compatibility
+"""""""""""""""""""""""""
+This release contains compatibility updates for using ``bokeh>=3.3.0`` with proxied Dask dashboards.
+Previously the contents of dashboard plots wouldn't be displayed.
+
+See :pr-distributed:`8347` and :pr-distributed:`8381` from `Jacob Tomlinson`_ for details.
+
+  
+.. dropdown:: Additional changes
+
+  - Add ``network`` marker to ``test_pyarrow_filesystem_option_real_data`` (:pr:`10653`) `Richard (Rick) Zamora`_
+  - Bump GPU CI to CUDA 11.8 (:pr:`10656`) `Charles Blackmon-Luca`_
+  - Tokenize ``pandas`` offsets deterministically (:pr:`10643`) `Patrick Hoefler`_
+  - Add tokenize ``pd.NA`` functionality (:pr:`10640`) `Patrick Hoefler`_
+  - Update gpuCI ``RAPIDS_VER`` to ``24.02`` (:pr:`10636`)
+  - Fix precision handling in ``array.linalg.norm`` (:pr:`10556`) `joanrue`_
+  - Add ``axis`` argument to ``DataFrame.clip`` and ``Series.clip`` (:pr:`10616`) `Richard (Rick) Zamora`_
+  - Update changelog entry for in-memory rechunking (:pr:`10630`) `Florian Jetter`_
+  - Fix flaky ``test_resources_reset_after_cancelled_task`` (:pr-distributed:`8373`) `crusaderky`_
+  - Bump GPU CI to CUDA 11.8 (:pr-distributed:`8376`) `Charles Blackmon-Luca`_
+  - Bump ``conda-incubator/setup-miniconda`` from 2.2.0 to 3.0.0 (:pr-distributed:`8372`)
+  - Add debug logs to P2P scheduler plugin (:pr-distributed:`8358`) `Hendrik Makait`_
+  - ``O(1)`` access for ``/info/task/`` endpoint (:pr-distributed:`8363`) `crusaderky`_
+  - Remove stringification from shuffle annotations (:pr-distributed:`8362`) `crusaderky`_
+  - Don't cast ``int`` metrics to ``float`` (:pr-distributed:`8361`) `crusaderky`_
+  - Drop asyncio TCP backend (:pr-distributed:`8355`) `Florian Jetter`_
+  - Add offload support to ``context_meter.add_callback`` (:pr-distributed:`8360`) `crusaderky`_
+  - Test that ``sync()`` propagates contextvars (:pr-distributed:`8354`) `crusaderky`_
+  - ``captured_context_meter`` (:pr-distributed:`8352`) `crusaderky`_
+  - ``context_meter.clear_callbacks`` (:pr-distributed:`8353`) `crusaderky`_
+  - Use ``@log_errors`` decorator (:pr-distributed:`8351`) `crusaderky`_
+  - Fix ``test_statistical_profiling_cycle`` (:pr-distributed:`8356`) `Florian Jetter`_
+  - Shuffle: don't parse dask.config at every RPC (:pr-distributed:`8350`) `crusaderky`_
+  - Replace ``Client.register_plugin`` s ``idempotent`` argument with ``.idempotent`` attribute on plugins (:pr-distributed:`8342`) `Hendrik Makait`_
+  - Fix test report generation (:pr-distributed:`8346`) `Hendrik Makait`_
+  - Install ``pyarrow-hotfix`` on ``mindeps-pandas`` CI (:pr-distributed:`8344`) `Hendrik Makait`_
+  - Reduce memory usage of scheduler process - optimize ``scheduler.py::TaskState`` class (:pr-distributed:`8331`) `Miles`_
+  - Bump ``pre-commit`` linters (:pr-distributed:`8340`) `crusaderky`_
+  - Update cuDF test with explicit ``dtype=object`` (:pr-distributed:`8339`) `Peter Andreas Entschev`_
+  - Fix ``Cluster`` / ``SpecCluster`` calls to async close methods (:pr-distributed:`8327`) `Peter Andreas Entschev`_
+
+
 .. _v2023.11.0:
 
 2023.11.0
@@ -7282,3 +7343,4 @@ Other
 .. _`Michael Leslie`: https://github.com/michaeldleslie
 .. _`Samantha Hughes`: https://github.com/shughes-uk
 .. _`Mario Šaško`: https://github.com/mariosasko
+.. _`joanrue`: https://github.com/joanrue
diff --git a/docs/source/index.rst b/docs/source/index.rst
index a31b63e50..23f17dc1b 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -136,7 +136,7 @@ messy situations in everyday problems.
 
    Install Dask <install.rst>
    10-minutes-to-dask.rst
-   presentations.rst
+   deploying.rst
    Best Practices <best-practices.rst>
    faq.rst
 
@@ -150,7 +150,6 @@ messy situations in everyday problems.
    DataFrame <dataframe.rst>
    Delayed <delayed.rst>
    futures.rst
-   deploying.rst
 
 .. toctree::
    :maxdepth: 1
@@ -174,6 +173,7 @@ messy situations in everyday problems.
    changelog.rst
    configuration.rst
    how-to/index.rst
+   presentations.rst
    maintainers.rst
 
 .. _`Anaconda Inc`: https://www.anaconda.com
diff --git a/docs/source/presentations.rst b/docs/source/presentations.rst
index 487a736f1..2f1099f8b 100644
--- a/docs/source/presentations.rst
+++ b/docs/source/presentations.rst
@@ -13,127 +13,13 @@ Talks & Tutorials
 
 Dask Tutorial
 -------------
+
 `Dask Tutorial <https://tutorial.dask.org>`__ provides an overview of Dask and is typically delivered in 3 hours.
 See `Parallel and Distributed Computing in Python with Dask <https://www.youtube.com/watch?v=EybGGLbLipI>`__ for the
 latest Dask Tutorial recording from SciPy 2020.
 
-Dask Slides
------------
-`Dask Slides <https://dask.org/slides>`__ provide a quick overview of the motivation for Dask.
-
 Dask YouTube channel
 --------------------
 You can find lots of videos about Dask on the `Dask YouTube channel <https://www.youtube.com/c/dask-dev>`__
 
 .. contents:: :local:
-
-Presentations
--------------
-
-* Dask Summit 2021
-
-  * `Keynotes <https://www.youtube.com/playlist?list=PLJ0vO2F_f6OBymP5LtgOC6W4pxd9Mw3cE>`__ 
-  * `Workshops and Tutorials <https://www.youtube.com/playlist?list=PLJ0vO2F_f6OBD1_iNeT1f7cpRoYwAuMPy>`__
-  * `Talks <https://www.youtube.com/playlist?list=PLJ0vO2F_f6OBcisTDubrdEsQAhigkayjE>`__
-
-* PyCon US 2021
-
-  * `Tutorial: Hacking Dask: Diving into Dask's Internals <https://www.youtube.com/watch?v=LQrgDhN-XOo>`__  (`materials <https://github.com/jrbourbeau/hacking-dask>`__)
-  * `Dask-SQL: Empowering Pythonistas for Scalable End-to-End Data Engineering <https://www.youtube.com/watch?v=z7xKikaScxg>`__
-
-
-* BlazingSQL Webinars, May 2021
-
-  * `Intro to distributed computing on GPUs with Dask in Python <https://www.youtube.com/watch?v=py1YPs6s6so>`__ (`materials <https://gist.github.com/jacobtomlinson/6f16abb716f50f81a6687bd67efd2f61>`__)
-
-* PyData DC, August 2021
-
-  * `Inside Dask <https://www.youtube.com/watch?v=X95WO41abXo>`__ (`materials <https://github.com/jsignell/inside-dask>`__)
-
-* PyCon US 2020
-
-  * `Deploying Python at Scale with Dask <https://www.youtube.com/watch?v=deX0GlW4uew>`__
-
-* PyCon Australia 2020
-
-  * `dask-image: distributed image processing for large data <https://www.youtube.com/watch?v=MpjgzNeISeI>`__
-
-* PyCon Korea 2019, August 2019
-
-  * `Adapting from Spark to Dask: what to expect (18 minutes)
-    <https://www.youtube.com/watch?v=tx7qTHSlHKw>`__
-
-* SciPy 2019, July 2019
-
-  * `Refactoring the SciPy Ecosystem for Heterogeneous Computing (29 minutes)
-    <https://www.youtube.com/watch?v=Q0DsdiY-jiw>`__
-  * `Renewable Power Forecast Generation with Dask & Visualization with Bokeh (31 minutes)
-    <https://www.youtube.com/watch?v=tYGcicSruck>`__
-  * `Efficient Atmospheric Analogue Selection with Xarray and Dask (18 minutes)
-    <https://www.youtube.com/watch?v=gdHiGsGUh3o>`__
-  * `Better and Faster Hyper Parameter Optimization with Dask (27 minutes)
-    <https://www.youtube.com/watch?v=x67K9FiPFBQ>`__
-  * `Dask image:A Library for Distributed Image Processing (22 minutes)
-    <https://www.youtube.com/watch?v=XGUS174vvLs>`__
-
-* EuroPython 2019, July 2019
-
-  * `Distributed Multi-GPU Computing with Dask, CuPy and RAPIDS (29 minutes)
-    <https://www.youtube.com/watch?v=en2zdTT-Vwk>`__
-
-* SciPy 2018, July 2018
-
-  * `Scalable Machine Learning with Dask (30 minutes)
-    <https://www.youtube.com/watch?v=ccfsbuqsjgI>`__
-
-* PyCon 2018, May 2018
-
-  * `Democratizing Distributed Computing with Dask and JupyterHub (32 minutes)
-    <https://www.youtube.com/watch?v=Iq72dt1gO9c>`__
-
-* AMS & ESIP, January 2018
-
-  * `Pangeo quick demo: Dask, XArray, Zarr on the cloud with JupyterHub (3 minutes)
-    <https://www.youtube.com/watch?v=rSOJKbfNBNk>`__
-  * `Pangeo talk: An open-source big data science platform with Dask, XArray, Zarr on the cloud with JupyterHub (43 minutes)
-    <https://www.youtube.com/watch?v=mDrjGxaXQT4>`__
-
-* PYCON.DE 2017, November 2017
-
-  * `Dask: Parallelism in Python (1 hour, 2 minutes)
-    <https://www.youtube.com/watch?v=rZlshXJydgQ>`__
-
-* PYCON 2017, May 2017
-
-  * `Dask: A Pythonic Distributed Data Science Framework (46 minutes)
-    <https://www.youtube.com/watch?v=RA_2qdipVng>`__
-
-* PLOTCON 2016, December 2016
-
-  * `Visualizing Distributed Computations with Dask and Bokeh (33 minutes)
-    <https://www.youtube.com/watch?v=FTJwDeXkggU>`__
-
-* PyData DC, October 2016
-
-  * `Using Dask for Parallel Computing in Python (44 minutes)
-    <https://www.youtube.com/watch?v=s4ChP7tc3tA>`__
-
-* SciPy 2016, July 2016
-
-  * `Dask Parallel and Distributed Computing (28 minutes)
-    <https://www.youtube.com/watch?v=PAGjm4BMKlk>`__
-
-* PyData NYC, December 2015
-
-  * `Dask Parallelizing NumPy and Pandas through Task Scheduling (33 minutes)
-    <https://www.youtube.com/watch?v=mHd8AI8GQhQ>`__
-
-* PyData Seattle, August 2015
-
-  * `Dask: out of core arrays with task scheduling (1 hour, 50 minutes)
-    <https://www.youtube.com/watch?v=ieW3G7ZzRZ0>`__
-
-* SciPy 2015, July 2015
-
-  * `Dask Out of core NumPy:Pandas through Task Scheduling (16 minutes)
-    <https://www.youtube.com/watch?v=1kkFZ4P-XHg>`__
diff --git a/pyproject.toml b/pyproject.toml
index a91256d40..0ef22fb6d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,7 @@ dataframe = [
     "dask[array]",
     "pandas >= 1.3",
 ]
-distributed = ["distributed == 2023.11.0"]
+distributed = ["distributed == 2023.12.0"]
 diagnostics = [
     "bokeh >= 2.4.2",
     "jinja2 >= 2.10.3",


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a05016600..a0cb9daa4 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -61,13 +61,13 @@ jobs:
         with:
           fetch-depth: 0  # Needed by codecov.io
       - name: Setup Java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           distribution: "zulu"
           java-version: "11"
       - name: Setup Conda Environment
-        uses: conda-incubator/setup-miniconda@v2.2.0
+        uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           miniforge-variant: Mambaforge
           miniforge-version: latest
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
index 518145cc6..0d4dd39d7 100644
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -4741,6 +4741,7 @@ def test_pyarrow_filesystem_option(tmp_path, fs):
 
 
 @PYARROW_MARK
+@pytest.mark.network
 @pytest.mark.slow
 def test_pyarrow_filesystem_option_real_data():
     # See: https://github.com/dask/dask/pull/10590
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
index a459e11e8..769933333 100644
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import contextlib
 import decimal
-import platform
 import warnings
 import weakref
 import xml.etree.ElementTree
@@ -4167,10 +4166,7 @@ def test_categorize_info():
     ddf.info(buf=buf, verbose=True)
 
     string_dtype = "object" if get_string_dtype() is object else "string"
-    if platform.architecture()[0] == "32bit":
-        memory_usage = "312.0"
-    else:
-        memory_usage = "463.0" if pyarrow_strings_enabled() else "496.0"
+    memory_usage = float(ddf.memory_usage().sum().compute())
 
     if pyarrow_strings_enabled():
         dtypes = f"category(1), int64(1), {string_dtype}(1)"
diff --git a/dask/tests/test_base.py b/dask/tests/test_base.py
index 776c360ad..c5217f697 100644
--- a/dask/tests/test_base.py
+++ b/dask/tests/test_base.py
@@ -354,6 +354,22 @@ def test_tokenize_pandas_extension_array():
         assert tokenize(arr) == tokenize(arr)
 
 
+@pytest.mark.skipif("not pd")
+def test_tokenize_na():
+    assert tokenize(pd.NA) == tokenize(pd.NA)
+
+
+@pytest.mark.skipif("not pd")
+def test_tokenize_offset():
+    for offset in [
+        pd.offsets.Second(1),
+        pd.offsets.MonthBegin(2),
+        pd.offsets.Day(1),
+        pd.offsets.BQuarterEnd(2),
+    ]:
+        assert tokenize(offset) == tokenize(offset)
+
+
 @pytest.mark.skipif("not pd")
 def test_tokenize_pandas_index():
     idx = pd.Index(["a", "b"])