Repository: scverse/scanpy

Issue:
Dask update causing CI failures
<details>
<summary> Errors look like: </summary>

```
FAILED scanpy/tests/test_highly_variable_genes.py::test_runs - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_supports_batch - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_batch_matches_batch - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_inplace[numpy_ndarray-single] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_inplace[numpy_ndarray-batched] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_inplace[scipy_csr-single] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_inplace[scipy_csr-batched] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_inplace[scipy_csc-single] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_inplace[scipy_csc-batched] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_inplace[dask_array_dense-single] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_no_inplace[dask_array_dense-batched] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_keep_layer[cell_ranger-None] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_keep_layer[cell_ranger-10] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_highly_variable_genes.py::test_compare_to_upstream[seurat-hvg] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_pca.py::test_pca_sparse - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_pca.py::test_pca_reproducible[numpy_ndarray] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_pca.py::test_pca_reproducible[scipy_csr] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_pca.py::test_pca_reproducible[scipy_csc] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_pca.py::test_pca_reproducible[dask_array_dense] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_pca.py::test_pca_chunked - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_pca.py::test_pca_n_pcs - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_pca.py::test_pca_layer - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_plotting.py::test_scrublet_plots[scrublet] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_plotting.py::test_scrublet_plots[scrublet_no_threshold] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_plotting.py::test_scrublet_plots[scrublet_with_batches] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_preprocessing_distributed.py::test_write_zarr[dask] - ValueError: buffer source array is read-only
FAILED scanpy/tests/test_scrublet.py::test_scrublet[sparse] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_scrublet.py::test_scrublet[dense] - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_scrublet.py::test_scrublet_batched - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_scrublet.py::test_scrublet_data - ValueError: assignment destination is read-only
FAILED scanpy/tests/test_scrublet.py::test_scrublet_simulate_doublets - ValueError: assignment destination is read-only
ERROR scanpy/tests/test_scrublet.py::test_scrublet_params[expected_doublet_rate-0.1] - ValueError: assignment destination is read-only
ERROR scanpy/tests/test_scrublet.py::test_scrublet_params[synthetic_doublet_umi_subsampling-0.8] - ValueError: assignment destination is read-only
ERROR scanpy/tests/test_scrublet.py::test_scrublet_params[knn_dist_metric-manhattan] - ValueError: assignment destination is read-only
ERROR scanpy/tests/test_scrublet.py::test_scrublet_params[normalize_variance-False] - ValueError: assignment destination is read-only
ERROR scanpy/tests/test_scrublet.py::test_scrublet_params[log_transform-True] - ValueError: assignment destination is read-only

```

</details>


<details>
<summary> Test failure traceback </summary>

```pytb
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../mambaforge/envs/scanpy-dev/lib/python3.11/site-packages/legacy_api_wrap/__init__.py:80: in fn_compatible
    return fn(*args_all, **kw)
scanpy/preprocessing/_highly_variable_genes.py:651: in highly_variable_genes
    df = _highly_variable_genes_single_batch(
scanpy/preprocessing/_highly_variable_genes.py:288: in _highly_variable_genes_single_batch
    df["highly_variable"] = _subset_genes(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

adata = AnnData object with n_obs × n_vars = 700 × 765
    obs: 'bulk_labels', 'n_genes', 'percent_mito', 'n_counts', 'S_score...'pca', 'rank_genes_groups', 'log1p'
    obsm: 'X_pca', 'X_umap'
    varm: 'PCs'
    obsp: 'distances', 'connectivities'

    def _subset_genes(
        adata: AnnData,
        *,
        mean: NDArray[np.float64] | DaskArray,
        dispersion_norm: NDArray[np.float64] | DaskArray,
        cutoff: _Cutoffs | int,
    ) -> NDArray[np.bool_] | DaskArray:
        """Get boolean mask of genes with normalized dispersion in bounds."""
        if isinstance(cutoff, _Cutoffs):
>           dispersion_norm[np.isnan(dispersion_norm)] = 0  # similar to Seurat
E           ValueError: assignment destination is read-only

scanpy/preprocessing/_highly_variable_genes.py:365: ValueError
```

</details>

Dependencies are different, looks like a dask update and a pyarrow added dep. I suspect this has to do with the new dask-expr.

----

I can replicate locally by install the new dask, dask-expr, and pyarrow.

----

Importing dask.dataframe changes the settings for pandas somehow:

```python
In [1]: import pandas as pd

In [2]: pd.DataFrame({"a": [1,2,3, None]})["a"].to_numpy().flags
Out[2]: 
  C_CONTIGUOUS : True
  F_CONTIGUOUS : True
  OWNDATA : False
  WRITEABLE : True
  ALIGNED : True
  WRITEBACKIFCOPY : False

In [3]: import dask.dataframe as ddf

In [4]: pd.DataFrame({"a": [1,2,3, None]})["a"].to_numpy().flags
Out[4]: 
  C_CONTIGUOUS : True
  F_CONTIGUOUS : True
  OWNDATA : False
  WRITEABLE : False
  ALIGNED : True
  WRITEBACKIFCOPY : False
```


Discussion:
Reported upstream in https://github.com/dask/dask/issues/10996

Resolving patch:
diff --git a/scanpy/preprocessing/_highly_variable_genes.py b/scanpy/preprocessing/_highly_variable_genes.py
--- a/scanpy/preprocessing/_highly_variable_genes.py
+++ b/scanpy/preprocessing/_highly_variable_genes.py
@@ -362,7 +362,7 @@ def _subset_genes(
 ) -> NDArray[np.bool_] | DaskArray:
     """Get boolean mask of genes with normalized dispersion in bounds."""
     if isinstance(cutoff, _Cutoffs):
-        dispersion_norm[np.isnan(dispersion_norm)] = 0  # similar to Seurat
+        dispersion_norm = np.nan_to_num(dispersion_norm)  # similar to Seurat
         return cutoff.in_bounds(mean, dispersion_norm)
     n_top_genes = cutoff
     del cutoff
@@ -462,8 +462,7 @@ def _highly_variable_genes_batched(
         )
         df["highly_variable"] = np.arange(df.shape[0]) < cutoff
     else:
-        dispersion_norm = df["dispersions_norm"].to_numpy()
-        dispersion_norm[np.isnan(dispersion_norm)] = 0  # similar to Seurat
+        df["dispersions_norm"] = df["dispersions_norm"].fillna(0)  # similar to Seurat
         df["highly_variable"] = cutoff.in_bounds(df["means"], df["dispersions_norm"])
 
     return df


Test patch:
diff --git a/scanpy/testing/_pytest/__init__.py b/scanpy/testing/_pytest/__init__.py
--- a/scanpy/testing/_pytest/__init__.py
+++ b/scanpy/testing/_pytest/__init__.py
@@ -1,4 +1,5 @@
 """A private pytest plugin"""
+
 from __future__ import annotations
 
 import os
@@ -6,6 +7,7 @@
 import warnings
 from typing import TYPE_CHECKING
 
+import pandas as pd
 import pytest
 
 from ..._utils import _import_name
@@ -71,7 +73,8 @@ def _fix_dask_df_warning():
         import dask  # noqa: F401
     except ImportError:
         return
-    with warnings.catch_warnings():
+    # reset COW mode after this block: https://github.com/dask/dask/issues/10996
+    with warnings.catch_warnings(), pd.option_context("mode.copy_on_write", True):
         warnings.filterwarnings(
             "ignore",
             category=DeprecationWarning,