Repository: pydata/xarray

Issue:
assert_identical fails to generate diff when an array is an attribute value
### What happened?

I'm writing a bunch of tests for one of my code bases that checks that the output of some operation done on a dataset has the expected results. As such, I'm using xarray.testing. assert_identical in these tests. I discovered that the assertion would fail occasionally with a ValueError rather than the expected AssertionError. Poking at different combinations of inputs, it appears to only fail when comparing a Dataset with non identical DataArrays that both contain an attribute that isn't comparable with `==`

### What did you expect to happen?

An AssertionError to be raised with the appropriate diff.

### Minimal Complete Verifiable Example

```Python
import xarray as xr
import numpy as np

ds1 = xr.Dataset({"t1": xr.DataArray([1], attrs={"test": np.array([0,1,2,3], dtype="byte")})})
ds2 = xr.Dataset({"t1": xr.DataArray([2], attrs={"test": np.array([0,1,2,3], dtype="byte")})})
xr.testing.assert_identical(ds1, ds2)
```


### MVCE confirmation

- [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray.
- [X] Complete example — the example is self-contained, including all data and the text of any traceback.
- [X] Verifiable example — the example copy & pastes into an IPython prompt or [Binder notebook](https://mybinder.org/v2/gh/pydata/xarray/main?urlpath=lab/tree/doc/examples/blank_template.ipynb), returning the result.
- [X] New issue — a search of GitHub Issues suggests this is not a duplicate.
- [X] Recent environment — the issue occurs with the latest version of xarray and its dependencies.

### Relevant log output

```Python
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[1], line 6
      4 ds1 = xr.Dataset({"t1": xr.DataArray([1], attrs={"test": np.array([0,1,2,3], dtype="byte")})})
      5 ds2 = xr.Dataset({"t1": xr.DataArray([2], attrs={"test": np.array([0,1,2,3], dtype="byte")})})
----> 6 xr.testing.assert_identical(ds1, ds2)

    [... skipping hidden 2 frame]

File ~/.dotfiles/pyenv/versions/3.12.3/envs/jupyter/lib/python3.12/site-packages/xarray/core/formatting.py:974, in diff_dataset_repr(a, b, compat)
    971 summary.append(diff_dim_summary(a, b))
    972 summary.append(diff_coords_repr(a.coords, b.coords, compat, col_width=col_width))
    973 summary.append(
--> 974     diff_data_vars_repr(a.data_vars, b.data_vars, compat, col_width=col_width)
    975 )
    977 if compat == "identical":
    978     summary.append(diff_attrs_repr(a.attrs, b.attrs, compat))

File ~/.dotfiles/pyenv/versions/3.12.3/envs/jupyter/lib/python3.12/site-packages/xarray/core/formatting.py:824, in _diff_mapping_repr(a_mapping, b_mapping, compat, title, summarizer, col_width, a_indexes, b_indexes)
    820 b_attrs = b_mapping[k].attrs
    822 attrs_to_print = set(a_attrs) ^ set(b_attrs)
    823 attrs_to_print.update(
--> 824     {k for k in set(a_attrs) & set(b_attrs) if a_attrs[k] != b_attrs[k]}
    825 )
    826 for m in (a_mapping, b_mapping):
    827     attr_s = "\n".join(
    828         "    " + summarize_attr(ak, av)
    829         for ak, av in m[k].attrs.items()
    830         if ak in attrs_to_print
    831     )

ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
```


### Anything else we need to know?

Comparing the underlying DataArray objects works as expected:

```python
import xarray as xr
import numpy as np

ds1 = xr.Dataset({"t1": xr.DataArray([1], attrs={"test": np.array([0,1,2,3], dtype="byte")})})
ds2 = xr.Dataset({"t1": xr.DataArray([2], attrs={"test": np.array([0,1,2,3], dtype="byte")})})
xr.testing.assert_identical(ds1.t1, ds2.t1)
```
<details>
<summary>Traceback</summary>

```python
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
Cell In[3], line 6
      4 ds1 = xr.Dataset({"t1": xr.DataArray([1], attrs={"test": np.array([0,1,2,3], dtype="byte")})})
      5 ds2 = xr.Dataset({"t1": xr.DataArray([2], attrs={"test": np.array([0,1,2,3], dtype="byte")})})
----> 6 xr.testing.assert_identical(ds1.t1, ds2.t1)

    [... skipping hidden 1 frame]

File [~/.dotfiles/pyenv/versions/3.12.3/envs/jupyter/lib/python3.12/site-packages/xarray/testing/assertions.py:215](http://localhost:8888/~/.dotfiles/pyenv/versions/3.12.3/envs/jupyter/lib/python3.12/site-packages/xarray/testing/assertions.py#line=214), in assert_identical(a, b, from_root)
    213 elif isinstance(a, DataArray):
    214     assert a.name == b.name
--> 215     assert a.identical(b), formatting.diff_array_repr(a, b, "identical")
    216 elif isinstance(a, (Dataset, Variable)):
    217     assert a.identical(b), formatting.diff_dataset_repr(a, b, "identical")

AssertionError: Left and right DataArray objects are not identical

Differing values:
L
    array([1])
R
    array([2])
```
</details>

This potentially looks related to #3711

### Environment

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.12.3 (main, May 23 2024, 16:39:17) [Clang 15.0.0 (clang-1500.3.9.4)]
python-bits: 64
OS: Darwin
OS-release: 23.5.0
machine: arm64
processor: arm
byteorder: little
LC_ALL: en_US.UTF-8
LANG: en_US.UTF-8
LOCALE: ('en_US', 'UTF-8')
libhdf5: 1.12.2
libnetcdf: 4.9.3-development

xarray: 2024.6.0
pandas: 2.2.2
numpy: 1.26.4
scipy: 1.13.1
netCDF4: 1.6.5
pydap: None
h5netcdf: None
h5py: None
zarr: None
cftime: 1.6.3
nc_time_axis: None
iris: None
bottleneck: None
dask: None
distributed: None
matplotlib: 3.9.0
cartopy: None
seaborn: None
numbagg: None
fsspec: None
cupy: None
pint: None
sparse: None
flox: None
numpy_groupies: None
setuptools: None
pip: 24.0
conda: None
pytest: None
mypy: None
IPython: 8.24.0
sphinx: None

</details>


Discussion:
dcherian: Oops, the property test did find something

```
first = {'': array([False, False])}, second = {'': array([False, False])}

    def equivalent(first: T, second: T) -> bool:
        """Compare two objects for equivalence (identity or equality), using
        array_equiv if either object is an ndarray. If both objects are lists,
        equivalent is sequentially called on all the elements.
        """
        # TODO: refactor to avoid circular import
        from xarray.core import duck_array_ops

        if first is second:
            return True
        if isinstance(first, np.ndarray) or isinstance(second, np.ndarray):
            return duck_array_ops.array_equiv(first, second)
        if isinstance(first, list) or isinstance(second, list):
            return list_equiv(first, second)
>       return (first == second) or (pd.isnull(first) and pd.isnull(second))
E       ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
```
keewis: true. To fix that, we'd have to support recursive attrs, which feels a bit much. If anything, I'd raise a `ValueError` stating that `assert_*` don't support those.

Patch:
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
index 0174e166..f3ab5d46 100644
--- a/doc/whats-new.rst
+++ b/doc/whats-new.rst
@@ -39,6 +39,8 @@ Bug fixes
 ~~~~~~~~~
 - Make :py:func:`testing.assert_allclose` work with numpy 2.0 (:issue:`9165`, :pull:`9166`).
   By `Pontus Lurcock <https://github.com/pont-us>`_.
+- Allow diffing objects with array attributes on variables (:issue:`9153`, :pull:`9169`).
+  By `Justus Magin <https://github.com/keewis>`_.
 - Promote floating-point numeric datetimes before decoding (:issue:`9179`, :pull:`9182`).
   By `Justus Magin <https://github.com/keewis>`_.
 
diff --git a/xarray/core/formatting.py b/xarray/core/formatting.py
index c15df34b..5c4a3015 100644
--- a/xarray/core/formatting.py
+++ b/xarray/core/formatting.py
@@ -765,6 +765,12 @@ def _diff_mapping_repr(
     a_indexes=None,
     b_indexes=None,
 ):
+    def compare_attr(a, b):
+        if is_duck_array(a) or is_duck_array(b):
+            return array_equiv(a, b)
+        else:
+            return a == b
+
     def extra_items_repr(extra_keys, mapping, ab_side, kwargs):
         extra_repr = [
             summarizer(k, mapping[k], col_width, **kwargs[k]) for k in extra_keys
@@ -801,11 +807,7 @@ def _diff_mapping_repr(
             is_variable = True
         except AttributeError:
             # compare attribute value
-            if is_duck_array(a_mapping[k]) or is_duck_array(b_mapping[k]):
-                compatible = array_equiv(a_mapping[k], b_mapping[k])
-            else:
-                compatible = a_mapping[k] == b_mapping[k]
-
+            compatible = compare_attr(a_mapping[k], b_mapping[k])
             is_variable = False
 
         if not compatible:
@@ -821,7 +823,11 @@ def _diff_mapping_repr(
 
                 attrs_to_print = set(a_attrs) ^ set(b_attrs)
                 attrs_to_print.update(
-                    {k for k in set(a_attrs) & set(b_attrs) if a_attrs[k] != b_attrs[k]}
+                    {
+                        k
+                        for k in set(a_attrs) & set(b_attrs)
+                        if not compare_attr(a_attrs[k], b_attrs[k])
+                    }
                 )
                 for m in (a_mapping, b_mapping):
                     attr_s = "\n".join(


Test patch:
diff --git a/properties/test_properties.py b/properties/test_properties.py
new file mode 100644
index 00000000..fc0a1955
--- /dev/null
+++ b/properties/test_properties.py
@@ -0,0 +1,17 @@
+import pytest
+
+pytest.importorskip("hypothesis")
+
+from hypothesis import given
+
+import xarray as xr
+import xarray.testing.strategies as xrst
+
+
+@given(attrs=xrst.simple_attrs)
+def test_assert_identical(attrs):
+    v = xr.Variable(dims=(), data=0, attrs=attrs)
+    xr.testing.assert_identical(v, v.copy(deep=True))
+
+    ds = xr.Dataset(attrs=attrs)
+    xr.testing.assert_identical(ds, ds.copy(deep=True))
diff --git a/xarray/testing/strategies.py b/xarray/testing/strategies.py
index 449d0c79..085b70e5 100644
--- a/xarray/testing/strategies.py
+++ b/xarray/testing/strategies.py
@@ -192,10 +192,14 @@ _small_arrays = npst.arrays(
         max_side=2,
         max_dims=2,
     ),
-    dtype=npst.scalar_dtypes(),
+    dtype=npst.scalar_dtypes()
+    | npst.byte_string_dtypes()
+    | npst.unicode_string_dtypes(),
 )
 _attr_values = st.none() | st.booleans() | _readable_strings | _small_arrays
 
+simple_attrs = st.dictionaries(_attr_keys, _attr_values)
+
 
 def attrs() -> st.SearchStrategy[Mapping[Hashable, Any]]:
     """
diff --git a/xarray/tests/test_formatting.py b/xarray/tests/test_formatting.py
index d7a46eea..6c49ab45 100644
--- a/xarray/tests/test_formatting.py
+++ b/xarray/tests/test_formatting.py
@@ -399,6 +399,36 @@ class TestFormatting:
         actual = formatting.diff_attrs_repr(attrs_a, attrs_c, "equals")
         assert expected == actual
 
+    def test__diff_mapping_repr_array_attrs_on_variables(self) -> None:
+        a = {
+            "a": xr.DataArray(
+                dims="x",
+                data=np.array([1], dtype="int16"),
+                attrs={"b": np.array([1, 2], dtype="int8")},
+            )
+        }
+        b = {
+            "a": xr.DataArray(
+                dims="x",
+                data=np.array([1], dtype="int16"),
+                attrs={"b": np.array([2, 3], dtype="int8")},
+            )
+        }
+        actual = formatting.diff_data_vars_repr(a, b, compat="identical", col_width=8)
+        expected = dedent(
+            """\
+            Differing data variables:
+            L   a   (x) int16 2B 1
+                Differing variable attributes:
+                    b: [1 2]
+            R   a   (x) int16 2B 1
+                Differing variable attributes:
+                    b: [2 3]
+            """.rstrip()
+        )
+
+        assert actual == expected
+
     def test_diff_dataset_repr(self) -> None:
         ds_a = xr.Dataset(
             data_vars={