Repository: capitalone/datacompy (python)

Issue:
Inequality when comparing two empty numpy arrays
Comparison of two empty `numpy` arrays currently return `False`, which results in showing diffs where there shouldn't be.

This is due to the way `numpy` compares empty arrays.
Running `bool(np.array([]) == np.array([]))` returns `False` and throws this warning:

```
The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.
```

Reproduce this bug with:
```
df1 = pd.DataFrame({"some_col": [np.array([]) for _ in range(10)], "id": [i for i in range(10)]})
df2 = pd.DataFrame({"some_col": [np.array([]) for _ in range(10)], "id": [i for i in range(10)]})

pdcompare = datacompy.Compare(df1, df2, join_columns="id")
print(pdcompare.report())
```
output:
```
DataComPy Comparison
--------------------

DataFrame Summary
-----------------

  DataFrame  Columns  Rows
0       df1        2    10
1       df2        2    10

Column Summary
--------------

Number of columns in common: 2
Number of columns in df1 but not in df2: 0
Number of columns in df2 but not in df1: 0

Row Summary
-----------

Matched on: id
Any duplicates on match values: No
Absolute Tolerance: 0
Relative Tolerance: 0
Number of rows in common: 10
Number of rows in df1 but not in df2: 0
Number of rows in df2 but not in df1: 0

Number of rows with some compared columns unequal: 10
Number of rows with all compared columns equal: 0

Column Comparison
-----------------

Number of columns compared with some values unequal: 1
Number of columns compared with all values equal: 1
Total number of values which compare unequal: 10

Columns with Unequal Values or Types
------------------------------------

     Column df1 dtype df2 dtype  # Unequal  Max Diff  # Null Diff
0  some_col    object    object         10         0            0

Sample Rows with Unequal Values
-------------------------------

   id some_col (df1) some_col (df2)
9   9             []             []
0   0             []             []
3   3             []             []
7   7             []             []
5   5             []             []
1   1             []             []
4   4             []             []
2   2             []             []
8   8             []             []
6   6             []             []
```


Pull request:
feat: Compatibility for Array and Lists
This should fix #58 

Adding in `compare_column` compatibility for:
- Pandas
  - `np.ndarray`
  - `list`
- Polars
  - `pl.List`
  - `pl.Array`

Patch:
diff --git a/.gitignore b/.gitignore
index 2e0f6e3..4edd8be 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,5 @@ docs/source/api/
 #edgetest
 .edgetest/
 .tmp/
+
+test.html
diff --git a/datacompy/core.py b/datacompy/core.py
index 9e88d05..09ea181 100644
--- a/datacompy/core.py
+++ b/datacompy/core.py
@@ -840,6 +840,13 @@ def columns_equal(
     As of version ``0.14.0`` If a column is of a mixed data type the compare will
     default to returning ``False``.
 
+    Notes
+    -----
+    - ``list`` and ``np.array`` types will be compared row wise using ``np.array_equal``.
+      Depending on the size of your data this might lead to performance issues.
+    - All the rows must be of the same type otherwise it is considered "mixed"
+      and will default to being ``False`` for everything.
+
     Parameters
     ----------
     col_1 : Pandas.Series
@@ -874,11 +881,21 @@ def columns_equal(
         if col_2.dtype.kind == "O" and pd.api.types.is_string_dtype(col_2):
             col_2 = col_2.str.upper()
 
-    # short circuit if comparing mixed type columns. We don't want to support this moving forward.
+    # short circuit if comparing mixed type columns. Check list/arrrays or just return false for everything else.
     if pd.api.types.infer_dtype(col_1).startswith("mixed") or pd.api.types.infer_dtype(
         col_2
     ).startswith("mixed"):
-        compare = pd.Series(False, index=col_1.index)
+        if all(isinstance(item, (list, np.ndarray)) for item in col_1) and all(  # noqa: UP038
+            isinstance(item, (list, np.ndarray))  # noqa: UP038
+            for item in col_2
+        ):  # compare list like
+            # join together and apply np.array_equal
+            temp_df = pd.DataFrame({"col_1": col_1, "col_2": col_2})
+            compare = temp_df.apply(
+                lambda row: np.array_equal(row.col_1, row.col_2, equal_nan=True), axis=1
+            )
+        else:
+            compare = pd.Series(False, index=col_1.index)
     elif pd.api.types.is_string_dtype(col_1) and pd.api.types.is_string_dtype(col_2):
         try:
             compare = pd.Series(
diff --git a/datacompy/polars.py b/datacompy/polars.py
index 666ffef..c375304 100644
--- a/datacompy/polars.py
+++ b/datacompy/polars.py
@@ -35,6 +35,7 @@ from datacompy.base import BaseCompare, temp_column_name
 LOG = logging.getLogger(__name__)
 
 STRING_TYPE = ["String", "Utf8"]
+LIST_TYPE = ["List", "Array"]
 
 
 class PolarsCompare(BaseCompare):
@@ -852,8 +853,13 @@ def columns_equal(
         if str(col_2.dtype) in STRING_TYPE:
             col_2 = col_2.str.to_uppercase()
 
-    if (str(col_1.dtype) in STRING_TYPE and str(col_2.dtype) in STRING_TYPE) or (
-        col_1.dtype.is_temporal() and col_2.dtype.is_temporal()
+    if (
+        (str(col_1.dtype) in STRING_TYPE and str(col_2.dtype) in STRING_TYPE)
+        or (col_1.dtype.is_temporal() and col_2.dtype.is_temporal())
+        or (
+            str(col_1.dtype.base_type()) in LIST_TYPE
+            and str(col_2.dtype.base_type()) in LIST_TYPE
+        )
     ):
         compare = pl.Series(
             (col_1.eq_missing(col_2)) | (col_1.is_null() & col_2.is_null())


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 1534cc4..4040c47 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1563,3 +1563,119 @@ def test_merge_date_ignore_spaces():
     )
     compare._dataframe_merge(ignore_spaces=True)
     assert len(compare.intersect_rows) == 4
+
+
+def test_columns_equal_numpy_arrays():
+    # all equal
+    df1 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    df2 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert actual.all()
+
+    # all mismatch
+    df1 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    df2 = pd.DataFrame({"array_col": [np.array([j]) for j in range(1, 11)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert not actual.all()
+
+    # some equal
+    df1 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    df2 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    df2.iloc[1] = df2.iloc[0]  # set an item to be off
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert (
+        actual
+        == np.array([True, False, True, True, True, True, True, True, True, True])
+    ).all()
+
+    # different shapes
+    df1 = pd.DataFrame(
+        {
+            "array_col": [
+                np.array([]),
+                np.array([np.nan]),
+                np.array([1, 2]),
+                np.array([1, 3]),
+                np.array([2, 3]),
+                np.array([1, 2, 3]),
+            ]
+        }
+    )
+    df2 = pd.DataFrame(
+        {
+            "array_col": [
+                np.array([]),
+                np.array([np.nan]),
+                np.array([1, 2, 3]),
+                np.array([1, 3]),
+                np.array([2, 3]),
+                np.array([1, 2]),
+            ]
+        }
+    )
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert (actual == np.array([True, True, False, True, True, False])).all()
+
+    # empty
+    df1 = pd.DataFrame({"array_col": [np.array([]) for _ in range(10)]})
+    df2 = pd.DataFrame({"array_col": [np.array([]) for _ in range(10)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert actual.all()
+
+
+def test_columns_equal_lists():
+    # all equal
+    df1 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    df2 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert actual.all()
+
+    # all mismatch
+    df1 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    df2 = pd.DataFrame({"array_col": [[i] for i in range(1, 11)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert not actual.all()
+
+    # some equal
+    df1 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    df2 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    df2.iloc[1] = df2.iloc[0]  # set an item to be off
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert (
+        actual
+        == np.array([True, False, True, True, True, True, True, True, True, True])
+    ).all()
+
+    # different shapes
+    df1 = pd.DataFrame(
+        {
+            "array_col": [
+                [],
+                [np.nan],
+                [1, 2],
+                [1, 3],
+                [2, 3],
+                [1, 2, 3],
+            ]
+        }
+    )
+    df2 = pd.DataFrame(
+        {
+            "array_col": [
+                [],
+                [np.nan],
+                [1, 2, 3],
+                [1, 3],
+                [2, 3],
+                [1, 2],
+            ]
+        }
+    )
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert (actual == np.array([True, True, False, True, True, False])).all()
+
+    # empty
+    df1 = pd.DataFrame({"array_col": [[] for _ in range(10)]})
+    df2 = pd.DataFrame({"array_col": [[] for _ in range(10)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert actual.all()
diff --git a/tests/test_polars.py b/tests/test_polars.py
index 2eff404..d4c2e77 100644
--- a/tests/test_polars.py
+++ b/tests/test_polars.py
@@ -1518,3 +1518,133 @@ def test_temporal_equal():
     col_b = df["b"].str.to_datetime(strict=False)
     actual_out = columns_equal(col_a, col_b)
     assert_series_equal(expect_out, actual_out, check_names=False)
+
+
+def test_columns_equal_arrays():
+    # all equal
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert actual.explode().all()
+
+    # all mismatch
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[2], [3], [4], [5], [6]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert not actual.explode().all()
+
+    # some equal
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[1], [1], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert (actual.explode() == pl.Series([True, False, True, True, True])).all()
+
+    # empty
+    df1 = pl.DataFrame(
+        {"array_col": [[], [], [], [], []]},
+        schema={"array_col": pl.Array(pl.Int64, 0)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[], [], [], [], []]},
+        schema={"array_col": pl.Array(pl.Int64, 0)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert actual.explode().all()
+
+
+def test_columns_equal_lists():
+    # all equal
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert actual.all()
+
+    # all mismatch
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[2], [3], [4], [5], [6]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert not actual.all()
+
+    # some equal
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[1], [1], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert (actual == pl.Series([True, False, True, True, True])).all()
+
+    # different shapes
+    df1 = pl.DataFrame(
+        {
+            "array_col": [
+                [],
+                [np.nan],
+                [1, 2],
+                [1, 3],
+                [2, 3],
+                [1, 2, 3],
+            ]
+        },
+        schema={"array_col": pl.List(pl.Float64)},
+    )
+    df2 = pl.DataFrame(
+        {
+            "array_col": [
+                [],
+                [np.nan],
+                [1, 2, 3],
+                [1, 3],
+                [2, 3],
+                [1, 2],
+            ]
+        },
+        schema={"array_col": pl.List(pl.Float64)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert (actual == pl.Series([True, True, False, True, True, False])).all()
+
+    # empty
+    df1 = pl.DataFrame(
+        {"array_col": [[], [], [], [], []]},
+        schema={"array_col": pl.Array(pl.Int64, 0)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[], [], [], [], []]},
+        schema={"array_col": pl.Array(pl.Int64, 0)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert actual.all()
diff --git a/tests/test_spark/test_sql_spark.py b/tests/test_spark/test_sql_spark.py
index ef5106f..d21695d 100644
--- a/tests/test_spark/test_sql_spark.py
+++ b/tests/test_spark/test_sql_spark.py
@@ -43,6 +43,13 @@ from datacompy.spark.sql import (
     temp_column_name,
 )
 from pandas.testing import assert_frame_equal, assert_series_equal
+from pyspark.sql.types import (
+    ArrayType,
+    IntegerType,
+    StringType,
+    StructField,
+    StructType,
+)
 
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 
@@ -1507,3 +1514,113 @@ def test_non_full_join_counts_some_matches(spark_session):
             ]
         ),
     )
+
+
+def test_columns_equal_arrays(spark_session):
+    data = [
+        (
+            "1",
+            [1],
+            [1],
+            [2],
+            [1],
+            [None],
+            [None],
+            [1],
+            [1],
+        ),
+        (
+            "2",
+            [2],
+            [2],
+            [3],
+            [2],
+            [None],
+            [None],
+            [
+                1,
+                2,
+            ],
+            [
+                1,
+                2,
+            ],
+        ),
+        (
+            "3",
+            [3],
+            [3],
+            [4],
+            [3],
+            [None],
+            [None],
+            [1, 2, 3],
+            [1, 2, 3],
+        ),
+        (
+            "4",
+            [4],
+            [4],
+            [5],
+            [4],
+            [None],
+            [None],
+            [1, 2, 3, 4],
+            [4, 3, 2, 1],
+        ),
+        (
+            "5",
+            [5],
+            [5],
+            [6],
+            [9],
+            [None],
+            [None],
+            [1, 2, 3, 4, 5],
+            [5, 4, 3, 2],
+        ),
+    ]
+    schema = StructType(
+        [
+            StructField("name", StringType(), True),
+            StructField("a", ArrayType(IntegerType()), True),
+            StructField("b", ArrayType(IntegerType()), True),
+            StructField("c", ArrayType(IntegerType()), True),
+            StructField("d", ArrayType(IntegerType()), True),
+            StructField("e", ArrayType(IntegerType()), True),
+            StructField("f", ArrayType(IntegerType()), True),
+            StructField("g", ArrayType(IntegerType()), True),
+            StructField("h", ArrayType(IntegerType()), True),
+        ]
+    )
+    df = spark_session.createDataFrame(data, schema)
+
+    # all equal
+    all_equal = columns_equal(df, "a", "b", "all_equal")
+    assert all_equal.toPandas()["all_equal"].all()
+
+    # all mismatch
+    all_mismatch = columns_equal(df, "a", "c", "all_mismatch")
+    assert not all_mismatch.toPandas()["all_mismatch"].all()
+
+    # some equal
+    some_equal = columns_equal(df, "a", "d", "some_equal")
+    assert (
+        some_equal.toPandas()["some_equal"]
+        == pd.Series([True, True, True, True, False])
+    ).all()
+
+    # null all
+    null_all = columns_equal(df, "e", "f", "null_all")
+    assert null_all.toPandas()["null_all"].all()
+
+    # empty all vs value
+    none_value_all = columns_equal(df, "a", "f", "none_value_all")
+    assert not none_value_all.toPandas()["none_value_all"].all()
+
+    # different shape arrays
+    diff_shapes = columns_equal(df, "g", "h", "diff_shapes")
+    assert (
+        diff_shapes.toPandas()["diff_shapes"]
+        == pd.Series([True, True, True, False, False])
+    ).all()