Repository: capitalone/datacompy
Pull request #406 (2025-04-22 14:58:43)

Release v0.16.6
Release v0.16.6
----------------

- #404 
- #405 


Patch:
diff --git a/.gitignore b/.gitignore
index 2e0f6e3..4edd8be 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,5 @@ docs/source/api/
 #edgetest
 .edgetest/
 .tmp/
+
+test.html
diff --git a/datacompy/__init__.py b/datacompy/__init__.py
index e29d6b6..879f32a 100644
--- a/datacompy/__init__.py
+++ b/datacompy/__init__.py
@@ -18,7 +18,7 @@ Originally started to be something of a replacement for SAS's PROC COMPARE for P
 Then extended to carry that functionality over to Spark Dataframes.
 """
 
-__version__ = "0.16.5"
+__version__ = "0.16.6"
 
 import platform
 from warnings import warn
diff --git a/datacompy/core.py b/datacompy/core.py
index 9e88d05..09ea181 100644
--- a/datacompy/core.py
+++ b/datacompy/core.py
@@ -840,6 +840,13 @@ def columns_equal(
     As of version ``0.14.0`` If a column is of a mixed data type the compare will
     default to returning ``False``.
 
+    Notes
+    -----
+    - ``list`` and ``np.array`` types will be compared row wise using ``np.array_equal``.
+      Depending on the size of your data this might lead to performance issues.
+    - All the rows must be of the same type otherwise it is considered "mixed"
+      and will default to being ``False`` for everything.
+
     Parameters
     ----------
     col_1 : Pandas.Series
@@ -874,11 +881,21 @@ def columns_equal(
         if col_2.dtype.kind == "O" and pd.api.types.is_string_dtype(col_2):
             col_2 = col_2.str.upper()
 
-    # short circuit if comparing mixed type columns. We don't want to support this moving forward.
+    # short circuit if comparing mixed type columns. Check list/arrrays or just return false for everything else.
     if pd.api.types.infer_dtype(col_1).startswith("mixed") or pd.api.types.infer_dtype(
         col_2
     ).startswith("mixed"):
-        compare = pd.Series(False, index=col_1.index)
+        if all(isinstance(item, (list, np.ndarray)) for item in col_1) and all(  # noqa: UP038
+            isinstance(item, (list, np.ndarray))  # noqa: UP038
+            for item in col_2
+        ):  # compare list like
+            # join together and apply np.array_equal
+            temp_df = pd.DataFrame({"col_1": col_1, "col_2": col_2})
+            compare = temp_df.apply(
+                lambda row: np.array_equal(row.col_1, row.col_2, equal_nan=True), axis=1
+            )
+        else:
+            compare = pd.Series(False, index=col_1.index)
     elif pd.api.types.is_string_dtype(col_1) and pd.api.types.is_string_dtype(col_2):
         try:
             compare = pd.Series(
diff --git a/datacompy/polars.py b/datacompy/polars.py
index 666ffef..c375304 100644
--- a/datacompy/polars.py
+++ b/datacompy/polars.py
@@ -35,6 +35,7 @@ from datacompy.base import BaseCompare, temp_column_name
 LOG = logging.getLogger(__name__)
 
 STRING_TYPE = ["String", "Utf8"]
+LIST_TYPE = ["List", "Array"]
 
 
 class PolarsCompare(BaseCompare):
@@ -852,8 +853,13 @@ def columns_equal(
         if str(col_2.dtype) in STRING_TYPE:
             col_2 = col_2.str.to_uppercase()
 
-    if (str(col_1.dtype) in STRING_TYPE and str(col_2.dtype) in STRING_TYPE) or (
-        col_1.dtype.is_temporal() and col_2.dtype.is_temporal()
+    if (
+        (str(col_1.dtype) in STRING_TYPE and str(col_2.dtype) in STRING_TYPE)
+        or (col_1.dtype.is_temporal() and col_2.dtype.is_temporal())
+        or (
+            str(col_1.dtype.base_type()) in LIST_TYPE
+            and str(col_2.dtype.base_type()) in LIST_TYPE
+        )
     ):
         compare = pl.Series(
             (col_1.eq_missing(col_2)) | (col_1.is_null() & col_2.is_null())
diff --git a/pyproject.toml b/pyproject.toml
index a4e6d28..2459c3e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,7 +13,7 @@ maintainers = [
   { name="Raymond Haffar", email="raymond.haffar@capitalone.com" },
 ]
 license = {text = "Apache Software License"}
-dependencies = ["pandas<=2.2.3,>=0.25.0", "numpy<=2.2.3,>=1.22.0", "ordered-set<=4.1.0,>=4.0.2", "polars[pandas]<=1.23.0,>=0.20.4"]
+dependencies = ["pandas<=2.2.3,>=0.25.0", "numpy<=2.2.5,>=1.22.0", "ordered-set<=4.1.0,>=4.0.2", "polars[pandas]<=1.27.1,>=0.20.4"]
 requires-python = ">=3.10.0"
 classifiers = [
     "Intended Audience :: Developers",


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 1534cc4..4040c47 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1563,3 +1563,119 @@ def test_merge_date_ignore_spaces():
     )
     compare._dataframe_merge(ignore_spaces=True)
     assert len(compare.intersect_rows) == 4
+
+
+def test_columns_equal_numpy_arrays():
+    # all equal
+    df1 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    df2 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert actual.all()
+
+    # all mismatch
+    df1 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    df2 = pd.DataFrame({"array_col": [np.array([j]) for j in range(1, 11)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert not actual.all()
+
+    # some equal
+    df1 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    df2 = pd.DataFrame({"array_col": [np.array([j]) for j in range(10)]})
+    df2.iloc[1] = df2.iloc[0]  # set an item to be off
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert (
+        actual
+        == np.array([True, False, True, True, True, True, True, True, True, True])
+    ).all()
+
+    # different shapes
+    df1 = pd.DataFrame(
+        {
+            "array_col": [
+                np.array([]),
+                np.array([np.nan]),
+                np.array([1, 2]),
+                np.array([1, 3]),
+                np.array([2, 3]),
+                np.array([1, 2, 3]),
+            ]
+        }
+    )
+    df2 = pd.DataFrame(
+        {
+            "array_col": [
+                np.array([]),
+                np.array([np.nan]),
+                np.array([1, 2, 3]),
+                np.array([1, 3]),
+                np.array([2, 3]),
+                np.array([1, 2]),
+            ]
+        }
+    )
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert (actual == np.array([True, True, False, True, True, False])).all()
+
+    # empty
+    df1 = pd.DataFrame({"array_col": [np.array([]) for _ in range(10)]})
+    df2 = pd.DataFrame({"array_col": [np.array([]) for _ in range(10)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert actual.all()
+
+
+def test_columns_equal_lists():
+    # all equal
+    df1 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    df2 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert actual.all()
+
+    # all mismatch
+    df1 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    df2 = pd.DataFrame({"array_col": [[i] for i in range(1, 11)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert not actual.all()
+
+    # some equal
+    df1 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    df2 = pd.DataFrame({"array_col": [[i] for i in range(10)]})
+    df2.iloc[1] = df2.iloc[0]  # set an item to be off
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert (
+        actual
+        == np.array([True, False, True, True, True, True, True, True, True, True])
+    ).all()
+
+    # different shapes
+    df1 = pd.DataFrame(
+        {
+            "array_col": [
+                [],
+                [np.nan],
+                [1, 2],
+                [1, 3],
+                [2, 3],
+                [1, 2, 3],
+            ]
+        }
+    )
+    df2 = pd.DataFrame(
+        {
+            "array_col": [
+                [],
+                [np.nan],
+                [1, 2, 3],
+                [1, 3],
+                [2, 3],
+                [1, 2],
+            ]
+        }
+    )
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert (actual == np.array([True, True, False, True, True, False])).all()
+
+    # empty
+    df1 = pd.DataFrame({"array_col": [[] for _ in range(10)]})
+    df2 = pd.DataFrame({"array_col": [[] for _ in range(10)]})
+    actual = datacompy.columns_equal(df1.array_col, df2.array_col)
+    assert actual.all()
diff --git a/tests/test_polars.py b/tests/test_polars.py
index 2eff404..d4c2e77 100644
--- a/tests/test_polars.py
+++ b/tests/test_polars.py
@@ -1518,3 +1518,133 @@ def test_temporal_equal():
     col_b = df["b"].str.to_datetime(strict=False)
     actual_out = columns_equal(col_a, col_b)
     assert_series_equal(expect_out, actual_out, check_names=False)
+
+
+def test_columns_equal_arrays():
+    # all equal
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert actual.explode().all()
+
+    # all mismatch
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[2], [3], [4], [5], [6]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert not actual.explode().all()
+
+    # some equal
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[1], [1], [3], [4], [5]]},
+        schema={"array_col": pl.Array(pl.Int64, 1)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert (actual.explode() == pl.Series([True, False, True, True, True])).all()
+
+    # empty
+    df1 = pl.DataFrame(
+        {"array_col": [[], [], [], [], []]},
+        schema={"array_col": pl.Array(pl.Int64, 0)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[], [], [], [], []]},
+        schema={"array_col": pl.Array(pl.Int64, 0)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert actual.explode().all()
+
+
+def test_columns_equal_lists():
+    # all equal
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert actual.all()
+
+    # all mismatch
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[2], [3], [4], [5], [6]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert not actual.all()
+
+    # some equal
+    df1 = pl.DataFrame(
+        {"array_col": [[1], [2], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[1], [1], [3], [4], [5]]},
+        schema={"array_col": pl.List(pl.Int64)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert (actual == pl.Series([True, False, True, True, True])).all()
+
+    # different shapes
+    df1 = pl.DataFrame(
+        {
+            "array_col": [
+                [],
+                [np.nan],
+                [1, 2],
+                [1, 3],
+                [2, 3],
+                [1, 2, 3],
+            ]
+        },
+        schema={"array_col": pl.List(pl.Float64)},
+    )
+    df2 = pl.DataFrame(
+        {
+            "array_col": [
+                [],
+                [np.nan],
+                [1, 2, 3],
+                [1, 3],
+                [2, 3],
+                [1, 2],
+            ]
+        },
+        schema={"array_col": pl.List(pl.Float64)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert (actual == pl.Series([True, True, False, True, True, False])).all()
+
+    # empty
+    df1 = pl.DataFrame(
+        {"array_col": [[], [], [], [], []]},
+        schema={"array_col": pl.Array(pl.Int64, 0)},
+    )
+    df2 = pl.DataFrame(
+        {"array_col": [[], [], [], [], []]},
+        schema={"array_col": pl.Array(pl.Int64, 0)},
+    )
+    actual = columns_equal(df1["array_col"], df2["array_col"])
+    assert actual.all()
diff --git a/tests/test_spark/test_sql_spark.py b/tests/test_spark/test_sql_spark.py
index ef5106f..d21695d 100644
--- a/tests/test_spark/test_sql_spark.py
+++ b/tests/test_spark/test_sql_spark.py
@@ -43,6 +43,13 @@ from datacompy.spark.sql import (
     temp_column_name,
 )
 from pandas.testing import assert_frame_equal, assert_series_equal
+from pyspark.sql.types import (
+    ArrayType,
+    IntegerType,
+    StringType,
+    StructField,
+    StructType,
+)
 
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 
@@ -1507,3 +1514,113 @@ def test_non_full_join_counts_some_matches(spark_session):
             ]
         ),
     )
+
+
+def test_columns_equal_arrays(spark_session):
+    data = [
+        (
+            "1",
+            [1],
+            [1],
+            [2],
+            [1],
+            [None],
+            [None],
+            [1],
+            [1],
+        ),
+        (
+            "2",
+            [2],
+            [2],
+            [3],
+            [2],
+            [None],
+            [None],
+            [
+                1,
+                2,
+            ],
+            [
+                1,
+                2,
+            ],
+        ),
+        (
+            "3",
+            [3],
+            [3],
+            [4],
+            [3],
+            [None],
+            [None],
+            [1, 2, 3],
+            [1, 2, 3],
+        ),
+        (
+            "4",
+            [4],
+            [4],
+            [5],
+            [4],
+            [None],
+            [None],
+            [1, 2, 3, 4],
+            [4, 3, 2, 1],
+        ),
+        (
+            "5",
+            [5],
+            [5],
+            [6],
+            [9],
+            [None],
+            [None],
+            [1, 2, 3, 4, 5],
+            [5, 4, 3, 2],
+        ),
+    ]
+    schema = StructType(
+        [
+            StructField("name", StringType(), True),
+            StructField("a", ArrayType(IntegerType()), True),
+            StructField("b", ArrayType(IntegerType()), True),
+            StructField("c", ArrayType(IntegerType()), True),
+            StructField("d", ArrayType(IntegerType()), True),
+            StructField("e", ArrayType(IntegerType()), True),
+            StructField("f", ArrayType(IntegerType()), True),
+            StructField("g", ArrayType(IntegerType()), True),
+            StructField("h", ArrayType(IntegerType()), True),
+        ]
+    )
+    df = spark_session.createDataFrame(data, schema)
+
+    # all equal
+    all_equal = columns_equal(df, "a", "b", "all_equal")
+    assert all_equal.toPandas()["all_equal"].all()
+
+    # all mismatch
+    all_mismatch = columns_equal(df, "a", "c", "all_mismatch")
+    assert not all_mismatch.toPandas()["all_mismatch"].all()
+
+    # some equal
+    some_equal = columns_equal(df, "a", "d", "some_equal")
+    assert (
+        some_equal.toPandas()["some_equal"]
+        == pd.Series([True, True, True, True, False])
+    ).all()
+
+    # null all
+    null_all = columns_equal(df, "e", "f", "null_all")
+    assert null_all.toPandas()["null_all"].all()
+
+    # empty all vs value
+    none_value_all = columns_equal(df, "a", "f", "none_value_all")
+    assert not none_value_all.toPandas()["none_value_all"].all()
+
+    # different shape arrays
+    diff_shapes = columns_equal(df, "g", "h", "diff_shapes")
+    assert (
+        diff_shapes.toPandas()["diff_shapes"]
+        == pd.Series([True, True, True, False, False])
+    ).all()