Repository: capitalone/datacompy

Issue:
Ignore case when comparing strings
Might be a nice to have a optional ignore case feature so that `STRING` == `string`.
Should be relatively simple to add. Happy to take it on.

@theianrobertson thoughts?

Discussion:

Patch:
diff --git a/datacompy/core.py b/datacompy/core.py
index c6553f8..e39fa5e 100644
--- a/datacompy/core.py
+++ b/datacompy/core.py
@@ -62,6 +62,8 @@ class Compare(object):
         A string name for the second dataframe
     ignore_spaces : bool, optional
         Flag to strip whitespace (including newlines) from string columns
+    ignore_case : bool, optional
+        Flag to ignore the case of string columns
 
     Attributes
     ----------
@@ -82,6 +84,7 @@ class Compare(object):
         df1_name="df1",
         df2_name="df2",
         ignore_spaces=False,
+        ignore_case=False,
     ):
 
         if on_index and join_columns is not None:
@@ -105,7 +108,7 @@ class Compare(object):
         self.rel_tol = rel_tol
         self.df1_unq_rows = self.df2_unq_rows = self.intersect_rows = None
         self.column_stats = []
-        self._compare(ignore_spaces)
+        self._compare(ignore_spaces, ignore_case)
 
     @property
     def df1(self):
@@ -154,7 +157,7 @@ class Compare(object):
             if len(dataframe.drop_duplicates(subset=self.join_columns)) < len(dataframe):
                 self._any_dupes = True
 
-    def _compare(self, ignore_spaces):
+    def _compare(self, ignore_spaces, ignore_case):
         """Actually run the comparison.  This tries to run df1.equals(df2)
         first so that if they're truly equal we can tell.
 
@@ -176,7 +179,7 @@ class Compare(object):
         LOG.info("Number of columns in df2 and not in df1: {}".format(len(self.df2_unq_columns())))
         LOG.debug("Merging dataframes")
         self._dataframe_merge(ignore_spaces)
-        self._intersect_compare(ignore_spaces)
+        self._intersect_compare(ignore_spaces, ignore_case)
         if self.matches():
             LOG.info("df1 matches df2")
         else:
@@ -270,7 +273,7 @@ class Compare(object):
             )
         )
 
-    def _intersect_compare(self, ignore_spaces):
+    def _intersect_compare(self, ignore_spaces, ignore_case):
         """Run the comparison on the intersect dataframe
 
         This loops through all columns that are shared between df1 and df2, and
@@ -295,6 +298,7 @@ class Compare(object):
                     self.rel_tol,
                     self.abs_tol,
                     ignore_spaces,
+                    ignore_case,
                 )
                 match_cnt = self.intersect_rows[col_match].sum()
                 max_diff = calculate_max_diff(
@@ -585,7 +589,7 @@ def render(filename, *fields):
         return file_open.read().format(*fields)
 
 
-def columns_equal(col_1, col_2, rel_tol=0, abs_tol=0, ignore_spaces=False):
+def columns_equal(col_1, col_2, rel_tol=0, abs_tol=0, ignore_spaces=False, ignore_case=False):
     """Compares two columns from a dataframe, returning a True/False series,
     with the same index as column 1.
 
@@ -609,6 +613,8 @@ def columns_equal(col_1, col_2, rel_tol=0, abs_tol=0, ignore_spaces=False):
         Absolute tolerance
     ignore_spaces : bool, optional
         Flag to strip whitespace (including newlines) from string columns
+    ignore_case : bool, optional
+        Flag to ignore the case of string columns
 
     Returns
     -------
@@ -637,6 +643,12 @@ def columns_equal(col_1, col_2, rel_tol=0, abs_tol=0, ignore_spaces=False):
                     if col_2.dtype.kind == "O":
                         col_2 = col_2.str.strip()
 
+                if ignore_case:
+                    if col_1.dtype.kind == "O":
+                        col_1 = col_1.str.upper()
+                    if col_2.dtype.kind == "O":
+                        col_2 = col_2.str.upper()
+
                 if set([col_1.dtype.kind, col_2.dtype.kind]) == set(["M", "O"]):
                     compare = compare_string_and_date_columns(col_1, col_2)
                 else:


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 3251d37..7e37ac4 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -108,6 +108,28 @@ something||False
     assert_series_equal(expect_out, actual_out, check_names=False)
 
 
+def test_string_columns_equal_with_ignore_spaces_and_case():
+    data = """a|b|expected
+Hi|Hi|True
+Yo|Yo|True
+Hey|Hey |True
+résumé|resume|False
+résumé|résumé|True
+💩|💩|True
+💩|🤔|False
+ | |True
+  |       |True
+datacompy|DataComPy|True
+something||False
+|something|False
+||True"""
+    df = pd.read_csv(six.StringIO(data), sep="|")
+    actual_out = datacompy.columns_equal(df.a, df.b, rel_tol=0.2, ignore_spaces=True,
+                                         ignore_case=True)
+    expect_out = df["expected"]
+    assert_series_equal(expect_out, actual_out, check_names=False)
+
+
 def test_date_columns_equal():
     data = """a|b|expected
 2017-01-01|2017-01-01|True
@@ -158,6 +180,32 @@ def test_date_columns_equal_with_ignore_spaces():
     assert_series_equal(expect_out, actual_out_rev, check_names=False)
 
 
+def test_date_columns_equal_with_ignore_spaces_and_case():
+    data = """a|b|expected
+2017-01-01|2017-01-01   |True
+2017-01-02  |2017-01-02|True
+2017-10-01  |2017-10-10   |False
+2017-01-01||False
+|2017-01-01|False
+||True"""
+    df = pd.read_csv(six.StringIO(data), sep="|")
+    # First compare just the strings
+    actual_out = datacompy.columns_equal(df.a, df.b, rel_tol=0.2, ignore_spaces=True,
+                                         ignore_case=True)
+    expect_out = df["expected"]
+    assert_series_equal(expect_out, actual_out, check_names=False)
+
+    # Then compare converted to datetime objects
+    df["a"] = pd.to_datetime(df["a"])
+    df["b"] = pd.to_datetime(df["b"])
+    actual_out = datacompy.columns_equal(df.a, df.b, rel_tol=0.2, ignore_spaces=True)
+    expect_out = df["expected"]
+    assert_series_equal(expect_out, actual_out, check_names=False)
+    # and reverse
+    actual_out_rev = datacompy.columns_equal(df.b, df.a, rel_tol=0.2, ignore_spaces=True)
+    assert_series_equal(expect_out, actual_out_rev, check_names=False)
+
+
 def test_date_columns_unequal():
     """I want datetime fields to match with dates stored as strings
     """
@@ -323,6 +371,25 @@ def test_mixed_column_with_ignore_spaces():
     assert_series_equal(expect_out, actual_out, check_names=False)
 
 
+def test_mixed_column_with_ignore_spaces_and_case():
+    df = pd.DataFrame(
+        [
+            {"a": "hi", "b": "hi ", "expected": True},
+            {"a": 1, "b": 1, "expected": True},
+            {"a": np.inf, "b": np.inf, "expected": True},
+            {"a": Decimal("1"), "b": Decimal("1"), "expected": True},
+            {"a": 1, "b": "1 ", "expected": False},
+            {"a": 1, "b": "yo ", "expected": False},
+            {"a": "Hi", "b": "hI ", "expected": True},
+            {"a": "HI", "b": "HI ", "expected": True},
+            {"a": "hi", "b": "hi ", "expected": True},
+        ]
+    )
+    actual_out = datacompy.columns_equal(df.a, df.b, ignore_spaces=True, ignore_case=True)
+    expect_out = df["expected"]
+    assert_series_equal(expect_out, actual_out, check_names=False)
+
+
 def test_compare_df_setter_bad():
     df = pd.DataFrame([{"a": 1, "A": 2}, {"a": 2, "A": 2}])
     with raises(TypeError, message="df1 must be a pandas DataFrame"):
@@ -668,6 +735,23 @@ def test_strings_with_joins_with_ignore_spaces():
     assert compare.intersect_rows_match()
 
 
+
+def test_strings_with_joins_with_ignore_case():
+    df1 = pd.DataFrame([{"a": "hi", "b": "a"}, {"a": "bye", "b": "A"}])
+    df2 = pd.DataFrame([{"a": "hi", "b": "A"}, {"a": "bye", "b": "a"}])
+    compare = datacompy.Compare(df1, df2, "a", ignore_case=False)
+    assert not compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert not compare.intersect_rows_match()
+
+    compare = datacompy.Compare(df1, df2, "a", ignore_case=True)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+
+
 def test_decimal_with_joins_with_ignore_spaces():
     df1 = pd.DataFrame([{"a": 1, "b": " A"}, {"a": 2, "b": "A"}])
     df2 = pd.DataFrame([{"a": 1, "b": "A"}, {"a": 2, "b": "A "}])
@@ -684,6 +768,22 @@ def test_decimal_with_joins_with_ignore_spaces():
     assert compare.intersect_rows_match()
 
 
+def test_decimal_with_joins_with_ignore_case():
+    df1 = pd.DataFrame([{"a": 1, "b": "a"}, {"a": 2, "b": "A"}])
+    df2 = pd.DataFrame([{"a": 1, "b": "A"}, {"a": 2, "b": "a"}])
+    compare = datacompy.Compare(df1, df2, "a", ignore_case=False)
+    assert not compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert not compare.intersect_rows_match()
+
+    compare = datacompy.Compare(df1, df2, "a", ignore_case=True)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+
+
 def test_index_with_joins_with_ignore_spaces():
     df1 = pd.DataFrame([{"a": 1, "b": " A"}, {"a": 2, "b": "A"}])
     df2 = pd.DataFrame([{"a": 1, "b": "A"}, {"a": 2, "b": "A "}])
@@ -700,6 +800,22 @@ def test_index_with_joins_with_ignore_spaces():
     assert compare.intersect_rows_match()
 
 
+def test_index_with_joins_with_ignore_case():
+    df1 = pd.DataFrame([{"a": 1, "b": "a"}, {"a": 2, "b": "A"}])
+    df2 = pd.DataFrame([{"a": 1, "b": "A"}, {"a": 2, "b": "a"}])
+    compare = datacompy.Compare(df1, df2, on_index=True, ignore_case=False)
+    assert not compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert not compare.intersect_rows_match()
+
+    compare = datacompy.Compare(df1, df2, "a", ignore_case=True)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+
+
 MAX_DIFF_DF = pd.DataFrame(
     {
         "base": [1, 1, 1, 1, 1],