Repository: capitalone/datacompy

Issue:
Doesnt take into account join columns when doing strip() of white spaces
Join columns can have whitespaces as well,so ideally below section should be enhanced to check for ignore_spaces and strip() if it is true for string columns
This is causing complete mismatch of 2 dataframes here in case 1 of join column has space:

if on_index and join_columns is not None:
            raise Exception("Only provide on_index or join_columns")
        elif on_index:
            self.on_index = True
            self.join_columns = []
        elif isinstance(join_columns, str):
            self.join_columns = [join_columns.lower()]
            self.on_index = False
        else:
            self.join_columns = [col.lower() for col in join_columns]
            self.on_index = False

Discussion:
fdosani: Yeah just for the check really. I don’t want to strip something which isn’t
a string as it will throw an error.

On Thu, May 14, 2020 at 9:46 PM Jan Borchmann <notifications@github.com>
wrote:

> *@jborchma* commented on this pull request.
>
> I have small question
> ------------------------------
>
> In datacompy/core.py
> <https://github.com/capitalone/datacompy/pull/62#discussion_r425506110>:
>
> > @@ -229,6 +230,14 @@ def _dataframe_merge(self, ignore_spaces):
>          else:
>              params = {"on": self.join_columns}
>
> +        if ignore_spaces:
> +            self.df1[self.join_columns] = self.df1[self.join_columns].apply(
> +                lambda x: x.str.strip() if x.dtype.kind == "O" else x
>
> Hmm, so apply is not the fastest...I guess you did this due to the
> element dtype check. How come you're doing that one? Maybe we could do it
> vectorized? All the str methods work as vectorized methods.
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/capitalone/datacompy/pull/62#pullrequestreview-412276832>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/ACPJA5VA6QFK2CQ3CY4VHXLRRSGFFANCNFSM4NBD2ILA>
> .
>


Patch:
diff --git a/datacompy/core.py b/datacompy/core.py
index e0cb46f..5bad7dc 100644
--- a/datacompy/core.py
+++ b/datacompy/core.py
@@ -61,7 +61,8 @@ class Compare:
     df2_name : str, optional
         A string name for the second dataframe
     ignore_spaces : bool, optional
-        Flag to strip whitespace (including newlines) from string columns
+        Flag to strip whitespace (including newlines) from string columns (including any join
+        columns)
     ignore_case : bool, optional
         Flag to ignore the case of string columns
 
@@ -229,6 +230,13 @@ class Compare:
         else:
             params = {"on": self.join_columns}
 
+        if ignore_spaces:
+            for column in self.join_columns:
+                if self.df1[column].dtype.kind == "O":
+                    self.df1[column] = self.df1[column].str.strip()
+                if self.df2[column].dtype.kind == "O":
+                    self.df2[column] = self.df2[column].str.strip()
+
         outer_join = self.df1.merge(
             self.df2, how="outer", suffixes=("_df1", "_df2"), indicator=True, **params
         )


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 280f576..65cc196 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -816,6 +816,41 @@ def test_index_with_joins_with_ignore_case():
     assert compare.intersect_rows_match()
 
 
+def test_strings_with_ignore_spaces_and_join_columns():
+    df1 = pd.DataFrame([{"a": "hi", "b": "A"}, {"a": "bye", "b": "A"}])
+    df2 = pd.DataFrame([{"a": " hi ", "b": "A"}, {"a": " bye ", "b": "A"}])
+    compare = datacompy.Compare(df1, df2, "a", ignore_spaces=False)
+    assert not compare.matches()
+    assert compare.all_columns_match()
+    assert not compare.all_rows_overlap()
+    assert compare.count_matching_rows() == 0
+
+    compare = datacompy.Compare(df1, df2, "a", ignore_spaces=True)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+    assert compare.count_matching_rows() == 2
+
+
+def test_integers_with_ignore_spaces_and_join_columns():
+    df1 = pd.DataFrame([{"a": 1, "b": "A"}, {"a": 2, "b": "A"}])
+    df2 = pd.DataFrame([{"a": 1, "b": "A"}, {"a": 2, "b": "A"}])
+    compare = datacompy.Compare(df1, df2, "a", ignore_spaces=False)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+    assert compare.count_matching_rows() == 2
+
+    compare = datacompy.Compare(df1, df2, "a", ignore_spaces=True)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+    assert compare.count_matching_rows() == 2
+
+
 MAX_DIFF_DF = pd.DataFrame(
     {
         "base": [1, 1, 1, 1, 1],