Repository: TTitcombe/PrivacyPanda

Issue:
Remove privacy-breaching columns
**The issue**
The simplest way to handle edge cases is to remove a column which contains _any_ breach of privacy.

**Proposed solution**
There should exist a function which identifies columns in a dataframe which contain privacy-breaching data and returns the dataframe with those columns removed


Discussion:

Patch:
diff --git a/privacypanda/__init__.py b/privacypanda/__init__.py
index a13b45a..5e891fe 100644
--- a/privacypanda/__init__.py
+++ b/privacypanda/__init__.py
@@ -1,3 +1,4 @@
 from .addresses import *
+from .anonymize import *
 
 __version__ = "0.1.0dev"
diff --git a/privacypanda/addresses.py b/privacypanda/addresses.py
index 5198ecc..6613cbc 100644
--- a/privacypanda/addresses.py
+++ b/privacypanda/addresses.py
@@ -4,12 +4,12 @@ Code for identifying addresses
 import re
 from typing import List
 
-import numpy as np
 import pandas as pd
+from numpy import dtype as np_dtype
 
 __all__ = ["check_addresses"]
 
-OBJECT_DTYPE = np.dtype("O")
+OBJECT_DTYPE = np_dtype("O")
 
 # ----- Regex constants ----- #
 LETTER = "[a-zA-Z]"
@@ -55,6 +55,7 @@ def check_addresses(df: pd.DataFrame) -> List:
         # Only check column if it may contain strings
         if row.dtype == OBJECT_DTYPE:
             for item in row:
+                item = str(item)  # convert incase column has mixed data types
 
                 if UK_POSTCODE_PATTERN.match(item) or SIMPLE_ADDRESS_PATTERN.match(
                     item
diff --git a/privacypanda/anonymize.py b/privacypanda/anonymize.py
new file mode 100644
index 0000000..44596a7
--- /dev/null
+++ b/privacypanda/anonymize.py
@@ -0,0 +1,39 @@
+"""
+Code for cleaning a dataframe of private data
+"""
+import pandas as pd
+from numpy import unique as np_unique
+
+from .addresses import check_addresses
+
+
+def anonymize(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Remove private data from a dataframe
+
+    Any column containing at least one piece of private data is removed from
+    the dataframe. This is a naive solution but limits the possibility of
+    false negatives.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        The dataframe to anonymize
+
+    Returns
+    -------
+    pd.DataFrame
+        The dataframe with columns containing private data removed
+    """
+    private_cols = []
+
+    checks = [check_addresses]
+    for check in checks:
+        new_private_cols = check(df)
+        private_cols += new_private_cols
+
+    # Get unique columns
+    private_cols = np_unique(private_cols).tolist()
+
+    # Drop columns
+    return df.drop(private_cols, axis=1)


Test patch:
diff --git a/tests/test_address_identification.py b/tests/test_address_identification.py
index f6100c0..396e172 100644
--- a/tests/test_address_identification.py
+++ b/tests/test_address_identification.py
@@ -51,3 +51,18 @@ def test_address_check_returns_empty_list_if_no_addresses_found():
     expected_private_columns = []
 
     assert actual_private_columns == expected_private_columns
+
+
+def test_check_addresses_can_handle_mixed_dtype_columns():
+    df = pd.DataFrame(
+        {
+            "privateColumn": [True, "AB1 1AB", "c"],
+            "privateColumn2": [1, "b", "10 Downing Street"],
+            "nonPrivateColumn": [0, True, "test"],
+        }
+    )
+
+    actual_private_columns = pp.check_addresses(df)
+    expected_private_columns = ["privateColumn", "privateColumn2"]
+
+    assert actual_private_columns == expected_private_columns
diff --git a/tests/test_anonymization.py b/tests/test_anonymization.py
new file mode 100644
index 0000000..916ba93
--- /dev/null
+++ b/tests/test_anonymization.py
@@ -0,0 +1,53 @@
+"""
+Test functions for anonymizing dataframes
+"""
+import pandas as pd
+import pytest
+
+import privacypanda as pp
+
+
+@pytest.mark.parametrize(
+    "address",
+    [
+        "10 Downing Street",
+        "10 downing street",
+        "1 the Road",
+        "01 The Road",
+        "1234 The Road",
+        "55 Maple Avenue",
+        "4 Python Way",
+        "AB1 1AB",
+        "AB12 1AB",
+    ],
+)
+def test_removes_columns_containing_addresses(address):
+    df = pd.DataFrame(
+        {
+            "privateData": ["a", "b", "c", address],
+            "nonPrivateData": ["a", "b", "c", "d"],
+            "nonPrivataData2": [1, 2, 3, 4],
+        }
+    )
+
+    expected_df = pd.DataFrame(
+        {"nonPrivateData": ["a", "b", "c", "d"], "nonPrivataData2": [1, 2, 3, 4]}
+    )
+
+    actual_df = pp.anonymize(df)
+
+    pd.testing.assert_frame_equal(actual_df, expected_df)
+
+
+def test_returns_empty_dataframe_if_all_columns_contain_private_information():
+    df = pd.DataFrame(
+        {
+            "nonPrivateData": ["a", "AB1 1AB", "c", "d"],
+            "PrivataData2": [1, 2, 3, "AB1 1AB"],
+        }
+    )
+
+    expected_df = pd.DataFrame(index=[0, 1, 2, 3])
+    actual_df = pp.anonymize(df)
+
+    pd.testing.assert_frame_equal(actual_df, expected_df)