Repository: TTitcombe/PrivacyPanda

Issue:
Identify emails
Email addresses should be considered a breach of privacy.

While we could naively assume `.*@.*` is an emaill, this would lead to a lot of false negatives. To begin with, we should identify emails using a whitelist of domains e.g. `gmail, hotmail` and `.co.uk, .com, .org, .edu` etc.

Discussion:

Patch:
diff --git a/privacypanda/__init__.py b/privacypanda/__init__.py
index ab09290..16ccf12 100644
--- a/privacypanda/__init__.py
+++ b/privacypanda/__init__.py
@@ -1,5 +1,6 @@
 from .addresses import *
 from .anonymize import *
+from .email import *
 from .report import *
 
 __version__ = "0.1.0dev"
diff --git a/privacypanda/anonymize.py b/privacypanda/anonymize.py
index 44596a7..7904b9e 100644
--- a/privacypanda/anonymize.py
+++ b/privacypanda/anonymize.py
@@ -5,6 +5,7 @@ import pandas as pd
 from numpy import unique as np_unique
 
 from .addresses import check_addresses
+from .email import check_emails
 
 
 def anonymize(df: pd.DataFrame) -> pd.DataFrame:
@@ -27,7 +28,7 @@ def anonymize(df: pd.DataFrame) -> pd.DataFrame:
     """
     private_cols = []
 
-    checks = [check_addresses]
+    checks = [check_addresses, check_emails]
     for check in checks:
         new_private_cols = check(df)
         private_cols += new_private_cols
diff --git a/privacypanda/email.py b/privacypanda/email.py
new file mode 100644
index 0000000..822f081
--- /dev/null
+++ b/privacypanda/email.py
@@ -0,0 +1,55 @@
+"""
+Code for identifying emails
+"""
+import re
+from typing import List
+
+import pandas as pd
+from numpy import dtype as np_dtype
+
+__all__ = ["check_emails"]
+
+OBJECT_DTYPE = np_dtype("O")
+
+# Whitelisted email suffix.
+# TODO extend this
+WHITELIST_EMAIL_SUFFIXES = [".co.uk", ".com", ".org", ".edu"]
+EMAIL_SUFFIX_REGEX = "[" + r"|".join(WHITELIST_EMAIL_SUFFIXES) + "]"
+
+# Simple email pattern
+SIMPLE_EMAIL_PATTERN = re.compile(".*@.*" + EMAIL_SUFFIX_REGEX, re.I)
+
+
+def check_emails(df: pd.DataFrame) -> List:
+    """
+    Check a dataframe for columns containing emails. Returns a list of column
+    names which contain at least one emails
+
+    "Emails" currently only concerns common emails, with one of the following
+    suffixes: ".co.uk", ".com", ".org", ".edu"
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        The dataframe to check
+
+    Returns
+    -------
+    List
+        The names of columns which contain at least one email
+    """
+    private_cols = []
+
+    for col in df:
+        row = df[col]
+
+        # Only check column if it may contain strings
+        if row.dtype == OBJECT_DTYPE:
+            for item in row:
+                item = str(item)  # convert incase column has mixed data types
+
+                if SIMPLE_EMAIL_PATTERN.match(item):
+                    private_cols.append(col)
+                    break  # 1 failure is enough
+
+    return private_cols
diff --git a/privacypanda/report.py b/privacypanda/report.py
index e8af3a3..7f3b1c7 100644
--- a/privacypanda/report.py
+++ b/privacypanda/report.py
@@ -5,6 +5,7 @@ from collections import defaultdict
 from typing import TYPE_CHECKING, List, Union
 
 from .addresses import check_addresses
+from .email import check_emails
 
 if TYPE_CHECKING:
     import pandas
@@ -57,7 +58,7 @@ class Report:
 def report_privacy(df: "pandas.DataFrame") -> Report:
     report = Report()
 
-    checks = {"address": check_addresses}
+    checks = {"address": check_addresses, "email": check_emails}
 
     for breach, check in checks.items():
         columns = check(df)


Test patch:
diff --git a/tests/test_anonymization.py b/tests/test_anonymization.py
index 916ba93..e0c26ee 100644
--- a/tests/test_anonymization.py
+++ b/tests/test_anonymization.py
@@ -39,6 +39,33 @@ def test_removes_columns_containing_addresses(address):
     pd.testing.assert_frame_equal(actual_df, expected_df)
 
 
+@pytest.mark.parametrize(
+    "email",
+    [
+        "a.test.email@email.com",
+        "a.n.other@testing.co.uk",
+        "a.person@blah.org",
+        "foo@bar.edu",
+    ],
+)
+def test_removes_columns_containing_emails(email):
+    df = pd.DataFrame(
+        {
+            "privateData": ["a", "b", "c", email],
+            "nonPrivateData": ["a", "b", "c", "d"],
+            "nonPrivataData2": [1, 2, 3, 4],
+        }
+    )
+
+    expected_df = pd.DataFrame(
+        {"nonPrivateData": ["a", "b", "c", "d"], "nonPrivataData2": [1, 2, 3, 4]}
+    )
+
+    actual_df = pp.anonymize(df)
+
+    pd.testing.assert_frame_equal(actual_df, expected_df)
+
+
 def test_returns_empty_dataframe_if_all_columns_contain_private_information():
     df = pd.DataFrame(
         {
diff --git a/tests/test_email_identification.py b/tests/test_email_identification.py
new file mode 100644
index 0000000..dd23f7b
--- /dev/null
+++ b/tests/test_email_identification.py
@@ -0,0 +1,48 @@
+"""
+Test functions for identifying emails in dataframes
+"""
+import pandas as pd
+import pytest
+
+import privacypanda as pp
+
+
+@pytest.mark.parametrize(
+    "email",
+    ["test@testing.com", "test@testing.co.uk", "test@testing.org", "test@testing.edu"],
+)
+def test_can_identify_column_whitelisted_suffixes(email):
+    df = pd.DataFrame(
+        {"privateColumn": ["a", email, "c"], "nonPrivateColumn": ["a", "b", "c"]}
+    )
+
+    actual_private_columns = pp.check_emails(df)
+    expected_private_columns = ["privateColumn"]
+
+    assert actual_private_columns == expected_private_columns
+
+
+def test_address_check_returns_empty_list_if_no_emails_found():
+    df = pd.DataFrame(
+        {"nonPrivateColumn1": ["a", "b", "c"], "nonPrivateColumn2": ["a", "b", "c"]}
+    )
+
+    actual_private_columns = pp.check_emails(df)
+    expected_private_columns = []
+
+    assert actual_private_columns == expected_private_columns
+
+
+def test_check_emails_can_handle_mixed_dtype_columns():
+    df = pd.DataFrame(
+        {
+            "privateColumn": [True, "a.name@gmail.com", "c"],
+            "privateColumn2": [1, "b", "j.o.blogg@harvard.edu"],
+            "nonPrivateColumn": [0, True, "test"],
+        }
+    )
+
+    actual_private_columns = pp.check_emails(df)
+    expected_private_columns = ["privateColumn", "privateColumn2"]
+
+    assert actual_private_columns == expected_private_columns
diff --git a/tests/test_report.py b/tests/test_report.py
index ced0754..93a1229 100644
--- a/tests/test_report.py
+++ b/tests/test_report.py
@@ -29,12 +29,42 @@ def test_can_report_addresses():
     assert actual_string == expected_string
 
 
-def test_report_can_accept_multiple_breaches_per_column():
-    report = pp.report.Report()
+def test_can_report_emails():
+    df = pd.DataFrame(
+        {
+            "col1": ["a", "b", "joe.bloggs@something.org"],
+            "col2": [1, 2, 3],
+            "col3": ["t.t.t@t.com", "b", "c"],
+        }
+    )
 
-    report.add_breach("col1", "address")
-    report.add_breach("col1", "phone number")
+    # Check correct breaches have been logged
+    report = pp.report_privacy(df)
+    expected_breaches = {"col1": ["email"], "col3": ["email"]}
+
+    assert report._breaches == expected_breaches
+
+    # Check string report
+    actual_string = str(report)
+    expected_string = "col1: ['email']\ncol3: ['email']\n"
+
+    assert actual_string == expected_string
+
+
+def test_report_can_accept_multiple_breaches_per_column():
+    df = pd.DataFrame(
+        {
+            "col1": ["a", "10 Downing Street", "joe.bloggs@something.org"],
+            "col2": [1, 2, "AB1 1AB"],
+            "col3": ["t.t.t@t.com", "b", "c"],
+        }
+    )
+    report = pp.report_privacy(df)
 
-    expected_breaches = {"col1": ["address", "phone number"]}
+    expected_breaches = {
+        "col1": ["address", "email"],
+        "col2": ["address"],
+        "col3": ["email"],
+    }
 
     assert report._breaches == expected_breaches