Repository: Standard-Labs/real-intent
Issue category: Performance Issue

Issue:
Email validation may be slow
Have a hunch that email validation with MillionVerifier is very slow. 

Use threads to concurrently process emails and phones (check rate limits on Numverify and MillionVerifier).


Hints:

Patch:
diff --git a/bigdbm/validate/email.py b/bigdbm/validate/email.py
index 1afb4a7..417e45a 100644
--- a/bigdbm/validate/email.py
+++ b/bigdbm/validate/email.py
@@ -1,6 +1,8 @@
 """Validate emails using MillionVerifier."""
 import requests
 
+from concurrent.futures import ThreadPoolExecutor
+
 from bigdbm.schemas import MD5WithPII
 from bigdbm.validate.base import BaseValidator
 
@@ -10,9 +12,10 @@ class EmailValidator(BaseValidator):
     Remove emails determined to not be 'good' by MillionVerifier.
     """
 
-    def __init__(self, million_key: str) -> None:
+    def __init__(self, million_key: str, max_threads: int = 10) -> None:
         """Initialize with MillionVerifier key."""
         self.api_key: str = million_key
+        self.max_threads: int = max_threads
 
     def _validate_email(self, email: str) -> bool:
         """Validate an email with MillionVerifier."""
@@ -35,9 +38,26 @@ def _validate_email(self, email: str) -> bool:
 
     def validate(self, md5s: list[MD5WithPII]) -> list[MD5WithPII]:
         """Remove any emails that are not 'good'."""
+        # Extract all the emails
+        all_emails: list[str] = []
+        for md5 in md5s:
+            all_emails.extend(md5.pii.emails)
+
+        # Validate all the emails
+        with ThreadPoolExecutor(max_workers=self.max_threads) as executor:
+            valid_emails_idx: list[bool] = list(
+                executor.map(self._validate_email, all_emails)
+            )
+
+        # Extract valid emails
+        valid_emails: list[str] = [
+            email for email, is_valid in zip(all_emails, valid_emails_idx) if is_valid
+        ]
+
+        # Remove invalid emails from MD5s
         for md5 in md5s:
             md5.pii.emails = [
-                email for email in md5.pii.emails if self._validate_email(email)
+                email for email in md5.pii.emails if email in valid_emails
             ]
 
         return md5s
diff --git a/bigdbm/validate/phone.py b/bigdbm/validate/phone.py
index 181b6f0..67b960f 100644
--- a/bigdbm/validate/phone.py
+++ b/bigdbm/validate/phone.py
@@ -1,6 +1,8 @@
 """Validate phone numbers using numverify."""
 import requests
 
+from concurrent.futures import ThreadPoolExecutor
+
 from bigdbm.schemas import MD5WithPII
 from bigdbm.validate.base import BaseValidator
 
@@ -10,9 +12,10 @@ class PhoneValidator(BaseValidator):
     Remove US phone numbers determined to not be 'valid' by Numverify.
     """
 
-    def __init__(self, numverify_key: str) -> None:
+    def __init__(self, numverify_key: str, max_threads: int = 10) -> None:
         """Initialize with numverify key."""
         self.api_key: str = numverify_key
+        self.max_threads: int = max_threads
 
     def _validate_phone(self, phone: str) -> bool:
         """Validate a US phone number with numverify."""
@@ -45,9 +48,26 @@ def _validate_phone(self, phone: str) -> bool:
 
     def validate(self, md5s: list[MD5WithPII]) -> list[MD5WithPII]:
         """Remove any phone numbers that are not 'good'."""
+        # Extract all the phone numbers
+        all_phones: list[str] = []
+        for md5 in md5s:
+            all_phones.extend([phone.phone for phone in md5.pii.mobile_phones])
+
+        # Validate all the phone numbers
+        with ThreadPoolExecutor(max_workers=self.max_threads) as executor:
+            valid_phones_idx: list[bool] = list(
+                executor.map(self._validate_phone, all_phones)
+            )
+
+        # Extract valid phone numbers
+        valid_phones: list[str] = [
+            phone for phone, is_valid in zip(all_phones, valid_phones_idx) if is_valid
+        ]
+
+        # Remove invalid phone numbers from MD5s
         for md5 in md5s:
             md5.pii.mobile_phones = [
-                phone for phone in md5.pii.mobile_phones if self._validate_phone(phone.phone)
+                phone for phone in md5.pii.mobile_phones if phone.phone in valid_phones
             ]
 
         return md5s


Test patch: