some fixes

requirements.txt

@@ -12,4 +12,5 @@ usaddress-scourgify
 openpyxl
 xlrd
 bs4
-chardet
+chardet
+pytest

src/donorframe.py

@@ -174,7 +174,7 @@ class DonorFrame:
             return self.curr_data
 
         logger.info(
-            f"Produced final dataframe with {data_for_preferred.count().iloc[0]} unique donors."
+            f"Produced final dataframe with {data_for_preferred.count().iloc[0]:,} unique donors."
         )
         logger.info(f"Transformation took {round(transform_time, 6)} seconds.")
 

src/handlers.py

@@ -93,12 +93,14 @@ class Ingester:
     def ingest(self) -> pd.DataFrame:
         if self.file_type == ".xml":
             return self._parse_xml()
-        elif self.file_type == ".csv":
+        elif self.file_type in [".csv", ".txt"]:
             return self._parse_csv()
         elif self.file_type == ".html":
             return self._parse_html()
         elif self.file_type in [".xls", ".xlsx"]:
             return self._parse_excel()
+        else:
+            raise OSError(f"File extension {self.file_type} not supported.")
 
     def _get_encoding(self):
         with open(self.path, "rb") as f:
@@ -109,33 +111,30 @@ class Ingester:
         sniffer = Sniffer()
         with open(self.path, newline="") as csvfile:
             to_detect = csvfile.read(1024)
-        return sniffer.sniff(to_detect)
+        return sniffer.sniff(to_detect).delimiter
 
     def _detect_row_skip(self) -> int:
         rs = 0
-        while len(pd.read_csv(self.path, skiprows=rs, nrows=1).columns) < 4:
+        while len(pd.read_csv(self.path, sep=self.delimiter, skiprows=rs, nrows=1).columns) < 4:
             rs += 1
         return rs
 
-    def _parse_csv(self):
-        try:
-            tmp_file = pd.read_csv(
-                self.path,
-                skiprows=self._detect_row_skip(),
-                index_col=False,
-                skip_blank_lines=True,
-                skipinitialspace=True,
-                escapechar="\\",
-            )
-        except UnicodeDecodeError:
-            tmp_file = pd.read_csv(
-                self.path,
-                skiprows=1,
-                index_col=False,
-                encoding_errors="ignore",
-            )
-        except ParserError:
-            tmp_file = pd.read_csv(
+    def get_base_csv_params(self) -> dict:
+        return {
+            "filepath_or_buffer": self.path,
+            "sep": self.delimiter, 
+            "skiprows": self._detect_row_skip(),
+            "index_col": False,
+            "skip_blank_lines": True,
+            "skipinitialspace": True,
+            "escapechar": "\\",
+        }
+
+    def _parse_csv_encoding_errors(self):
+        return pd.read_csv(**self.get_base_csv_params(), encoding_errors="ignore")
+    
+    def _parse_csv_parser_error(self):
+        return pd.read_csv(
                 self.path,
                 index_col=False,
                 encoding="unicode_escape",
@@ -145,6 +144,14 @@ class Ingester:
                 quotechar='"',
                 on_bad_lines="skip",
             )
+
+    def _parse_csv(self):
+        try:
+            tmp_file = pd.read_csv(**self.get_base_csv_params())
+        except UnicodeDecodeError:
+            tmp_file = self._parse_csv_encoding_errors()
+        except ParserError:
+            tmp_file = self._parse_csv_parser_error()
         return tmp_file
 
     def _parse_excel(self):
@@ -252,11 +259,11 @@ class Polisher:
                 input_df[col] = (
                     input_df[col].astype(str).map(lambda x: x.lower().strip())
                 )
+                input_df[col] = input_df[col].str.replace(r"[^a-z0-9 ]", "", regex=True)
             elif col == "state":
                 input_df[col] = (
                     input_df[col].astype(str).map(lambda x: x.upper().strip())
                 )
-
         return input_df
 
     def _dedupe(self, input_df: pd.DataFrame) -> pd.DataFrame:

src/tranformers/state_transforms.py

@@ -305,6 +305,8 @@ class LA_C(StateTransformer):
 
 class MA(StateTransformer):
     def format_data(self, input_df: pd.DataFrame, **kwargs) -> str:
+        #### temporary fix for column issue
+        input_df.rename(columns={'full_name': 'last_name'}, inplace=True)
         input_df["donation_amount"] = input_df["donation_amount"].replace(
             r"$|,", "", regex=True
         )

tests/donorframe_test.py

@@ -0,0 +1,5 @@
+from src.donorframe import DonorFrame
+import pytest
+
+def test():
+    pass