virjinyuh

app.py

@@ -164,7 +164,8 @@ app_ui = ui.page_fluid(
                     },
                 },
             ),
-            ui.input_text("recipient_name", "Filter Recipient (TN Only)", ""),
+            ui.panel_conditional("input.source === 'VA'", ui.input_text("recipient_name", "Filter Recipient (TN Only)", "")),
+            ui.panel_conditional("input.source === 'VA'", ui.input_text("va_report_link", "Contributions URL", "")),
             ui_card(
                 "File Facts:",
                 ui.output_text("total_donors"),
@@ -199,6 +200,8 @@ def server(input, output, session):
 
     @reactive.Calc
     def get_file_name():
+        if input.source() == 'VA':
+            return "va_candidate_donors"
         paths = [file["name"] for file in input.donor_file()]
         tmp_path = paths[0].split(".")[0]
         return tmp_path
@@ -206,16 +209,24 @@ def server(input, output, session):
     @reactive.Calc
     def compile_donors():
         file: list[FileInfo] | None = input.donor_file()
-        if file is None:
+        url = input.va_report_link()
+        if file is None and not url:
             return pd.DataFrame()
-        paths = [file["datapath"] for file in input.donor_file()]
-        files = [DonorFrame(path, input.source(), recip_filter()) for path in paths]
-        print(files[0].recipient_name)
-        try:
-            dataframes = [file.format_donors(export=False) for file in files]
-        except KeyError:
-            raise SilentException()
-        return pd.concat(dataframes, ignore_index=True)
+        if file:
+            paths = [file["datapath"] for file in input.donor_file()]
+            files = [DonorFrame(path, input.source(), recip_filter()) for path in paths]
+            print(files[0].recipient_name)
+            try:
+                dataframes = [file.format_donors(export=False) for file in files]
+            except KeyError:
+                raise SilentException()
+            return pd.concat(dataframes, ignore_index=True)
+        elif url and not file:
+            try:
+                dataframe = DonorFrame(data_path=url, source='VA').format_donors(export=False)
+            except ValueError as e:
+                raise e
+            return dataframe
 
     @output
     @render.data_frame

requirements.txt

@@ -13,4 +13,6 @@ openpyxl
 xlrd
 bs4
 chardet
-pytest
+pytest
+xmltodict
+tqdm

src/add_funcs.py

@@ -1,11 +1,17 @@
+import grequests
+import requests
+from requests import Response
 import re
 import usaddress
+from typing import List, Dict
 import pandas as pd
 from scourgify import normalize_address_record
 import warnings
-warnings.filterwarnings(category=DeprecationWarning, action='ignore')
 from bs4 import BeautifulSoup, SoupStrainer
-from lxml import etree, objectify
+from bs4.element import Tag
+warnings.filterwarnings(category=DeprecationWarning, action='ignore')
+import xmltodict
+from tqdm import tqdm
 ### need to refactor these into a special class
 
 
@@ -58,39 +64,66 @@ def nevada(input_path):
 
     return nv_df
 
+#### virginia parsers
 
-def virginia(input_xml):
-    metadata = input_xml
-    parser = etree.XMLParser(remove_blank_text=True)
-    tree = etree.parse(metadata, parser)
-    root = tree.getroot()
+def get_downloadable_reports(link: str) -> List[str]: 
+    va_base_url = "https://cfreports.elections.virginia.gov/Report/ReportXML/{report_id}"
+    resp = requests.get(link)
+    parsed = BeautifulSoup(resp.content, "html.parser").find("div", {"id": "ScheduledReports"})
+    ids = [extract_va_report_id(i) for i in parsed.find_all("a", {"title": "Click to view report"})]
+    return [va_base_url.format(report_id = rep_id) for rep_id in ids]
 
-    ####
-    for elem in root.getiterator():
-        if not hasattr(elem.tag, "find"):
-            continue  # guard for Comment tags
-        i = elem.tag.find("}")
-        if i >= 0:
-            elem.tag = elem.tag[i + 1 :]
-    objectify.deannotate(root, cleanup_namespaces=True)
-    ####
+def extract_va_report_id(tag: Tag) -> str:
+    return str(tag['href']).split("/")[-1]
 
-    to_parse = etree.tostring(tree, encoding='utf-8', method='xml')
+def get_contributions_from_report(report: Response) -> List[Dict]:
+    content = report.content
     try:
-        df = pd.read_xml(to_parse, xpath="//LiA")
-        df = df.drop("Contributor", axis=1)
-        df_2 = pd.read_xml(to_parse, xpath="//Contributor")
-        df_3 = pd.read_xml(to_parse, xpath="//ScheduleA//LiA//Contributor//Address")
-        final_df = pd.concat(
-            [df, df_2.loc[:, ["FirstName", "LastName", "IsIndividual"]], df_3],
-            axis="columns",
-        )
-        final_df.columns = [col.lower() for col in final_df.columns]
-
-        return final_df
-    except ValueError:
-        raise Exception("No valid ScheduleA donations.")
-
+        report = xmltodict.parse(content)["Report"]
+    except KeyError:
+        report = xmltodict.parse(content)["FinalReport"]
+    if report.get("ScheduleA", None):
+        return report["ScheduleA"]["LiA"]
+    return None
+
+def make_all_requests(report_urls: List[str]) -> list:
+    results = []
+    for idx, i in enumerate(report_urls):
+        print(idx)
+        results.append(requests.get(i))
+    return results
+
+def parse_va_xml(xml_dict: Dict[str,str]) -> Dict:
+    if xml_dict is not None and isinstance(xml_dict, dict):
+        if xml_dict['Contributor']['@IsIndividual'] == 'true':
+            return {
+                'first_name': xml_dict['Contributor'].get("FirstName", None),
+                'last_name': xml_dict['Contributor'].get("LastName", None),
+                'addr1': xml_dict['Contributor']['Address'].get("Line1", None),
+                'addr2': xml_dict['Contributor']['Address'].get("Line2", None),
+                'city': xml_dict['Contributor']['Address'].get("City", None),
+                'state': xml_dict['Contributor']['Address'].get("State", None),
+                'zip': xml_dict['Contributor']['Address'].get("ZipCode", None),
+                'donation_amount': xml_dict.get("Amount", None),
+                'donation_date': xml_dict.get("TransactionDate")
+            }
+    return None
+
+def virginia(report_url: str) -> pd.DataFrame:
+    reports = get_downloadable_reports(report_url)
+    requests = make_all_requests(reports)
+    contributions = []
+    for r in requests:
+        xml_dict = get_contributions_from_report(r)
+        if xml_dict:
+            contributions.extend(xml_dict)
+    parsed_dicts = [parse_va_xml(x) for x in contributions if x]
+    if not parsed_dicts:
+        raise ValueError("No valid records for this candidate.")
+    return pd.DataFrame.from_records([d for d in parsed_dicts if d])
+
+
+### html parsers for Kansas and Missouri
 
 def missouri(input_path):
     import pandas as pd

src/handlers.py

@@ -10,6 +10,7 @@ import chardet
 import re
 from typing import Optional
 from pandas.errors import EmptyDataError
+import validators
 
 class ColumnCoder:
     def __init__(self, input_df: Optional[pd.DataFrame] = pd.DataFrame()):
@@ -81,17 +82,19 @@ class ColumnSelector:
 
 class Ingester:
     def __init__(self, path: str, source):
-        self.path = Path(path)
+        self.path = path if source == 'VA' else Path(path)
         self.encoding = self._get_encoding()
         self.delimiter = self._detect_delimiter()
         self.source = source
 
     @property
     def file_type(self):
-        return self.path.suffix
+        if isinstance(self.path, Path):
+            return self.path.suffix
+        return "url"
 
     def ingest(self) -> pd.DataFrame:
-        if self.file_type == ".xml":
+        if self.file_type == "url":
             return self._parse_xml()
         elif self.file_type in [".csv", ".txt"]:
             return self._parse_csv()
@@ -103,12 +106,14 @@ class Ingester:
             raise OSError(f"File extension {self.file_type} not supported.")
 
     def _get_encoding(self):
+        if self.file_type == 'url':
+            return None
         with open(self.path, "rb") as f:
             to_detect = f.read()
         return chardet.detect(to_detect)["encoding"]
 
     def _detect_delimiter(self) -> str:
-        if self.file_type in ['.xlsx', '.xls']:
+        if self.file_type in ['.xlsx', '.xls', 'url']:
             return None
         if self.file_type == ['.txt']:
             return "\t"

src/tranformers/state_transforms.py

@@ -619,7 +619,7 @@ class TX(StateTransformer):
 
 class VA(StateTransformer):
     def format_data(self, input_df: pd.DataFrame, **kwargs) -> pd.DataFrame:
-        input_df = input_df[input_df["isindividual"] == True]
+        input_df['zip'] = input_df['zip'].str[:5]
         return input_df