Upload standardize-us-states.py

standardize-us-states.py

@@ -0,0 +1,43 @@
+import pandas as pd
+import re
+
+# dictionary of state names to abbreviations
+state_abbreviations = {
+    'Alabama': 'AL', 'Alaska': 'AK', 'Arizona': 'AZ', 'Arkansas': 'AR', 'California': 'CA',
+    'Colorado': 'CO', 'Connecticut': 'CT', 'Delaware': 'DE', 'Florida': 'FL', 'Georgia': 'GA',
+    'Hawaii': 'HI', 'Idaho': 'ID', 'Illinois': 'IL', 'Indiana': 'IN', 'Iowa': 'IA',
+    'Kansas': 'KS', 'Kentucky': 'KY', 'Louisiana': 'LA', 'Maine': 'ME', 'Maryland': 'MD',
+    'Massachusetts': 'MA', 'Michigan': 'MI', 'Minnesota': 'MN', 'Mississippi': 'MS', 'Missouri': 'MO',
+    'Montana': 'MT', 'Nebraska': 'NE', 'Nevada': 'NV', 'New Hampshire': 'NH', 'New Jersey': 'NJ',
+    'New Mexico': 'NM', 'New York': 'NY', 'North Carolina': 'NC', 'North Dakota': 'ND', 'Ohio': 'OH',
+    'Oklahoma': 'OK', 'Oregon': 'OR', 'Pennsylvania': 'PA', 'Rhode Island': 'RI', 'South Carolina': 'SC',
+    'South Dakota': 'SD', 'Tennessee': 'TN', 'Texas': 'TX', 'Utah': 'UT', 'Vermont': 'VT',
+    'Virginia': 'VA', 'Washington DC': 'DC', 'Washington': 'WA', 'West Virginia': 'WV', 'Wisconsin': 'WI', 'Wyoming': 'WY'
+}
+
+df = pd.read_csv('data/2019-climate-all.csv')
+
+# remove duplicates
+df.drop_duplicates(subset=['Username', 'Content'], inplace=True)
+
+def get_state(location):
+    if not isinstance(location, str):
+        return None
+    
+    # check for DC first
+    if re.search(r'\b(Washington DC|DC|D\.C)\b', location, re.IGNORECASE):
+        return 'Washington DC'
+
+    for state, abbrev in state_abbreviations.items():
+        pattern = rf'\b({re.escape(state)}|{re.escape(abbrev)})\b'
+        if re.search(pattern, location, re.IGNORECASE):
+            return state
+        
+    return None
+
+df['Filtered Location'] = df['User Location'].apply(get_state)
+
+# filter rows where 'User Location (State)' is not blank
+filtered_df = df[df['Filtered Location'].notna() & (df['Filtered Location'] != '')]
+
+filtered_df.to_csv('data/2019-climate-usa-redo.csv', index=False)