rollups working

db/db_utils.py

@@ -54,13 +54,8 @@ def initialize_db(conn):
         )
     ''')
     cursor.execute('''
-        CREATE TABLE IF NOT EXISTS donations (
-            run_key TEXT,
-            run_row INTEGER,
-            year TEXT,
-            from TEXT,
-            to TEXT,
-            total_emissions_reduction REAL,
+        CREATE TABLE IF NOT EXISTS donors (
+            name TEXT,
             created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
             updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
         )
@@ -110,7 +105,7 @@ def initialize_db(conn):
         CREATE INDEX IF NOT EXISTS idx_dictionary_word ON mappings(dictionary_word);
         CREATE INDEX IF NOT EXISTS idx_description ON dictionary(description);
         CREATE UNIQUE INDEX IF NOT EXISTS run_row_run_key_uniq ON results(run_key text_ops,run_row int4_ops);
-        CREATE UNIQUE INDEX IF NOT EXISTS run_row_run_key_uniq ON donations(run_key text_ops,run_row int4_ops);
+        CREATE UNIQUE INDEX IF NOT EXISTS rollups_run_key_key ON rollups(run_key text_ops);
     ''')
 
     conn.commit()

rollups.py

@@ -1,4 +1,4 @@
-from db_utils import get_connection
+from db.db_utils import get_connection
 import re
 
 # Function to extract the year
@@ -15,6 +15,7 @@ db_cursor = db_conn.cursor()
 db_cursor.execute("SELECT name from donors")
 donor_results = db_cursor.fetchall()
 list_of_valid_donors = [row[0] for row in donor_results]
+list_of_valid_donors_lower = [donor.lower() for donor in list_of_valid_donors]
 
 
 db_cursor.execute("SELECT distinct(run_key) FROM results")
@@ -30,10 +31,12 @@ for row in result_rows:
     year = extract_year(run_key)
     run_keys_and_years.append((run_key, year))
 
+print(run_keys_and_years)
 
 for run_key, year in run_keys_and_years:
     donations_to = 0
     donations_from = 0
+    print(f"Processing run_key: {run_key} and year: {year}")
 
     # find all run_keys for the same year
     run_keys_in_year = [run_key for run_key, y in run_keys_and_years if y == year]
@@ -44,17 +47,19 @@ for run_key, year in run_keys_and_years:
     if result:
         donations_from = result[0]
     
-    # divide the donations_from by 2 because we attribute the emissions reductions to both the donor and the recipient
-    donations_from = donations_from / 2
-    
     # find all donations to the current donor
-    db_cursor.execute("SELECT sum(total_emissions_reduction) FROM results WHERE run_key = %s donor in %s", (run_key, tuple(list_of_valid_donors)))
+    query = """
+      SELECT sum(total_emissions_reduction) 
+      FROM results 
+      WHERE run_key = %s 
+      AND LOWER(TRIM(donor)) = ANY(%s::text[])
+    """
+
+    db_cursor.execute(query, (run_key, list_of_valid_donors_lower))
     result = db_cursor.fetchone()
     if result:
         donations_to = result[0]
     
-    # divide the donations_to by 2 because we attribute the emissions reductions to both the donor and the recipient
-    donations_to = donations_to / 2
 
     # calculate the total_donations
     donations = donations_to + donations_from
@@ -67,7 +72,22 @@ for run_key, year in run_keys_and_years:
     else:
         total_emissions_reduction_pre = 0
 
+    #  divide the donations_from by 2 because we attribute the emissions reductions to both the donor and the recipient
+    donations = donations / 2
     total_emissions_reduction = total_emissions_reduction_pre - donations
 
     # store this data in the rollups table
-    db_cursor.execute("INSERT INTO rollups (run_key, year, donations_discount, total_emissions_reduction_pre, total_emissions_reduction) VALUES (%s, %s, %s, %s, %s)", (run_key, year, donations, total_emissions_reduction_pre, total_emissions_reduction, ))
+    db_cursor.execute("""
+        INSERT INTO rollups (run_key, year, donations_discount, total_emissions_reduction_pre, total_emissions_reduction)
+        VALUES (%s, %s, %s, %s, %s)
+        ON CONFLICT (run_key)
+        DO UPDATE SET
+            year = EXCLUDED.year,
+            donations_discount = EXCLUDED.donations_discount,
+            total_emissions_reduction_pre = EXCLUDED.total_emissions_reduction_pre,
+            total_emissions_reduction = EXCLUDED.total_emissions_reduction
+    """, (run_key, year, donations_discount, total_emissions_reduction_pre, total_emissions_reduction))
+    db_conn.commit()
+
+db_cursor.close()
+db_conn.close()