improvements

audits/1720034413.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1720034790.csv

@@ -0,0 +1,2 @@
+input_word,original_dictionary_word,new_dictionary_word
+Tea,"Tea, bubble","Tea, hot, leaf, green"

audits/1720034901.csv

@@ -0,0 +1,2 @@
+input_word,original_dictionary_word,new_dictionary_word
+Snack cakes,"Snacks, popcorn, cakes","Snack cake, chocolate"

audits/1720035211.csv

@@ -0,0 +1,16 @@
+input_word,original_dictionary_word,new_dictionary_word
+arrabiata sauce,"Beef and noodles, no sauce",Spaghetti sauce with meat
+Canned Meats,"Meat, NFS","Luncheon meat, pork, canned"
+Chips,Vegetable chips,"Potato chips, NFS"
+graham crumbs,"Cereals ready-to-eat, QUAKER, HONEY GRAHAM OH!S",Graham crackers
+Chedder Popcorn,"Popcorn, NFS","Popcorn, ready-to-eat, cheese flavored"
+Rice Krispies,"Snacks, rice cakes, brown rice, corn","Snacks, KELLOGG, KELLOGG'S RICE KRISPIES TREATS Squares"
+"Beverages, Propel, with electrolytes and sweeteners, fruit-flavored, water","Beverages, Mixed vegetable and fruit juice drink, with added nutrients","Beverages, Water with added vitamins and minerals, bottles, sweetened, assorted fruit flavors"
+Spag. Sause,"Sauce, pasta, spaghetti/marinara, ready-to-serve",Spaghetti sauce
+Baking mix,"Cake or cupcake, white with white icing, bakery",Baked Products
+Cereal,"Cereal, other, plain","Cereal, other, NFS"
+RiceCrisps,"Snacks, rice cakes, brown rice, sesame seed",Rice cake
+Riced Cauliflower,Fried cauliflower,"Cauliflower, cooked, as ingredient"
+Canned Juice,"Carrot juice, canned","Fruit juice, NFS"
+Cliff,"Snacks, granola bars, soft, uncoated, plain",Clif Z bar
+Crackers Bulk,"Cookies, graham crackers, plain or honey (includes cinnamon)","Crackers, wheat"

audits/1720035904.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

chatgpt_audit.py

@@ -10,6 +10,8 @@ from Levenshtein import distance
 from tqdm import tqdm
 from db.db_utils import get_connection
 from ask_gpt import query_gpt
+from utils import generate_embedding, cosine_similarity
+from sentence_transformers import SentenceTransformer, util
 
 # For any unreviewed mappings, we ask chatgpt to consider:
 # 1. The similar_words list
@@ -26,6 +28,15 @@ client = OpenAI(api_key=api_key)
 
 output_file_path = f'./audits/{int(time.time())}.csv'
 
+model_name = 'sentence-transformers/all-mpnet-base-v2'
+model = SentenceTransformer(model_name)
+
+def compare_embeddings(old_dictionary_word, new_dictionary_word):
+    old_embedding = generate_embedding(model, old_dictionary_word)
+    new_embedding = generate_embedding(model, new_dictionary_word)
+    cosine_similarity_score = cosine_similarity(old_embedding, new_embedding)
+    return cosine_similarity_score
+
 def update_csv(results):
     df_results = pd.DataFrame(results, columns=['input_word', 'original_dictionary_word', 'new_dictionary_word',])    
     df_results.to_csv(output_file_path, index=False)
@@ -89,7 +100,7 @@ dictionary = db_cursor.fetchall()
 dictionary = [item[0] for item in dictionary]
 
 # select all mappings that have not been reviewed
-db_cursor.execute("SELECT input_word, dictionary_word, similar_words, is_food FROM mappings WHERE reviewed = false")
+db_cursor.execute("SELECT input_word, dictionary_word, similar_words, is_food FROM mappings WHERE gpt_reviewed = false and is_food = true and dictionary_word != 'Heterogeneous Mixture'")
 results = db_cursor.fetchall()
 
 # iterate through each row, grab the input_word and ask chatgpt to compare it to the dictionary_word
@@ -117,7 +128,7 @@ for row in results:
         new_row = None
         if response == dictionary_word and response in dictionary:
             print(f" - Mapping is correct")
-            db_cursor.execute("UPDATE mappings SET reviewed = true WHERE input_word = %s", (input_word,))
+            db_cursor.execute("UPDATE mappings SET gpt_reviewed = true WHERE input_word = %s", (input_word,))
             db_conn.commit()
         else:
             # We should update the mapping in the database
@@ -125,6 +136,18 @@ for row in results:
             # We should set reviewed to 1
             # first confirm that the response is in the dictionary
             if response in dictionary:
+
+                # If the response is similar to the original dictionary word, lets just skip it
+                compare_score = compare_embeddings(dictionary_word, response)
+                if compare_score > 0.8:
+                    print(f" - Mapping is already similar to the dictionary word")
+                    db_cursor.execute("UPDATE mappings SET gpt_reviewed = true WHERE input_word = %s", (input_word,))
+                    db_conn.commit()
+                    continue
+                else:
+                    print(f" - Mapping is not similar to the dictionary word")
+                    print(f" - Cosine Similarity: {compare_score}")
+
                 # prompt the user if we want to make the change or not
                 print("------")
                 print("Updating mapping to the following:")
@@ -142,9 +165,9 @@ for row in results:
 
                 if confirm.lower() == 'y':
                     if response == 'Non-Food Item':
-                        sql = "UPDATE mappings SET dictionary_word = %s, is_food = FALSE, reviewed = true WHERE input_word = %s"
+                        sql = "UPDATE mappings SET dictionary_word = %s, is_food = FALSE, description = 'Non-Food Item', gpt_reviewed = true, reviewed = true WHERE input_word = %s"
                     else:
-                        sql = "UPDATE mappings SET dictionary_word = %s, reviewed = true, is_food = true WHERE input_word = %s"
+                        sql = "UPDATE mappings SET dictionary_word = %s, gpt_reviewed = true, reviewed = true, is_food = true WHERE input_word = %s"
                     
                     print(f" - Updating mapping with {response}")
                     db_cursor.execute(sql, (response, input_word))
@@ -156,7 +179,7 @@ for row in results:
                     }
                 elif confirm.lower() == 'i':
                     print(f" - Ignoring mapping")
-                    sql = "UPDATE mappings SET ignore = true, reviewed = true WHERE input_word = %s"
+                    sql = "UPDATE mappings SET ignore = true, gpt_reviewed = true, reviewed = true WHERE input_word = %s"
                     db_cursor.execute(sql, (input_word,))
                     db_conn.commit()
                 elif confirm.lower() == 'd':
@@ -166,11 +189,11 @@ for row in results:
                     db_conn.commit()
                 elif confirm.lower() == 'm':
                     print(f" - Heterogeneous Mixture")
-                    sql = "UPDATE mappings SET reviewed = true, dictionary_word = 'Heterogeneous Mixture', is_food = true WHERE input_word = %s"
+                    sql = "UPDATE mappings SET gpt_reviewed = true, reviewed = true, dictionary_word = 'Heterogeneous Mixture', is_food = true WHERE input_word = %s"
                     db_cursor.execute(sql, (input_word,))
                     db_conn.commit()
                 else:
-                    db_cursor.execute("UPDATE mappings SET reviewed = true WHERE input_word = %s", (input_word,))
+                    db_cursor.execute("UPDATE mappings SET gpt_reviewed = true, reviewed = true WHERE input_word = %s", (input_word,))
                     db_conn.commit()
             else:
                 print(f" - Response {response} is not in the dictionary")