skip anything non-csv

algo.py

@@ -13,7 +13,7 @@ from ask_gpt import query_gpt
 from multi_food_item_detector import extract_items, has_delimiters
 from mapping_template import empty_template
 
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.WARNING, format='%(asctime)s - %(levelname)s - %(message)s')
 similarity_threshold = 0.75
 
 

audits/1719530507.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719530709.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719531384.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719531486.csv

@@ -0,0 +1,17 @@
+input_word,original_dictionary_word,new_dictionary_word
+Eyeglasses,"Candies, REESE'S Peanut Butter Cups",Non-Food Item
+Mini Crackers,"Pretzels, NFS","Crackers, NFS"
+Pool Shock,"Sports drink, NFS",Non-Food Item
+Office Paper,Rice paper,Non-Food Item
+Games,"Sports drink, NFS",Non-Food Item
+Comforter,Pig in a blanket,Non-Food Item
+Mulch,Barley,Non-Food Item
+Yoga Mats,"Drumstick pods, raw",Non-Food Item
+Floor Mats,"Drumstick pods, raw",Non-Food Item
+Knives Cultlery Packs,"Snacks, fruit leather, pieces",Non-Food Item
+Bar Stools,"Cereal or granola bar, with coconut, chocolate coated",Non-Food Item
+Grill Cover,"Chicken fillet, grilled",Non-Food Item
+Pet Products,"Hot dog, meat and poultry",Non-Food Item
+Needles,Cough drops,Non-Food Item
+Cart Covers,"Tomatoes, raw",Non-Food Item
+Ear Buds,"Cereal, ready-to-eat, NFS",Non-Food Item

audits/1719531575.csv

@@ -0,0 +1,2 @@
+input_word,original_dictionary_word,new_dictionary_word
+Assorted Deli Sandwiches Frozen,"Ham, sliced, restaurant",Mixed Food Items

audits/1719531601.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719531628.csv

@@ -0,0 +1,2 @@
+input_word,original_dictionary_word,new_dictionary_word
+All-purpose cleaner,Screwdriver,Non-Food Item

audits/1719531665.csv

@@ -0,0 +1,19 @@
+input_word,original_dictionary_word,new_dictionary_word
+door locks,Miscellaneous Items,Non-Food Item
+Asst. Vgetables,"Vegetables, mixed, frozen, cooked, boiled, drained, with salt",Assorted Vegetables
+Choc. Candy,Dark chocolate candy,Chocolate candy
+Corn chowder,Potato chowder,"Corn, creamed"
+Frozen bagels,"Croissants, cheese",Bagel
+misc. boxes,"Cereal, chocolate crispy",Non-Food Item
+purex,"Water, bottled, generic",Non-Food Item
+Propel Lemon,"Beverages, Water with added vitamins and minerals, bottles, sweetened, assorted fruit flavors","Beverages, Propel Zero, fruit-flavored, non-carbonated"
+Honey Buns,"Rolls, hamburger or hotdog, plain","Cinnamon buns, frosted (includes honey buns)"
+Powerade Lemonade,"Lemon juice, raw",Sports drink (Powerade)
+Non-Food Office,Assorted Edible Items,Non-Food Item
+Landscaping,Mixed Food Items,Non-Food Item
+Fresh Pineapple,"Pineapple, dried","Pineapple, raw"
+White Cheddar Popcorn,"Popcorn, NFS","Popcorn, ready-to-eat, cheese flavored"
+Signs,Mixed Food Items,Non-Food Item
+Mixed Produce Box,"Vegetables, mixed, canned, drained solids",Mixed Produce
+Vegetablesl,"Classic mixed vegetables, canned, cooked with oil",Mixed Vegetables
+Mixed Food Canned,Non-Food Item,Mixed Food Items

audits/1719531862.csv

@@ -0,0 +1,10 @@
+input_word,original_dictionary_word,new_dictionary_word
+misc CVS items,"Cholesterol, Cheese, swiss, slices (CA2, NC) - 18c-16-03-Chol",Misc Items
+"Paper, standard, letter size",Rice paper,Non-Food Item
+nacho dip,Chocolate dip,Layer dip
+Syrup Assorted Chololate,"Syrup, NFS",Chocolate syrup
+Tools,Screwdriver,Non-Food Item
+Bottle Caps,"Water, bottled, plain",Non-Food Item
+Can Beans,"Green beans, canned, cooked with oil",Baked beans
+Beef Sticks,"Snacks, popcorn, cakes",Beef jerky
+Peanut butter packets,"Crackers, sandwich, peanut butter filled",Peanut butter

audits/1719532037.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719532097.csv

@@ -0,0 +1,11 @@
+input_word,original_dictionary_word,new_dictionary_word
+LED Bulbs,"Fennel bulb, raw",Non-Food Item
+Powerstrips,"BURGER KING, Chicken Strips",Non-Food Item
+S/P Shakers,"Candies, REESE'S Peanut Butter Cups",Non-Food Item
+Twix Choc Milk,"Milk, NFS","Chocolate milk, NFS"
+vegetales,"Vegetables, mixed, canned, drained solids",Misc Vegetables
+powerade powder,Sports drink (Gatorade G),Sports drink (Powerade)
+Broth & Salsa,Mixed Food Items,Salsa
+Vegetables and Lettuce,Assorted Fruit and Veg,Assorted Vegetables
+King Hawaiian Bread,Ham and cheese loaf or roll,"Rolls, dinner, wheat"
+Fresh Cranberries,"Cranberries, dried","Cranberries, raw"

audits/1719532313.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719532660.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719532743.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719532810.csv

@@ -0,0 +1,2 @@
+input_word,original_dictionary_word,new_dictionary_word
+cheese and alfredo sauce,"Ravioli, cheese-filled, with tomato sauce",Alfredo sauce

audits/1719532915.csv

@@ -0,0 +1,2 @@
+input_word,original_dictionary_word,new_dictionary_word
+cleaning & paper,Screwdriver,Non-Food Item

audits/1719532985.csv

@@ -0,0 +1,3 @@
+input_word,original_dictionary_word,new_dictionary_word
+Condiments and Paper,"Salad dressing, mayonnaise, regular",Non-Food Item
+Pet & Non-Food Items,"Hot dog, meat and poultry",Non-Food Item

audits/1719533030.csv

@@ -0,0 +1,2 @@
+input_word,original_dictionary_word,new_dictionary_word
+Assorted Quiche & Pot Pies,"Cheese quiche, meatless","Pot pie, beef"

audits/1719533703.csv

@@ -0,0 +1,2 @@
+input_word,original_dictionary_word,new_dictionary_word
+Shampoo & Conditioner Packets,Mixed Food Items,Non-Food Item

audits/1719533786.csv

@@ -0,0 +1,3 @@
+input_word,original_dictionary_word,new_dictionary_word
+Biscuits & Fruit,Mixed Food Items,Biscuit with fruit
+sour & tartar sauce,Mixed Food Items,Tartar sauce

audits/1719533844.csv

@@ -0,0 +1,3 @@
+input_word,original_dictionary_word,new_dictionary_word
+baking & toppings,"Bread, white, commercially prepared",Mixed Food Items
+snacks & canned vegs.,"Crackers, saltines (includes oyster, soda, soup)",Mixed Food Items

audits/1719533924.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

audits/1719534233.csv

@@ -0,0 +1 @@
+input_word,original_dictionary_word,new_dictionary_word

chatgpt_audit.py

@@ -140,13 +140,13 @@ for row in results:
                     print(" - Mapping is already marked as a Non-Food Item")
                     confirm = 'y'
                 else:
-                    confirm = input("Press 'y' to confirm the update, 'i' to ignore the item, any other key to skip: ")
+                    confirm = input("Press 'y' to confirm, 'i' to ignore, 'd' to delete, 'm' for mixture, any other key to skip: ")
 
                 if confirm.lower() == 'y':
                     if response == 'Non-Food Item':
                         sql = "UPDATE mappings SET dictionary_word = %s, is_food = FALSE, reviewed = true WHERE input_word = %s"
                     else:
-                        sql = "UPDATE mappings SET dictionary_word = %s, reviewed = true WHERE input_word = %s"
+                        sql = "UPDATE mappings SET dictionary_word = %s, reviewed = true, is_food = true WHERE input_word = %s"
                     
                     print(f" - Updating mapping with {response}")
                     db_cursor.execute(sql, (response, input_word))
@@ -158,8 +158,18 @@ for row in results:
                     }
                 elif confirm.lower() == 'i':
                     print(f" - Ignoring mapping")
-                    sql = "UPDATE mappings SET ignored = true, reviewed = true WHERE input_word = %s"
-                    db_cursor.execute(sql, (response, input_word))
+                    sql = "UPDATE mappings SET ignore = true, reviewed = true WHERE input_word = %s"
+                    db_cursor.execute(sql, (input_word,))
+                    db_conn.commit()
+                elif confirm.lower() == 'd':
+                    print(f" - Deleting mapping")
+                    sql = "DELETE FROM mappings WHERE input_word = %s"
+                    db_cursor.execute(sql, (input_word,))
+                    db_conn.commit()
+                elif confirm.lower() == 'm':
+                    print(f" - Mixed food items")
+                    sql = "UPDATE mappings SET reviewed = true, dictionary_word = 'Mixed Food Items', is_food = true WHERE input_word = %s"
+                    db_cursor.execute(sql, (input_word,))
                     db_conn.commit()
                 else:
                     db_cursor.execute("UPDATE mappings SET reviewed = true WHERE input_word = %s", (input_word,))

food_nonfood.py

@@ -3,7 +3,7 @@ import numpy as np
 import torch
 import logging
 from transformers import pipeline
-
+from autocorrect import Speller
 # Load a pre-trained SBERT model
 
 # Set seeds for reproducibility of zero-shot classification
@@ -17,8 +17,11 @@ def set_seed(seed):
 
 set_seed(1)
 
+
+
 # Load a pre-trained model and tokenizer
 classifier = pipeline("zero-shot-classification", model="roberta-large-mnli")
+spell = Speller()
 
 # Classify item as food or non-food
 def classify_as_food_nonfood(item):
@@ -40,6 +43,16 @@ def classify_as_food_nonfood(item):
             label = "food"
             score = drink_score
 
+        # try correcting the spelling
+        if label == "non-food":
+            spell_fix_item = spell(cleaned_item)
+            result = classifier(cleaned_item, candidate_labels=["food", "non-food"])
+            food_label = result["labels"][0]
+            food_score = result["scores"][0]
+            if food_label == "food" and food_score >= 0.7:
+                label = "food"
+                score = food_score
+
     # logging.info(f"Item: {item}, Label: {label}, Score: {score}")
     return label, score
 

requirements.txt

@@ -1,4 +1,5 @@
 --extra-index-url https://download.pytorch.org/whl/cu113
+autocorrect==2.6.1
 gradio==4.36.1
 mistralai==0.4.0
 numpy==1.26.4

run.py

@@ -14,8 +14,11 @@ if __name__ == "__main__":
 
     # get all files in the raw folder and iterate through them
     raw_files = os.listdir('./raw')
-    for raw_file_name in raw_files:
     # for raw_file_name in ['test.csv']:
+    for raw_file_name in raw_files:
+        if not raw_file_name.endswith('.csv'):
+            continue
+
         # chop off the extension for the results run key
         # result_file_name = raw_file_name.split('.')[0]
         # run_key = f"{result_file_name}-{int(time.time())}"

similarity_fast.py

@@ -3,7 +3,6 @@ import re
 import pickle
 import pandas as pd
 import requests
-from tqdm import tqdm
 from sentence_transformers import SentenceTransformer, util
 from db.db_utils import get_connection, initialize_db, store_mapping_to_db
 from food_nonfood import classify_as_food_nonfood