upadtes to category mapper

audits/1719415577.csv

@@ -1,2 +1,3 @@
 input_word,original_dictionary_word,new_dictionary_word
 Oatmeal Cereal,"Cereal, oat bunches","Oatmeal, NFS"
+Jars,"Water, bottled, plain",Non-Food Item

category_mapper.py

@@ -14,7 +14,7 @@ from db.db_utils import get_connection
 load_dotenv()
 
 api_key = os.getenv("OPENAI_API_KEY")
-use_openai = False
+use_openai = True
 
 if use_openai:
     client = OpenAI(api_key=api_key)
@@ -30,17 +30,19 @@ file_path = './dictionary/final_corrected_wweia_food_category_complete - final_c
 spreadsheet = pd.read_csv(file_path)
 
 def find_best_category(food_item, category, dataframe):
-    # filtered_df = dataframe
-    filtered_df = dataframe[dataframe['closest_category'] == category]
-    if filtered_df.empty:
-        filtered_df = dataframe
+    filtered_df = dataframe
+    # filtered_df = dataframe[dataframe['closest_category'] == category]
+    # if filtered_df.empty:
+    #     filtered_df = dataframe
 
     descriptions = filtered_df['wweia_food_category_description'].tolist()
 
     prompt = (
-        f"Given the food item '{food_item}' and the category '{category}', choose the most appropriate category from the following options:\n{descriptions}\n\n"
+        f"Given the food item '{food_item}' and the classification of '{category}', choose the most appropriate category from the following options:\n{descriptions}\n\n"
+        f"Only respond with a category from the above. Do not come up with a new category. Do not respond with 'Legumes and Legume Products'.\n\n"
         f"You should respond in json format with an object that has the key `guess`, and the value is the categoy."
     )
+
     if use_openai:
         completion = client.chat.completions.create(
             messages=[
@@ -89,8 +91,8 @@ for row in tqdm(rows, desc="Processing"):
     print(f"Processing '{food_item}'")
 
     # fix the category for Breakfast Cereals
-    if category == 'Breakfast Cereals':
-        category = 'Cereal Grains and Pasta'
+    # if category == 'Breakfast Cereals':
+    #     category = 'Cereal Grains and Pasta'
     # elif category == 'Fast Foods':
     #     # TODO
     # elif category == 'American Indian/Alaska Native Foods':
@@ -108,6 +110,11 @@ for row in tqdm(rows, desc="Processing"):
     print(f"A: '{best_category}'")
 
     if best_category:
+        # ensure that the best_category is in the spreadsheet
+        if best_category not in spreadsheet['wweia_food_category_description'].values:
+            print(f"Error: '{best_category}' not found in the spreadsheet")
+            continue
+
         if use_openai:
             db_cursor.execute('UPDATE dictionary SET wweia_category = %s WHERE fdc_id = %s', (best_category, fdc_id))
         else: