Fixed transactions rag jupyter notebook errors

app/categorization/categorizer.py

@@ -3,6 +3,7 @@ import re
 import ast
 import json
 import logging
+import os
 from typing import Any, List, Tuple, Optional, Dict, Union
 
 # Third-party library imports
@@ -14,12 +15,12 @@ from tenacity import retry, wait_random_exponential, stop_after_attempt
 from pydantic import ValidationError
 
 # Local application/library specific imports
-from langchain.chat_models import ChatOpenAI
+from langchain_openai import ChatOpenAI
 from langchain.chains import LLMChain
 from langchain.output_parsers import PydanticOutputParser, OutputFixingParser
 from langchain.prompts import PromptTemplate
-import template as template
-from config import CATEGORY_REFERENCE_OUTPUT_FILE, TX_PER_LLM_RUN
+import app.categorization.template as CATEGORY_TEMPLATE
+from app.categorization.config import CATEGORY_REFERENCE_OUTPUT_FILE, TX_PER_LLM_RUN
 
 
 def fuzzy_match_list_categorizer(
@@ -68,12 +69,14 @@ async def llm_list_categorizer(tx_list: pd.DataFrame) -> pd.DataFrame:
     """
 
     # Initialize language model and prompt
-    llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0125")
-    prompt = PromptTemplate.from_template(template=template.CATEGORY_TEMPLATE)    
+    # openai_api_key = os.environ['OPENAI_API_KEY']
+    # print("apikey: " + openai_api_key)
+    llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0125", api_key="sk-codepath-group-project-3WwlG0qG8GDG7SMVjgFLT3BlbkFJOHIlVsq0UXFqaOV7hl3O")
+    prompt = PromptTemplate.from_template(template=CATEGORY_TEMPLATE)    
     chain = LLMChain(llm=llm, prompt=prompt)
 
     # Iterate over the DataFrame in batches of TX_PER_LLM_RUN transactions
-    tasks = [llm_sublist_categorizer(tx_list.attrs['file_name'], chain=chain, tx_descriptions="\n".join(chunk['description']).strip())
+    tasks = [llm_sublist_categorizer(tx_list.attrs['file_name'], chain=chain, tx_descriptions="\n".join(chunk['name/description']).strip())
              for chunk in np.array_split(tx_list, tx_list.shape[0] // TX_PER_LLM_RUN + 1)]
 
     # Gather results and extract (valid) outputs
@@ -87,7 +90,7 @@ async def llm_list_categorizer(tx_list: pd.DataFrame) -> pd.DataFrame:
     valid_outputs = [output for valid_result in valid_results for output in valid_result]
 
     # Return a DataFrame with the valid outputs
-    return pd.DataFrame(valid_outputs, columns=['description', 'category'])
+    return pd.DataFrame(valid_outputs, columns=['name/description', 'category'])
 
 
 @retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6))

app/categorization/categorizer_list.py

@@ -6,8 +6,8 @@ from datetime import datetime
 import pandas as pd
 
 # Local application/library specific imports
-from config import CATEGORY_REFERENCE_OUTPUT_FILE
-from categorizer import llm_list_categorizer, fuzzy_match_list_categorizer
+from app.categorization.config import CATEGORY_REFERENCE_OUTPUT_FILE
+from app.categorization.categorizer import llm_list_categorizer, fuzzy_match_list_categorizer
 
 
 async def categorize_list(tx_list: pd.DataFrame) -> pd.DataFrame:
@@ -29,14 +29,14 @@ async def categorize_list(tx_list: pd.DataFrame) -> pd.DataFrame:
     if os.path.exists(CATEGORY_REFERENCE_OUTPUT_FILE):
         # Read description-category pairs from the reference file
         description_category_pairs = pd.read_csv(
-            CATEGORY_REFERENCE_OUTPUT_FILE, header=None, names=['description', 'category']
+            CATEGORY_REFERENCE_OUTPUT_FILE, header=None, names=['name/description', 'category']
         )
 
         # Extract only descriptions for faster matching
-        descriptions = description_category_pairs['description'].values
+        descriptions = description_category_pairs['name/description'].values
 
         # Use fuzzy matching to find similar descriptions and assign the category
-        tx_list['category'] = tx_list['description'].apply(
+        tx_list['category'] = tx_list['name/description'].apply(
             fuzzy_match_list_categorizer,
             args=(descriptions, description_category_pairs),
         )
@@ -44,14 +44,14 @@ async def categorize_list(tx_list: pd.DataFrame) -> pd.DataFrame:
     # Filter out uncategorized transactions, deduplicate, and sort by description
     uncategorized_descriptions = (
         tx_list[tx_list['category'].isnull()]
-        .drop_duplicates(subset=['description'])
-        .sort_values(by=['description'])
+        .drop_duplicates(subset=['name/description'])
+        .sort_values(by=['name/description'])
     )
 
     # Ask the language model to categorize the remaining descriptions
     if not uncategorized_descriptions.empty:
         categorized_descriptions = await llm_list_categorizer(
-            uncategorized_descriptions[['description', 'category']]
+            uncategorized_descriptions[['name/description', 'category']]
         )
 
         categorized_descriptions.dropna(inplace=True)
@@ -59,8 +59,8 @@ async def categorize_list(tx_list: pd.DataFrame) -> pd.DataFrame:
         # Update the category for uncategorized transactions based on the language model results
         if not categorized_descriptions.empty:
             tx_list['category'] = tx_list['category'].fillna(
-                tx_list['description'].map(
-                    categorized_descriptions.set_index('description')['category']
+                tx_list['name/description'].map(
+                    categorized_descriptions.set_index('name/description')['category']
                 )
             )
         

app/categorization/file_processing.py

@@ -9,8 +9,8 @@ from datetime import datetime
 import pandas as pd
 from dateparser import parse
 
-from categorizer_list import categorize_list
-from config import RESULT_OUTPUT_FILE, CATEGORY_REFERENCE_OUTPUT_FILE
+from app.categorization.categorizer_list import categorize_list
+from app.categorization.config import RESULT_OUTPUT_FILE, CATEGORY_REFERENCE_OUTPUT_FILE
 
 # Read file and process it (e.g. categorize transactions)
 async def process_file(file_path: str) -> Dict[str, Union[str, pd.DataFrame]]:

app/transactions_rag/categorize_transactions.ipynb

@@ -134,40 +134,411 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!python3 -m venv venv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!source venv/bin/activate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Requirement already satisfied: setuptools in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (70.0.0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip3 install --upgrade setuptools"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Defaulting to user installation because normal site-packages is not writeable',\n",
+       " 'Requirement already satisfied: langchain in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 1)) (0.2.1)',\n",
+       " 'Collecting langchain_openai (from -r requirements.txt (line 2))',\n",
+       " '  Downloading langchain_openai-0.1.7-py3-none-any.whl.metadata (2.5 kB)',\n",
+       " 'Collecting python-dotenv (from -r requirements.txt (line 3))',\n",
+       " '  Using cached python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)',\n",
+       " 'Collecting openai (from -r requirements.txt (line 4))',\n",
+       " '  Using cached openai-1.30.4-py3-none-any.whl.metadata (21 kB)',\n",
+       " 'Requirement already satisfied: tenacity in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 5)) (8.3.0)',\n",
+       " 'Requirement already satisfied: rapidfuzz in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 6)) (3.9.1)',\n",
+       " 'Requirement already satisfied: pydantic in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 7)) (2.7.1)',\n",
+       " 'Requirement already satisfied: dateparser in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 8)) (1.2.0)',\n",
+       " 'Requirement already satisfied: pandas in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 9)) (2.0.3)',\n",
+       " 'Collecting path (from -r requirements.txt (line 10))',\n",
+       " '  Using cached path-16.14.0-py3-none-any.whl.metadata (6.3 kB)',\n",
+       " 'Collecting aiostream==0.5.2 (from -r requirements.txt (line 12))',\n",
+       " '  Using cached aiostream-0.5.2-py3-none-any.whl.metadata (9.9 kB)',\n",
+       " 'Collecting cachetools==5.3.3 (from -r requirements.txt (line 13))',\n",
+       " '  Using cached cachetools-5.3.3-py3-none-any.whl.metadata (5.3 kB)',\n",
+       " 'Collecting docx2txt==0.8 (from -r requirements.txt (line 14))',\n",
+       " '  Using cached docx2txt-0.8.tar.gz (2.8 kB)',\n",
+       " '  Preparing metadata (setup.py): started',\n",
+       " \"  Preparing metadata (setup.py): finished with status 'done'\",\n",
+       " 'Collecting fastapi==0.109.1 (from -r requirements.txt (line 15))',\n",
+       " '  Using cached fastapi-0.109.1-py3-none-any.whl.metadata (25 kB)',\n",
+       " 'Collecting llama-index-agent-openai==0.2.2 (from -r requirements.txt (line 16))',\n",
+       " '  Using cached llama_index_agent_openai-0.2.2-py3-none-any.whl.metadata (677 bytes)',\n",
+       " 'Collecting llama-index-core==0.10.28 (from -r requirements.txt (line 17))',\n",
+       " '  Using cached llama_index_core-0.10.28-py3-none-any.whl.metadata (3.6 kB)',\n",
+       " 'Collecting llama-index-vector-stores-pinecone==0.1.3 (from -r requirements.txt (line 18))',\n",
+       " '  Using cached llama_index_vector_stores_pinecone-0.1.3-py3-none-any.whl.metadata (725 bytes)',\n",
+       " 'Collecting llama-index==0.10.28 (from -r requirements.txt (line 19))',\n",
+       " '  Using cached llama_index-0.10.28-py3-none-any.whl.metadata (11 kB)',\n",
+       " 'Collecting python-dotenv (from -r requirements.txt (line 3))',\n",
+       " '  Using cached python_dotenv-1.0.0-py3-none-any.whl.metadata (21 kB)',\n",
+       " '\\x1b[31mERROR: Ignored the following versions that require a different python version: 0.0.10 Requires-Python >=3.9,<4.0; 0.0.11 Requires-Python >=3.9,<4.0; 0.0.12 Requires-Python >=3.9,<4.0; 0.0.13 Requires-Python >=3.9,<4.0; 0.0.14 Requires-Python >=3.9,<4.0; 0.0.15 Requires-Python >=3.9,<4.0; 0.0.16 Requires-Python >=3.9,<4.0; 0.0.17 Requires-Python >=3.9,<4.0; 0.0.18 Requires-Python >=3.9,<4.0; 0.0.19 Requires-Python >=3.9,<4.0; 0.0.20 Requires-Python >=3.9,<4.0; 0.0.21 Requires-Python >=3.9,<4.0; 0.0.22 Requires-Python >=3.9,<4.0; 0.0.23 Requires-Python >=3.9,<4.0; 0.0.24 Requires-Python >=3.9,<4.0; 0.0.25 Requires-Python >=3.9,<4.0; 0.14.3 Requires-Python >=3.9,<4; 0.14.4 Requires-Python <4,>=3.9; 0.14.5 Requires-Python <4,>=3.9; 0.15.0 Requires-Python <4,>=3.9; 0.15.1 Requires-Python <4,>=3.9; 0.15.10 Requires-Python <4,>=3.9; 0.15.11 Requires-Python <4,>=3.9; 0.15.12 Requires-Python <4,>=3.9; 0.15.13 Requires-Python <4,>=3.9; 0.15.2 Requires-Python <4,>=3.9; 0.15.3 Requires-Python <4,>=3.9; 0.15.4 Requires-Python <4,>=3.9; 0.15.5 Requires-Python <4,>=3.9; 0.15.6 Requires-Python <4,>=3.9; 0.15.7 Requires-Python <4,>=3.9; 0.15.8 Requires-Python <4,>=3.9; 0.15.9 Requires-Python <4,>=3.9; 0.16.0 Requires-Python <4,>=3.9; 0.16.1 Requires-Python <4,>=3.9; 0.16.2 Requires-Python <4,>=3.9; 0.16.3 Requires-Python <4,>=3.9; 0.16.4 Requires-Python <4,>=3.9; 0.16.5 Requires-Python <4,>=3.9; 0.16.6 Requires-Python <4,>=3.9; 0.16.7 Requires-Python <4,>=3.9; 0.16.8 Requires-Python <4,>=3.9; 0.16.9 Requires-Python <4,>=3.9; 0.17.0 Requires-Python <4,>=3.9; 0.17.1 Requires-Python <4,>=3.9; 0.17.2 Requires-Python <4,>=3.9; 0.17.3 Requires-Python <4,>=3.9; 0.17.4 Requires-Python <4,>=3.9; 0.17.5 Requires-Python <4,>=3.9; 0.17.6 Requires-Python <4,>=3.9; 0.17.7 Requires-Python <4,>=3.9; 0.18.0 Requires-Python <4,>=3.9; 0.18.1 Requires-Python <4,>=3.9; 0.18.2 Requires-Python <4,>=3.9; 0.19.0 Requires-Python <4,>=3.9; 0.20.0 Requires-Python <4,>=3.9; 0.21.0 Requires-Python <4,>=3.9\\x1b[0m\\x1b[31m',\n",
+       " '\\x1b[0m\\x1b[31mERROR: Could not find a version that satisfies the requirement traceloop-sdk==0.15.11 (from versions: 0.0.26, 0.0.27b0, 0.0.27, 0.0.28, 0.0.29, 0.0.31, 0.0.32, 0.0.33, 0.0.34, 0.0.35, 0.0.36, 0.0.37, 0.0.38, 0.0.39, 0.0.40, 0.0.41, 0.0.42, 0.0.43b0, 0.0.43b2, 0.0.43b3, 0.0.43b4, 0.0.43, 0.0.44b0, 0.0.44, 0.0.46, 0.0.47, 0.0.48, 0.0.49, 0.0.50, 0.0.51, 0.0.52, 0.0.53, 0.0.54, 0.0.55, 0.0.56a0, 0.0.56a1, 0.0.56a2, 0.0.56a3, 0.0.56a4, 0.0.56a5, 0.0.56a6, 0.0.56, 0.0.57, 0.0.58, 0.0.59, 0.0.60a0, 0.0.60a1, 0.0.60, 0.0.61a0, 0.0.61, 0.0.62, 0.0.63, 0.0.64a0, 0.0.64, 0.0.65, 0.0.66, 0.0.67, 0.0.68, 0.0.69, 0.0.70, 0.1.3, 0.1.5, 0.1.6, 0.1.7, 0.1.8, 0.1.9, 0.1.10, 0.1.11, 0.1.12, 0.2.0, 0.2.1, 0.3.0, 0.3.1, 0.3.2, 0.3.3, 0.3.4, 0.3.5, 0.3.6, 0.4.0, 0.4.1, 0.4.2, 0.5.0, 0.5.1, 0.5.2, 0.5.3, 0.6.0, 0.7.0, 0.8.0, 0.8.2, 0.9.0, 0.9.1, 0.9.2, 0.9.3, 0.9.4, 0.10.0, 0.10.1, 0.10.2, 0.10.3, 0.10.4, 0.10.5, 0.11.0, 0.11.1, 0.11.2, 0.11.3, 0.12.0, 0.12.1, 0.12.2, 0.12.3, 0.12.4, 0.12.5, 0.13.0, 0.13.1, 0.13.2, 0.13.3, 0.14.0, 0.14.1, 0.14.2)\\x1b[0m\\x1b[31m',\n",
+       " '\\x1b[0m\\x1b[31mERROR: No matching distribution found for traceloop-sdk==0.15.11\\x1b[0m\\x1b[31m',\n",
+       " '\\x1b[0m']"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "!!pip3 install -r requirements.txt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Collecting dateparser\n",
+      "  Using cached dateparser-1.2.0-py2.py3-none-any.whl.metadata (28 kB)\n",
+      "Requirement already satisfied: python-dateutil in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from dateparser) (2.9.0.post0)\n",
+      "Requirement already satisfied: pytz in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from dateparser) (2024.1)\n",
+      "Collecting regex!=2019.02.19,!=2021.8.27 (from dateparser)\n",
+      "  Downloading regex-2024.5.15-cp38-cp38-macosx_10_9_x86_64.whl.metadata (40 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m95.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hCollecting tzlocal (from dateparser)\n",
+      "  Downloading tzlocal-5.2-py3-none-any.whl.metadata (7.8 kB)\n",
+      "Requirement already satisfied: six>=1.5 in /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/site-packages (from python-dateutil->dateparser) (1.15.0)\n",
+      "Collecting backports.zoneinfo (from tzlocal->dateparser)\n",
+      "  Downloading backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl.metadata (4.7 kB)\n",
+      "Downloading dateparser-1.2.0-py2.py3-none-any.whl (294 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m510.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading regex-2024.5.15-cp38-cp38-macosx_10_9_x86_64.whl (281 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.8/281.8 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25hDownloading tzlocal-5.2-py3-none-any.whl (17 kB)\n",
+      "Downloading backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl (35 kB)\n",
+      "Installing collected packages: regex, backports.zoneinfo, tzlocal, dateparser\n",
+      "\u001b[33m  WARNING: The script dateparser-download is installed in '/Users/patrickalexis/Library/Python/3.8/bin' which is not on PATH.\n",
+      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
+      "\u001b[0mSuccessfully installed backports.zoneinfo-0.2.1 dateparser-1.2.0 regex-2024.5.15 tzlocal-5.2\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip3 install dateparser"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Collecting rapidfuzz\n",
+      "  Using cached rapidfuzz-3.9.1-cp38-cp38-macosx_10_9_x86_64.whl.metadata (11 kB)\n",
+      "Using cached rapidfuzz-3.9.1-cp38-cp38-macosx_10_9_x86_64.whl (2.1 MB)\n",
+      "Installing collected packages: rapidfuzz\n",
+      "Successfully installed rapidfuzz-3.9.1\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip3 install rapidfuzz"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'categorization'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Process the transactions csv and get the results from the categorization llm utility\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcategorization\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mfile_processing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m process_file, save_results\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdotenv\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m load_dotenv\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01masyncio\u001b[39;00m\n",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'categorization'"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Collecting dotenv\n",
+      "  Downloading dotenv-0.0.5.tar.gz (2.4 kB)\n",
+      "  Preparing metadata (setup.py) ... \u001b[?25lerror\n",
+      "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
+      "  \n",
+      "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
+      "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
+      "  \u001b[31m╰─>\u001b[0m \u001b[31m[76 lines of output]\u001b[0m\n",
+      "  \u001b[31m   \u001b[0m /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/__init__.py:80: _DeprecatedInstaller: setuptools.installer and fetch_build_eggs are deprecated.\n",
+      "  \u001b[31m   \u001b[0m !!\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m         ********************************************************************************\n",
+      "  \u001b[31m   \u001b[0m         Requirements should be satisfied by a PEP 517 installer.\n",
+      "  \u001b[31m   \u001b[0m         If you are using pip, you can try `pip install --use-pep517`.\n",
+      "  \u001b[31m   \u001b[0m         ********************************************************************************\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m !!\n",
+      "  \u001b[31m   \u001b[0m   dist.fetch_build_eggs(dist.setup_requires)\n",
+      "  \u001b[31m   \u001b[0m   \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m×\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m╰─>\u001b[0m \u001b[31m[16 lines of output]\u001b[0m\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"<string>\", line 2, in <module>\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"<pip-setuptools-caller>\", line 14, in <module>\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/__init__.py\", line 2, in <module>\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from setuptools.extension import Extension, Library\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/extension.py\", line 5, in <module>\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from setuptools.dist import _get_unpatched\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/dist.py\", line 7, in <module>\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from setuptools.command.install import install\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/command/__init__.py\", line 8, in <module>\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from setuptools.command import install_scripts\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/command/install_scripts.py\", line 3, in <module>\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from pkg_resources import Distribution, PathMetadata, ensure_directory\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/pkg_resources.py\", line 1518, in <module>\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     register_loader_type(importlib_bootstrap.SourceFileLoader, DefaultProvider)\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m AttributeError: module 'importlib._bootstrap' has no attribute 'SourceFileLoader'\n",
+      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m \u001b[31m[end of output]\u001b[0m\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m   \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
+      "  \u001b[31m   \u001b[0m \u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m \u001b[31m×\u001b[0m Encountered error while generating package metadata.\n",
+      "  \u001b[31m   \u001b[0m \u001b[31m╰─>\u001b[0m See above for output.\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m \u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
+      "  \u001b[31m   \u001b[0m \u001b[1;36mhint\u001b[0m: See above for details.\n",
+      "  \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/installer.py\", line 101, in _fetch_build_egg_no_warn\n",
+      "  \u001b[31m   \u001b[0m     subprocess.check_call(cmd)\n",
+      "  \u001b[31m   \u001b[0m   File \"/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/subprocess.py\", line 364, in check_call\n",
+      "  \u001b[31m   \u001b[0m     raise CalledProcessError(retcode, cmd)\n",
+      "  \u001b[31m   \u001b[0m subprocess.CalledProcessError: Command '['/Applications/Xcode.app/Contents/Developer/usr/bin/python3', '-m', 'pip', '--disable-pip-version-check', 'wheel', '--no-deps', '-w', '/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/tmpnw2yadh3', '--quiet', 'distribute']' returned non-zero exit status 1.\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m The above exception was the direct cause of the following exception:\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
+      "  \u001b[31m   \u001b[0m   File \"<string>\", line 2, in <module>\n",
+      "  \u001b[31m   \u001b[0m   File \"<pip-setuptools-caller>\", line 34, in <module>\n",
+      "  \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-install-ac3_ml88/dotenv_7766945cfd7d4e62965375b30e9a0c21/setup.py\", line 13, in <module>\n",
+      "  \u001b[31m   \u001b[0m     setup(name='dotenv',\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/__init__.py\", line 102, in setup\n",
+      "  \u001b[31m   \u001b[0m     _install_setup_requires(attrs)\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/__init__.py\", line 75, in _install_setup_requires\n",
+      "  \u001b[31m   \u001b[0m     _fetch_build_eggs(dist)\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/__init__.py\", line 80, in _fetch_build_eggs\n",
+      "  \u001b[31m   \u001b[0m     dist.fetch_build_eggs(dist.setup_requires)\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/dist.py\", line 641, in fetch_build_eggs\n",
+      "  \u001b[31m   \u001b[0m     return _fetch_build_eggs(self, requires)\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/installer.py\", line 38, in _fetch_build_eggs\n",
+      "  \u001b[31m   \u001b[0m     resolved_dists = pkg_resources.working_set.resolve(\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/pkg_resources/__init__.py\", line 787, in resolve\n",
+      "  \u001b[31m   \u001b[0m     dist = self._resolve_dist(\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/pkg_resources/__init__.py\", line 823, in _resolve_dist\n",
+      "  \u001b[31m   \u001b[0m     dist = best[req.key] = env.best_match(\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/pkg_resources/__init__.py\", line 1093, in best_match\n",
+      "  \u001b[31m   \u001b[0m     return self.obtain(req, installer)\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/pkg_resources/__init__.py\", line 1104, in obtain\n",
+      "  \u001b[31m   \u001b[0m     return installer(requirement) if installer else None\n",
+      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/installer.py\", line 103, in _fetch_build_egg_no_warn\n",
+      "  \u001b[31m   \u001b[0m     raise DistutilsError(str(e)) from e\n",
+      "  \u001b[31m   \u001b[0m distutils.errors.DistutilsError: Command '['/Applications/Xcode.app/Contents/Developer/usr/bin/python3', '-m', 'pip', '--disable-pip-version-check', 'wheel', '--no-deps', '-w', '/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/tmpnw2yadh3', '--quiet', 'distribute']' returned non-zero exit status 1.\n",
+      "  \u001b[31m   \u001b[0m \u001b[31m[end of output]\u001b[0m\n",
+      "  \n",
+      "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
+      "\u001b[?25h\u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
+      "\n",
+      "\u001b[31m×\u001b[0m Encountered error while generating package metadata.\n",
+      "\u001b[31m╰─>\u001b[0m See above for output.\n",
+      "\n",
+      "\u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
+      "\u001b[1;36mhint\u001b[0m: See above for details.\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip3 install dotenv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create data folder\n",
+    "!mkdir -p data/tx_data/output\n",
+    "!mkdir -p data/ref_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Collecting openai\n",
+      "  Using cached openai-1.30.4-py3-none-any.whl.metadata (21 kB)\n",
+      "Collecting anyio<5,>=3.5.0 (from openai)\n",
+      "  Downloading anyio-4.4.0-py3-none-any.whl.metadata (4.6 kB)\n",
+      "Collecting distro<2,>=1.7.0 (from openai)\n",
+      "  Downloading distro-1.9.0-py3-none-any.whl.metadata (6.8 kB)\n",
+      "Collecting httpx<1,>=0.23.0 (from openai)\n",
+      "  Downloading httpx-0.27.0-py3-none-any.whl.metadata (7.2 kB)\n",
+      "Requirement already satisfied: pydantic<3,>=1.9.0 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from openai) (2.7.1)\n",
+      "Collecting sniffio (from openai)\n",
+      "  Downloading sniffio-1.3.1-py3-none-any.whl.metadata (3.9 kB)\n",
+      "Requirement already satisfied: tqdm>4 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from openai) (4.66.4)\n",
+      "Requirement already satisfied: typing-extensions<5,>=4.7 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from openai) (4.12.0)\n",
+      "Requirement already satisfied: idna>=2.8 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
+      "Collecting exceptiongroup>=1.0.2 (from anyio<5,>=3.5.0->openai)\n",
+      "  Downloading exceptiongroup-1.2.1-py3-none-any.whl.metadata (6.6 kB)\n",
+      "Requirement already satisfied: certifi in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from httpx<1,>=0.23.0->openai) (2024.2.2)\n",
+      "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai)\n",
+      "  Downloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n",
+      "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai)\n",
+      "  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
+      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
+      "Requirement already satisfied: pydantic-core==2.18.2 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pydantic<3,>=1.9.0->openai) (2.18.2)\n",
+      "Downloading openai-1.30.4-py3-none-any.whl (320 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading anyio-4.4.0-py3-none-any.whl (86 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading distro-1.9.0-py3-none-any.whl (20 kB)\n",
+      "Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading sniffio-1.3.1-py3-none-any.whl (10 kB)\n",
+      "Downloading exceptiongroup-1.2.1-py3-none-any.whl (16 kB)\n",
+      "Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hInstalling collected packages: sniffio, h11, exceptiongroup, distro, httpcore, anyio, httpx, openai\n",
+      "\u001b[33m  WARNING: The script distro is installed in '/Users/patrickalexis/Library/Python/3.8/bin' which is not on PATH.\n",
+      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33m  WARNING: The script httpx is installed in '/Users/patrickalexis/Library/Python/3.8/bin' which is not on PATH.\n",
+      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33m  WARNING: The script openai is installed in '/Users/patrickalexis/Library/Python/3.8/bin' which is not on PATH.\n",
+      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
+      "\u001b[0mSuccessfully installed anyio-4.4.0 distro-1.9.0 exceptiongroup-1.2.1 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 openai-1.30.4 sniffio-1.3.1\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip3 install openai"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Processing file\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "ERROR:root:| File: transactions_2024.csv | Unexpected Error: expected str, got module\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ERROR processing file transactions_2024.csv: expected str, got module\n",
+      "\n",
+      "Processed 1 files: 0 successful, 1 with errors\n",
+      "\n",
+      "Errors in the following files:\n",
+      "  transactions_2024.csv: expected str, got module\n",
+      "\n",
+      "\n",
+      "[{'file_name': 'transactions_2024.csv', 'output': Empty DataFrame\n",
+      "Columns: []\n",
+      "Index: [], 'error': 'expected str, got module'}]\n",
+      "['name/description,category\\n']\n",
+      "None\n"
      ]
     }
    ],
    "source": [
     "# Process the transactions csv and get the results from the categorization llm utility\n",
-    "from categorization.file_processing import process_file, save_results\n",
-    "from dotenv import load_dotenv\n",
+    "from app.categorization.file_processing import process_file, save_results\n",
+    "from app.categorization.config import CATEGORY_REFERENCE_OUTPUT_FILE\n",
+    "# from dotenv import load_dotenv\n",
     "import asyncio\n",
+    "import os.path\n",
+    "\n",
+    "# load_dotenv()\n",
     "\n",
-    "load_dotenv()\n",
+    "# relative_dir = os.path.dirname(__file__)\n",
+    "# abs_file_path = os.path.join(relative_dir, CATEGORY_REFERENCE_OUTPUT_FILE)\n",
     "\n",
     "async def apply_categorization():\n",
-    "    processed_file = process_file(\"transactions_2024.csv\")\n",
+    "    processed_file = process_file(\"app/transactions_rag/transactions_2024.csv\")\n",
     "\n",
     "    print(\"\\nProcessing file\")\n",
     "    result = await asyncio.gather(processed_file)\n",
     "\n",
-    "    save_results(results)\n",
-    "    print(results)\n",
-    "    \n",
+    "    save_results(result)\n",
+    "    print(result)\n",
+    "\n",
+    "    output_file = open(CATEGORY_REFERENCE_OUTPUT_FILE, \"r+\")\n",
+    "    print(output_file.readlines())\n",
+    "\n",
     "\n",
-    "asyncio.run(apply_categorization())"
+    "result = await apply_categorization()\n",
+    "print(result)\n",
+    "\n"
    ]
   }
  ],

requirements.txt

@@ -1,3 +1,14 @@
+langchain
+langchain_openai
+python-dotenv
+openai
+tenacity
+rapidfuzz
+pydantic
+dateparser
+pandas
+path
+
 aiostream==0.5.2
 cachetools==5.3.3
 docx2txt==0.8
@@ -8,14 +19,4 @@ llama-index-vector-stores-pinecone==0.1.3
 llama-index==0.10.28
 python-dotenv==1.0.0
 traceloop-sdk==0.15.11
-uvicorn==0.23.2
-
-langchain
-python-dotenv
-openai
-tenacity
-rapidfuzz
-pydantic
-dateparser
-pandas
-path
+uvicorn==0.23.2