app/transactions_rag/categorize_transactions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: pandas in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (2.0.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: numpy>=1.20.3 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pandas) (1.24.4)\n",
      "Requirement already satisfied: six>=1.5 in /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas) (1.15.0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name / Description</th>\n",
       "      <th>Expense/Income</th>\n",
       "      <th>Amount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023-12-30</td>\n",
       "      <td>Comcast Internet</td>\n",
       "      <td>Expense</td>\n",
       "      <td>9.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023-12-30</td>\n",
       "      <td>Lemonade Home Insurance</td>\n",
       "      <td>Expense</td>\n",
       "      <td>17.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023-12-30</td>\n",
       "      <td>Monthly Appartment Rent</td>\n",
       "      <td>Expense</td>\n",
       "      <td>2000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023-12-30</td>\n",
       "      <td>Staples Office Supplies</td>\n",
       "      <td>Expense</td>\n",
       "      <td>12.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023-12-29</td>\n",
       "      <td>Selling Paintings</td>\n",
       "      <td>Income</td>\n",
       "      <td>13.63</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date       Name / Description Expense/Income   Amount\n",
       "0  2023-12-30         Comcast Internet        Expense     9.96\n",
       "1  2023-12-30  Lemonade Home Insurance        Expense    17.53\n",
       "2  2023-12-30  Monthly Appartment Rent        Expense  2000.00\n",
       "3  2023-12-30  Staples Office Supplies        Expense    12.46\n",
       "4  2023-12-29        Selling Paintings         Income    13.63"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the transactions csv\n",
    "!pip3 install pandas\n",
    "\n",
    "import pandas as pd\n",
    "df = pd.read_csv(\"transactions_2024.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Lemonade Home Insurance', 'Monthly Appartment Rent',\n",
       "       'Staples Office Supplies', 'Selling Paintings', 'Spotify',\n",
       "       'Target', 'IT Consulting', 'Phone', 'ML Consulting'], dtype=object)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get Unique transactions in the Name/Description column\n",
    "unique_transactions = df[\"Name / Description\"].unique()\n",
    "unique_transactions[1:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 -m venv venv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "!source venv/bin/activate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: setuptools in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (70.0.0)\n"
     ]
    }
   ],
   "source": [
    "!pip3 install --upgrade setuptools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Defaulting to user installation because normal site-packages is not writeable',\n",
       " 'Requirement already satisfied: langchain in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 1)) (0.2.1)',\n",
       " 'Collecting langchain_openai (from -r requirements.txt (line 2))',\n",
       " '  Downloading langchain_openai-0.1.7-py3-none-any.whl.metadata (2.5 kB)',\n",
       " 'Collecting python-dotenv (from -r requirements.txt (line 3))',\n",
       " '  Using cached python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)',\n",
       " 'Collecting openai (from -r requirements.txt (line 4))',\n",
       " '  Using cached openai-1.30.4-py3-none-any.whl.metadata (21 kB)',\n",
       " 'Requirement already satisfied: tenacity in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 5)) (8.3.0)',\n",
       " 'Requirement already satisfied: rapidfuzz in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 6)) (3.9.1)',\n",
       " 'Requirement already satisfied: pydantic in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 7)) (2.7.1)',\n",
       " 'Requirement already satisfied: dateparser in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 8)) (1.2.0)',\n",
       " 'Requirement already satisfied: pandas in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from -r requirements.txt (line 9)) (2.0.3)',\n",
       " 'Collecting path (from -r requirements.txt (line 10))',\n",
       " '  Using cached path-16.14.0-py3-none-any.whl.metadata (6.3 kB)',\n",
       " 'Collecting aiostream==0.5.2 (from -r requirements.txt (line 12))',\n",
       " '  Using cached aiostream-0.5.2-py3-none-any.whl.metadata (9.9 kB)',\n",
       " 'Collecting cachetools==5.3.3 (from -r requirements.txt (line 13))',\n",
       " '  Using cached cachetools-5.3.3-py3-none-any.whl.metadata (5.3 kB)',\n",
       " 'Collecting docx2txt==0.8 (from -r requirements.txt (line 14))',\n",
       " '  Using cached docx2txt-0.8.tar.gz (2.8 kB)',\n",
       " '  Preparing metadata (setup.py): started',\n",
       " \"  Preparing metadata (setup.py): finished with status 'done'\",\n",
       " 'Collecting fastapi==0.109.1 (from -r requirements.txt (line 15))',\n",
       " '  Using cached fastapi-0.109.1-py3-none-any.whl.metadata (25 kB)',\n",
       " 'Collecting llama-index-agent-openai==0.2.2 (from -r requirements.txt (line 16))',\n",
       " '  Using cached llama_index_agent_openai-0.2.2-py3-none-any.whl.metadata (677 bytes)',\n",
       " 'Collecting llama-index-core==0.10.28 (from -r requirements.txt (line 17))',\n",
       " '  Using cached llama_index_core-0.10.28-py3-none-any.whl.metadata (3.6 kB)',\n",
       " 'Collecting llama-index-vector-stores-pinecone==0.1.3 (from -r requirements.txt (line 18))',\n",
       " '  Using cached llama_index_vector_stores_pinecone-0.1.3-py3-none-any.whl.metadata (725 bytes)',\n",
       " 'Collecting llama-index==0.10.28 (from -r requirements.txt (line 19))',\n",
       " '  Using cached llama_index-0.10.28-py3-none-any.whl.metadata (11 kB)',\n",
       " 'Collecting python-dotenv (from -r requirements.txt (line 3))',\n",
       " '  Using cached python_dotenv-1.0.0-py3-none-any.whl.metadata (21 kB)',\n",
       " '\\x1b[31mERROR: Ignored the following versions that require a different python version: 0.0.10 Requires-Python >=3.9,<4.0; 0.0.11 Requires-Python >=3.9,<4.0; 0.0.12 Requires-Python >=3.9,<4.0; 0.0.13 Requires-Python >=3.9,<4.0; 0.0.14 Requires-Python >=3.9,<4.0; 0.0.15 Requires-Python >=3.9,<4.0; 0.0.16 Requires-Python >=3.9,<4.0; 0.0.17 Requires-Python >=3.9,<4.0; 0.0.18 Requires-Python >=3.9,<4.0; 0.0.19 Requires-Python >=3.9,<4.0; 0.0.20 Requires-Python >=3.9,<4.0; 0.0.21 Requires-Python >=3.9,<4.0; 0.0.22 Requires-Python >=3.9,<4.0; 0.0.23 Requires-Python >=3.9,<4.0; 0.0.24 Requires-Python >=3.9,<4.0; 0.0.25 Requires-Python >=3.9,<4.0; 0.14.3 Requires-Python >=3.9,<4; 0.14.4 Requires-Python <4,>=3.9; 0.14.5 Requires-Python <4,>=3.9; 0.15.0 Requires-Python <4,>=3.9; 0.15.1 Requires-Python <4,>=3.9; 0.15.10 Requires-Python <4,>=3.9; 0.15.11 Requires-Python <4,>=3.9; 0.15.12 Requires-Python <4,>=3.9; 0.15.13 Requires-Python <4,>=3.9; 0.15.2 Requires-Python <4,>=3.9; 0.15.3 Requires-Python <4,>=3.9; 0.15.4 Requires-Python <4,>=3.9; 0.15.5 Requires-Python <4,>=3.9; 0.15.6 Requires-Python <4,>=3.9; 0.15.7 Requires-Python <4,>=3.9; 0.15.8 Requires-Python <4,>=3.9; 0.15.9 Requires-Python <4,>=3.9; 0.16.0 Requires-Python <4,>=3.9; 0.16.1 Requires-Python <4,>=3.9; 0.16.2 Requires-Python <4,>=3.9; 0.16.3 Requires-Python <4,>=3.9; 0.16.4 Requires-Python <4,>=3.9; 0.16.5 Requires-Python <4,>=3.9; 0.16.6 Requires-Python <4,>=3.9; 0.16.7 Requires-Python <4,>=3.9; 0.16.8 Requires-Python <4,>=3.9; 0.16.9 Requires-Python <4,>=3.9; 0.17.0 Requires-Python <4,>=3.9; 0.17.1 Requires-Python <4,>=3.9; 0.17.2 Requires-Python <4,>=3.9; 0.17.3 Requires-Python <4,>=3.9; 0.17.4 Requires-Python <4,>=3.9; 0.17.5 Requires-Python <4,>=3.9; 0.17.6 Requires-Python <4,>=3.9; 0.17.7 Requires-Python <4,>=3.9; 0.18.0 Requires-Python <4,>=3.9; 0.18.1 Requires-Python <4,>=3.9; 0.18.2 Requires-Python <4,>=3.9; 0.19.0 Requires-Python <4,>=3.9; 0.20.0 Requires-Python <4,>=3.9; 0.21.0 Requires-Python <4,>=3.9\\x1b[0m\\x1b[31m',\n",
       " '\\x1b[0m\\x1b[31mERROR: Could not find a version that satisfies the requirement traceloop-sdk==0.15.11 (from versions: 0.0.26, 0.0.27b0, 0.0.27, 0.0.28, 0.0.29, 0.0.31, 0.0.32, 0.0.33, 0.0.34, 0.0.35, 0.0.36, 0.0.37, 0.0.38, 0.0.39, 0.0.40, 0.0.41, 0.0.42, 0.0.43b0, 0.0.43b2, 0.0.43b3, 0.0.43b4, 0.0.43, 0.0.44b0, 0.0.44, 0.0.46, 0.0.47, 0.0.48, 0.0.49, 0.0.50, 0.0.51, 0.0.52, 0.0.53, 0.0.54, 0.0.55, 0.0.56a0, 0.0.56a1, 0.0.56a2, 0.0.56a3, 0.0.56a4, 0.0.56a5, 0.0.56a6, 0.0.56, 0.0.57, 0.0.58, 0.0.59, 0.0.60a0, 0.0.60a1, 0.0.60, 0.0.61a0, 0.0.61, 0.0.62, 0.0.63, 0.0.64a0, 0.0.64, 0.0.65, 0.0.66, 0.0.67, 0.0.68, 0.0.69, 0.0.70, 0.1.3, 0.1.5, 0.1.6, 0.1.7, 0.1.8, 0.1.9, 0.1.10, 0.1.11, 0.1.12, 0.2.0, 0.2.1, 0.3.0, 0.3.1, 0.3.2, 0.3.3, 0.3.4, 0.3.5, 0.3.6, 0.4.0, 0.4.1, 0.4.2, 0.5.0, 0.5.1, 0.5.2, 0.5.3, 0.6.0, 0.7.0, 0.8.0, 0.8.2, 0.9.0, 0.9.1, 0.9.2, 0.9.3, 0.9.4, 0.10.0, 0.10.1, 0.10.2, 0.10.3, 0.10.4, 0.10.5, 0.11.0, 0.11.1, 0.11.2, 0.11.3, 0.12.0, 0.12.1, 0.12.2, 0.12.3, 0.12.4, 0.12.5, 0.13.0, 0.13.1, 0.13.2, 0.13.3, 0.14.0, 0.14.1, 0.14.2)\\x1b[0m\\x1b[31m',\n",
       " '\\x1b[0m\\x1b[31mERROR: No matching distribution found for traceloop-sdk==0.15.11\\x1b[0m\\x1b[31m',\n",
       " '\\x1b[0m']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!!pip3 install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting dateparser\n",
      "  Using cached dateparser-1.2.0-py2.py3-none-any.whl.metadata (28 kB)\n",
      "Requirement already satisfied: python-dateutil in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from dateparser) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from dateparser) (2024.1)\n",
      "Collecting regex!=2019.02.19,!=2021.8.27 (from dateparser)\n",
      "  Downloading regex-2024.5.15-cp38-cp38-macosx_10_9_x86_64.whl.metadata (40 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m95.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting tzlocal (from dateparser)\n",
      "  Downloading tzlocal-5.2-py3-none-any.whl.metadata (7.8 kB)\n",
      "Requirement already satisfied: six>=1.5 in /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/site-packages (from python-dateutil->dateparser) (1.15.0)\n",
      "Collecting backports.zoneinfo (from tzlocal->dateparser)\n",
      "  Downloading backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl.metadata (4.7 kB)\n",
      "Downloading dateparser-1.2.0-py2.py3-none-any.whl (294 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m510.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading regex-2024.5.15-cp38-cp38-macosx_10_9_x86_64.whl (281 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.8/281.8 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading tzlocal-5.2-py3-none-any.whl (17 kB)\n",
      "Downloading backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl (35 kB)\n",
      "Installing collected packages: regex, backports.zoneinfo, tzlocal, dateparser\n",
      "\u001b[33m  WARNING: The script dateparser-download is installed in '/Users/patrickalexis/Library/Python/3.8/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0mSuccessfully installed backports.zoneinfo-0.2.1 dateparser-1.2.0 regex-2024.5.15 tzlocal-5.2\n"
     ]
    }
   ],
   "source": [
    "!pip3 install dateparser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting rapidfuzz\n",
      "  Using cached rapidfuzz-3.9.1-cp38-cp38-macosx_10_9_x86_64.whl.metadata (11 kB)\n",
      "Using cached rapidfuzz-3.9.1-cp38-cp38-macosx_10_9_x86_64.whl (2.1 MB)\n",
      "Installing collected packages: rapidfuzz\n",
      "Successfully installed rapidfuzz-3.9.1\n"
     ]
    }
   ],
   "source": [
    "!pip3 install rapidfuzz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting dotenv\n",
      "  Downloading dotenv-0.0.5.tar.gz (2.4 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25lerror\n",
      "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
      "  \n",
      "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
      "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
      "  \u001b[31m╰─>\u001b[0m \u001b[31m[76 lines of output]\u001b[0m\n",
      "  \u001b[31m   \u001b[0m /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/__init__.py:80: _DeprecatedInstaller: setuptools.installer and fetch_build_eggs are deprecated.\n",
      "  \u001b[31m   \u001b[0m !!\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m         ********************************************************************************\n",
      "  \u001b[31m   \u001b[0m         Requirements should be satisfied by a PEP 517 installer.\n",
      "  \u001b[31m   \u001b[0m         If you are using pip, you can try `pip install --use-pep517`.\n",
      "  \u001b[31m   \u001b[0m         ********************************************************************************\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m !!\n",
      "  \u001b[31m   \u001b[0m   dist.fetch_build_eggs(dist.setup_requires)\n",
      "  \u001b[31m   \u001b[0m   \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m   \u001b[31m×\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m╰─>\u001b[0m \u001b[31m[16 lines of output]\u001b[0m\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"<string>\", line 2, in <module>\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"<pip-setuptools-caller>\", line 14, in <module>\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/__init__.py\", line 2, in <module>\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from setuptools.extension import Extension, Library\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/extension.py\", line 5, in <module>\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from setuptools.dist import _get_unpatched\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/dist.py\", line 7, in <module>\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from setuptools.command.install import install\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/command/__init__.py\", line 8, in <module>\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from setuptools.command import install_scripts\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/setuptools/command/install_scripts.py\", line 3, in <module>\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     from pkg_resources import Distribution, PathMetadata, ensure_directory\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-wheel-66v7g1h6/distribute_6e0b47b8750a4aa8bfff62a3f867a97f/pkg_resources.py\", line 1518, in <module>\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m     register_loader_type(importlib_bootstrap.SourceFileLoader, DefaultProvider)\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m AttributeError: module 'importlib._bootstrap' has no attribute 'SourceFileLoader'\n",
      "  \u001b[31m   \u001b[0m   \u001b[31m   \u001b[0m \u001b[31m[end of output]\u001b[0m\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m   \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "  \u001b[31m   \u001b[0m \u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m \u001b[31m×\u001b[0m Encountered error while generating package metadata.\n",
      "  \u001b[31m   \u001b[0m \u001b[31m╰─>\u001b[0m See above for output.\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m \u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
      "  \u001b[31m   \u001b[0m \u001b[1;36mhint\u001b[0m: See above for details.\n",
      "  \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/installer.py\", line 101, in _fetch_build_egg_no_warn\n",
      "  \u001b[31m   \u001b[0m     subprocess.check_call(cmd)\n",
      "  \u001b[31m   \u001b[0m   File \"/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/subprocess.py\", line 364, in check_call\n",
      "  \u001b[31m   \u001b[0m     raise CalledProcessError(retcode, cmd)\n",
      "  \u001b[31m   \u001b[0m subprocess.CalledProcessError: Command '['/Applications/Xcode.app/Contents/Developer/usr/bin/python3', '-m', 'pip', '--disable-pip-version-check', 'wheel', '--no-deps', '-w', '/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/tmpnw2yadh3', '--quiet', 'distribute']' returned non-zero exit status 1.\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m The above exception was the direct cause of the following exception:\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
      "  \u001b[31m   \u001b[0m   File \"<string>\", line 2, in <module>\n",
      "  \u001b[31m   \u001b[0m   File \"<pip-setuptools-caller>\", line 34, in <module>\n",
      "  \u001b[31m   \u001b[0m   File \"/private/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/pip-install-ac3_ml88/dotenv_7766945cfd7d4e62965375b30e9a0c21/setup.py\", line 13, in <module>\n",
      "  \u001b[31m   \u001b[0m     setup(name='dotenv',\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/__init__.py\", line 102, in setup\n",
      "  \u001b[31m   \u001b[0m     _install_setup_requires(attrs)\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/__init__.py\", line 75, in _install_setup_requires\n",
      "  \u001b[31m   \u001b[0m     _fetch_build_eggs(dist)\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/__init__.py\", line 80, in _fetch_build_eggs\n",
      "  \u001b[31m   \u001b[0m     dist.fetch_build_eggs(dist.setup_requires)\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/dist.py\", line 641, in fetch_build_eggs\n",
      "  \u001b[31m   \u001b[0m     return _fetch_build_eggs(self, requires)\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/installer.py\", line 38, in _fetch_build_eggs\n",
      "  \u001b[31m   \u001b[0m     resolved_dists = pkg_resources.working_set.resolve(\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/pkg_resources/__init__.py\", line 787, in resolve\n",
      "  \u001b[31m   \u001b[0m     dist = self._resolve_dist(\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/pkg_resources/__init__.py\", line 823, in _resolve_dist\n",
      "  \u001b[31m   \u001b[0m     dist = best[req.key] = env.best_match(\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/pkg_resources/__init__.py\", line 1093, in best_match\n",
      "  \u001b[31m   \u001b[0m     return self.obtain(req, installer)\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/pkg_resources/__init__.py\", line 1104, in obtain\n",
      "  \u001b[31m   \u001b[0m     return installer(requirement) if installer else None\n",
      "  \u001b[31m   \u001b[0m   File \"/Users/patrickalexis/Library/Python/3.8/lib/python/site-packages/setuptools/installer.py\", line 103, in _fetch_build_egg_no_warn\n",
      "  \u001b[31m   \u001b[0m     raise DistutilsError(str(e)) from e\n",
      "  \u001b[31m   \u001b[0m distutils.errors.DistutilsError: Command '['/Applications/Xcode.app/Contents/Developer/usr/bin/python3', '-m', 'pip', '--disable-pip-version-check', 'wheel', '--no-deps', '-w', '/var/folders/g9/flbt409n1b7g0ry4b9ffdvnc0000gn/T/tmpnw2yadh3', '--quiet', 'distribute']' returned non-zero exit status 1.\n",
      "  \u001b[31m   \u001b[0m \u001b[31m[end of output]\u001b[0m\n",
      "  \n",
      "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "\u001b[?25h\u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
      "\n",
      "\u001b[31m×\u001b[0m Encountered error while generating package metadata.\n",
      "\u001b[31m╰─>\u001b[0m See above for output.\n",
      "\n",
      "\u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
      "\u001b[1;36mhint\u001b[0m: See above for details.\n"
     ]
    }
   ],
   "source": [
    "!pip3 install dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create data folder\n",
    "!mkdir -p data/tx_data/output\n",
    "!mkdir -p data/ref_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting openai\n",
      "  Using cached openai-1.30.4-py3-none-any.whl.metadata (21 kB)\n",
      "Collecting anyio<5,>=3.5.0 (from openai)\n",
      "  Downloading anyio-4.4.0-py3-none-any.whl.metadata (4.6 kB)\n",
      "Collecting distro<2,>=1.7.0 (from openai)\n",
      "  Downloading distro-1.9.0-py3-none-any.whl.metadata (6.8 kB)\n",
      "Collecting httpx<1,>=0.23.0 (from openai)\n",
      "  Downloading httpx-0.27.0-py3-none-any.whl.metadata (7.2 kB)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from openai) (2.7.1)\n",
      "Collecting sniffio (from openai)\n",
      "  Downloading sniffio-1.3.1-py3-none-any.whl.metadata (3.9 kB)\n",
      "Requirement already satisfied: tqdm>4 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from openai) (4.66.4)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from openai) (4.12.0)\n",
      "Requirement already satisfied: idna>=2.8 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
      "Collecting exceptiongroup>=1.0.2 (from anyio<5,>=3.5.0->openai)\n",
      "  Downloading exceptiongroup-1.2.1-py3-none-any.whl.metadata (6.6 kB)\n",
      "Requirement already satisfied: certifi in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from httpx<1,>=0.23.0->openai) (2024.2.2)\n",
      "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai)\n",
      "  Downloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n",
      "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai)\n",
      "  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.2 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pydantic<3,>=1.9.0->openai) (2.18.2)\n",
      "Downloading openai-1.30.4-py3-none-any.whl (320 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading anyio-4.4.0-py3-none-any.whl (86 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading distro-1.9.0-py3-none-any.whl (20 kB)\n",
      "Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading sniffio-1.3.1-py3-none-any.whl (10 kB)\n",
      "Downloading exceptiongroup-1.2.1-py3-none-any.whl (16 kB)\n",
      "Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: sniffio, h11, exceptiongroup, distro, httpcore, anyio, httpx, openai\n",
      "\u001b[33m  WARNING: The script distro is installed in '/Users/patrickalexis/Library/Python/3.8/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  WARNING: The script httpx is installed in '/Users/patrickalexis/Library/Python/3.8/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  WARNING: The script openai is installed in '/Users/patrickalexis/Library/Python/3.8/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0mSuccessfully installed anyio-4.4.0 distro-1.9.0 exceptiongroup-1.2.1 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 openai-1.30.4 sniffio-1.3.1\n"
     ]
    }
   ],
   "source": [
    "!pip3 install openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:| File: transactions_2024.csv | Unexpected Error: expected str, got module\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR processing file transactions_2024.csv: expected str, got module\n",
      "\n",
      "Processed 1 files: 0 successful, 1 with errors\n",
      "\n",
      "Errors in the following files:\n",
      "  transactions_2024.csv: expected str, got module\n",
      "\n",
      "\n",
      "[{'file_name': 'transactions_2024.csv', 'output': Empty DataFrame\n",
      "Columns: []\n",
      "Index: [], 'error': 'expected str, got module'}]\n",
      "['name/description,category\\n']\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Process the transactions csv and get the results from the categorization llm utility\n",
    "from app.categorization.file_processing import process_file, save_results\n",
    "from app.categorization.config import CATEGORY_REFERENCE_OUTPUT_FILE\n",
    "# from dotenv import load_dotenv\n",
    "import asyncio\n",
    "import os.path\n",
    "\n",
    "# load_dotenv()\n",
    "\n",
    "# relative_dir = os.path.dirname(__file__)\n",
    "# abs_file_path = os.path.join(relative_dir, CATEGORY_REFERENCE_OUTPUT_FILE)\n",
    "\n",
    "async def apply_categorization():\n",
    "    processed_file = process_file(\"app/transactions_rag/transactions_2024.csv\")\n",
    "\n",
    "    print(\"\\nProcessing file\")\n",
    "    result = await asyncio.gather(processed_file)\n",
    "\n",
    "    await save_results(result)\n",
    "    print(result)\n",
    "\n",
    "    output_file = open(CATEGORY_REFERENCE_OUTPUT_FILE, \"r+\")\n",
    "    print(output_file.readlines())\n",
    "\n",
    "\n",
    "result = await apply_categorization()\n",
    "print(result)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}