chore: init

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Auto detect text files and perform LF normalization
+* text=auto

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+.DS_Store
+.git

README.md

@@ -1,13 +1,149 @@
 ---
-title: HF Customer Service Chatbot
-emoji: 🐠
-colorFrom: red
-colorTo: pink
-sdk: gradio
-sdk_version: 5.32.1
+title: Customer Service Chatbot
+emoji: 🔮
+colorFrom: indigo
+colorTo: indigo
+sdk: streamlit
+sdk_version: 1.34.0
 app_file: app.py
 pinned: false
-short_description: Customer Service Chatbot
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+This is a customer service chatbot that helps analyze user input and pull out relevant record from the database.
+
+The app is built with Langchain, Ollama and Streamlit.
+
+What the chatbot exactly does?
+- extract transaction id from user input
+- determine transaction type based on the transaction id
+- find record from database table based on transaction id and type
+
+A transaction id may starts with `payment`, which indicates a payment transaction; or starts with `payout`, which indicates a payout transaction.
+
+There are `payment` and `payout` tables in the database, each contains some records. Details can be found in `db.py`.
+
+Prompts are fine tuned using following techniques for better result:
+- clear instructions
+- asking for justification
+- use delimiters
+- chain of thought
+
+
+Example UI:
+<img src="example.jpg" alt="API" width="800"/>
+
+Steps to run
+- Install Ollama server locally, and download the llama3 model (not llama3.2).
+- Install Python dependencies by executing ```pip install -r requirements.txt```
+- Start the app by executing the following command from the project root directory
+```python -m streamlit run app.py --server.port=8501 --server.address=0.0.0.0```
+
+
+## Examples of Prompt Engineering
+### Bot returns example value when transaction id not found
+The system prompt `extract_system_prompt` was defined as the following, which includes examples for payment and payout transaction id formats.
+```python
+extract_system_prompt = (
+    "You are a customer officer that helps extract transaction id from the user input and determine the transaction type based on the transaction id. "
+    "There are two types of transactions: payment and payout. "
+    "1. A payout transaction, whose transaction id starts with 'payout_', "
+    "===>START EXAMPLE"
+    "payout_a1b2c3d4"
+    "<===END EXAMPLE"
+    "2. A payment transaction, whose transaction id starts with 'payment_', "
+    "===>START EXAMPLE"
+    "payment_a1b2c3d4"
+    "<===END EXAMPLE"
+    "===>START USER INPUT"
+    "{input}"
+    "<===END USER INPUT"
+    "Respond with the following information: "
+    "1. a flag named found which indicates if a transaction id is found "
+    "2. the transaction id named transaction_id if found "
+    "3. the transaction type (in lower case) named transaction_type if found "
+    "4. the explanation named justification which explains how you deduce transaction id and transaction type "
+    "You must give answer in JSON format without any extra content, and don't make up information."
+)
+```
+
+The bot sometimes gives wrong answer that returns the example value or unrelated text when it cannot find a transaction id in the user input.
+
+input:
+My transaction failed
+
+wrong answer1: 
+{ "found": true, "transaction_id": "payment_a1b2c3d4", "transaction_type": "payment", "justification": "The transaction id starts with 'payment_', which indicates it's a payment transaction." }
+
+wrong answer2:
+{"found": true, "transaction_id": "My transaction", "transaction_type": "payment", "justification": "The transaction id 'My transaction' starts with 'payment_', indicating a payment transaction." }
+
+solution:
+Examples are removed.
+
+```python
+extract_system_prompt = (
+    "You are a customer officer that helps extract transaction id from the USER INPUT and determine the transaction type based on the transaction id. "
+    "A transaction id can take one of two forms: "
+    "1. A transaction id may start with 'payout_', which indicates a payout transaction. "
+    "2. A transaction id may start with 'payment_', which indicates a payment transaction. "
+    "===>START USER INPUT"
+    "{input}"
+    "<===END USER INPUT"
+    "Respond with the following information: "
+    "1. a flag named found, it is true if you can find a transaction id in the USER INPUT, otherwise it is false "
+    "2. the transaction id named transaction_id if found "
+    "3. the transaction type (in lower case) named transaction_type if found "
+    "4. the explanation named justification which explains how you deduce transaction id and transaction type "
+    "You must give answer in JSON format without any extra content, and don't make up transaction id if no exact text appears in the USER INPUT."
+)
+```
+
+### bot treats invalid transaction id as valid
+The system prompt `extract_system_prompt` was defined as the following, which includes detailed steps to verify and extract transaction id.
+```python
+extract_system_prompt = (
+    "You are a customer officer that helps extract transaction id from the USER INPUT and determine the transaction type based on the transaction id. "
+    "To find transaction id, follow all the steps below: "
+    "Step 1. **Look for prefix**: for each word in the USER INPUT, check if it starts with 'payout' or 'payment', if so, you should go to Step 2, otherwise go to Step 7. "
+    "Step 2. **Find a single dash**: check the character immediately after the prefix 'payout' or 'payment', if it is a dash, you should go to Step 3, otherwise go to Step 7.. "
+    "Step 3. **Find digits and characters**: check if there is at least one digit or one character after the dash, if so, you should go to Step 4, otherwise go to Step 7.. "
+    "Step 4. **Extract transaction id**: extract transaction id from the USER INPUT, then go to Step 5. "
+    "Step 5. **Verify transaction id**: verify the extracted transaction id by checking if the USER INPUT contains exact the same text, if so, you should go to Step 6, otherwise go to Step 7. "
+    "Step 6. It is a valid transaction id, when constructing the response, return the flag found as true. "
+    "Step 7. It is not a valid transaction id, when constructing the response, return the flag found as false. "
+    "To determine the transaction type, follow the rule below: "
+    "There are two types of transactions: "
+    "a. A transaction id starting with 'payout' indicates a payout transaction. "
+    "b. A transaction id starting with 'payment' indicates a payment transaction. "
+    "===>START USER INPUT"
+    "{input}"
+    "<===END USER INPUT"
+    "Respond with the following information: "
+    "1. a flag named found, which indicates if a valid transaction id is found "
+    "2. the transaction id named transaction_id if found "
+    "3. the transaction type (in lower case) named transaction_type if found "
+    "4. the explanation named justification which explains how you deduce transaction id and transaction type "
+    "You must give answer in JSON format without any extra content, and don't make up transaction id if no exact text appears in the USER INPUT."
+)
+```
+
+The bot treats invalid transaction id such as payment3 or payout5 as valid.
+
+input:
+My payment3 failed
+
+(here payment3 is not a valid transaction id since there is no dash after 'payment')
+
+wrong answer: 
+{"found": true, "transaction_id": "payment-", "transaction_type": "payment", "justification": "The user input contains a string pattern that matches a valid transaction id starting with 'payment-' and followed by a single dash."}
+
+possible solutions (not implemented yet):
+- Use Python REPL tool with Regex instead of describing rules in prompt, or 
+- Use a better LLM (my test is using llama3), or 
+- Split the complex prompt into multiple smaller prompts then feed each prompt into a LLM.
+
+
+### SQL agent returns error "Invalid Format: Missing 'Action:' after 'Thought:"
+
+solution:
+create SQL agent every time for every user input, i.e. do not reuse SQL agent.

app.py

@@ -0,0 +1,115 @@
+
+import os
+from dotenv import load_dotenv
+
+st.set_page_config(page_title="Chat", page_icon=":page_facing_up:")
+
+load_dotenv()
+
+from huggingface_hub import login
+login(token=os.getenv("HUGGINGFACEHUB_API_KEY"))
+
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import gradio as gr
+
+# model_name="deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" # 15G
+model_name="meta-llama/Llama-3.2-3B-Instruct"   # 6.5G
+
+# # 
+# # HuggingFaceTB/SmolLM2-135M-Instruct
+# # deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+# checkpoint = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
+# checkpoint = "meta-llama/Llama-3.2-3B-Instruct"   # 6.5G
+device = "mps"  # "cuda" or "cpu"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
+
+# def predict(message, history):
+#     history.append({"role": "user", "content": message})
+#     input_text = tokenizer.apply_chat_template(history, tokenize=False)
+#     inputs = tokenizer.encode(input_text, return_tensors="pt").to(device)  
+#     outputs = model.generate(inputs, max_new_tokens=100, temperature=0.2, top_p=0.9, do_sample=True)
+#     decoded = tokenizer.decode(outputs[0])
+#     response = decoded.split("<|im_start|>assistant\n")[-1].split("<|im_end|>")[0]
+#     return response
+
+# demo = gr.ChatInterface(predict, type="messages")
+
+# demo.launch()
+
+# import os
+# from dotenv import load_dotenv
+
+# load_dotenv()
+
+# from huggingface_hub import login
+# login(token=os.getenv("HUGGINGFACEHUB_API_KEY"))
+
+# pipe = pipeline(model=model_name, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)
+# prompt = """Let's go through this step-by-step:
+# 1. You start with 15 muffins.
+# 2. You eat 2 muffins, leaving you with 13 muffins.
+# 3. You give 5 muffins to your neighbor, leaving you with 8 muffins.
+# 4. Your partner buys 6 more muffins, bringing the total number of muffins to 14.
+# 5. Your partner eats 2 muffins, leaving you with 12 muffins.
+# If you eat 6 muffins, how many are left?"""
+
+# torch.device("mps")
+
+# pipeline = pipeline.to("mps")
+
+# outputs = pipe(prompt, max_new_tokens=20, do_sample=True, top_k=10)
+# print(f"processing")
+
+# for output in outputs:
+#     print(f"Result: {output['generated_text']}")
+
+system_prompt = (
+    "You are a customer officer that helps extract transaction id from the USER INPUT and determine the transaction type based on the transaction id. "
+    "To find transaction id, follow all the steps below: "
+    "Step 1. **Look for prefix**: for each word in the USER INPUT, check if it starts with 'payout' or 'payment', if so, you should go to Step 2, otherwise go to Step 7. "
+    "Step 2. **Find a single dash**: check the character immediately after the prefix 'payout' or 'payment', if it is a dash, you should go to Step 3, otherwise go to Step 7.. "
+    "Step 3. **Find digits and characters**: check if there is at least one digit or one character after the dash, if so, you should go to Step 4, otherwise go to Step 7.. "
+    "Step 4. **Extract transaction id**: extract transaction id from the USER INPUT, then go to Step 5. "
+    "Step 5. **Verify transaction id**: verify the extracted transaction id by checking if the USER INPUT contains exact the same text, if so, you should go to Step 6, otherwise go to Step 7. "
+    "Step 6. It is a valid transaction id, when constructing the response, return the flag found as true. "
+    "Step 7. It is not a valid transaction id, when constructing the response, return the flag found as false. "
+    "To determine the transaction type, follow the rule below: "
+    "There are two types of transactions: "
+    "a. A transaction id starting with 'payout' indicates a payout transaction. "
+    "b. A transaction id starting with 'payment' indicates a payment transaction. "
+    "===>USER INPUT BEGINS"
+    "{input}"
+    "<===USER INPUT ENDS"
+    "Respond with the following 4 parameters in JSON format: "
+    "1. a mandatory flag named found, which indicates if a valid transaction id is found "
+    "2. the transaction id named transaction_id if found "
+    "3. the transaction type (in lower case) named transaction_type if found "
+    "4. a mandatory explanation named justification which explains how you deduce transaction id and transaction type, don't make up explanation if no exact text appears in the USER INPUT. "
+    "You must give answer in valid JSON format without any extra content"
+)
+
+examples = [
+    "My transaction payment-a1c1 failed", 
+    "Why is my withdrawal payout-b2c2 pending for 3 days", 
+    "There is an issue with my transaction payout-87l2k3",
+    "I am having trouble with my transaction",
+]
+
+def predict(message, history):
+    history.append({"role": "system", "content": system_prompt})
+    history.append({"role": "user", "content": message})
+    input_text = tokenizer.apply_chat_template(history, tokenize=False)
+    inputs = tokenizer.encode(input_text, return_tensors="pt").to(device)  
+    outputs = model.generate(inputs, max_new_tokens=100, temperature=0.2, top_p=0.9, do_sample=True)
+    decoded = tokenizer.decode(outputs[0])
+    response = decoded.split("<|im_start|>assistant\n")[-1].split("<|im_end|>")[0]
+    # response:  I'm ready to help you with your math homework. What's your math problem?
+    print(f"Response: {response}, outputs: {outputs}")
+    return response
+
+demo = gr.ChatInterface(predict, type="messages", examples=examples)
+
+demo.launch()
+

db.py

@@ -0,0 +1,54 @@
+import sqlite3
+from langchain_community.agent_toolkits.sql.base import create_sql_agent
+from langchain_community.agent_toolkits.sql.toolkit import SQLDatabaseToolkit
+from langchain_community.utilities import SQLDatabase
+from langchain.agents import AgentExecutor
+from langchain.agents.agent_types import AgentType
+
+CHAT_DB = 'chat.db'
+
+prompt_format_instructions = """Use the following format:
+
+Question: the input question you must answer
+Thought: you should always think about what to do
+Action: the action to take, should be one of [{tool_names}]
+Action Input: the input to the action
+Observation: the result of the action
+... (this Thought/Action/Action Input/Observation can repeat N times)
+Thought: I now know the final answer
+Final Answer: the final answer to the original input question, which should include the following 4 parameters: 1. a mandatory flag named found, if no record is found or "no records" is returned by your SQL query, return found as false, otherwise return as true. 2. the record found. 3. the table name named table_name from which a record is found. 4. a mandatory explanation named justification which contains the reason and SQL query yo used to search for record. You must give Final Answer in valid JSON format without any extra content.
+
+"""
+
+
+def prepare_data():
+    conn = sqlite3.connect(CHAT_DB)
+    c = conn.cursor()
+    c.execute('CREATE TABLE IF NOT EXISTS payment (id TEXT PRIMARY KEY, amount REAL, created_at TEXT)')
+    c.execute('CREATE TABLE IF NOT EXISTS payout (id TEXT PRIMARY KEY, amount REAL, created_at TEXT)')
+
+    c.execute('INSERT INTO payment (id, amount, created_at) VALUES (?, ?, ?) ON CONFLICT DO NOTHING', ('payment-a1c1', 100.0, '2021-01-01 00:00:00'))
+    c.execute('INSERT INTO payment (id, amount, created_at) VALUES (?, ?, ?) ON CONFLICT DO NOTHING', ('payment-b1c2', 200.0, '2021-01-02 00:00:00'))
+
+    c.execute('INSERT INTO payout (id, amount, created_at) VALUES (?, ?, ?) ON CONFLICT DO NOTHING', ('payout-a2c1', 50.0, '2021-01-01 00:00:00'))
+    c.execute('INSERT INTO payout (id, amount, created_at) VALUES (?, ?, ?) ON CONFLICT DO NOTHING', ('payout-b2c2', 100.0, '2021-01-02 00:00:00'))
+    conn.commit()
+    conn.close()
+
+def create_sql_agent_executor(llm):
+    db = SQLDatabase.from_uri(f"sqlite:///{CHAT_DB}")
+
+    toolkit = SQLDatabaseToolkit(db=db, llm=llm)
+    agent_executor = create_sql_agent(
+        format_instructions=prompt_format_instructions,
+        llm=llm,
+        toolkit=toolkit,
+        verbose=True,
+        agent_type=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
+        agent_executor_kwargs={'handle_parsing_errors':True},
+        early_stopping_method='force',
+        max_iterations=20,
+    )
+    
+    return agent_executor
+

requirements.txt

@@ -0,0 +1,17 @@
+
+python-dotenv>=1.0.1
+
+langchain>=0.3.20,<0.4.0
+langchain_community>=0.3.13
+langchain_core>=0.3.28
+langchain_experimental>=0.3.4
+
+huggingface-hub>=0.27.0
+langchain-huggingface>=0.1.2
+langchain-ollama>=0.2.3
+
+accelerate>=1.3.0
+
+streamlit>=1.34.0
+streamlit_datalist>=0.0.5
+transformers==4.49.0