Add project

Makefile

@@ -0,0 +1,4 @@
+##@ Clean-up
+clean: ## remove output files from pytest & coverage
+	@find . | grep -E "(__pycache__|\.pyc|\.pyo)" | xargs rm -rf
+	@find . | grep -E ".ipynb_checkpoints" | xargs rm -rf

app.py

@@ -0,0 +1,32 @@
+import gradio as gr
+
+from model.model import model
+
+# Making question examples
+example1 = "Apa dashboard terkait KPI operational?"
+example2 = "Apa link Dashboard Operational KPIs 2023?"
+example3 = "Mau lihat data performance cabang, ada di dashboard apa?"
+example4 = "Apa saja data di Dashboard Transactions?"
+
+# Making UI
+with gr.Blocks() as demo:
+  gr.Markdown(
+  """
+  # Chatbot Dashboard
+  This project is a chatbot based on LLM using the RAG method. 
+  This chatbot will answer questions related to the company's dashboard. 
+  You can ask questions according to the dashboard data below.
+  """)
+  gr.Interface(fn=model,
+             inputs="text",
+             outputs="text",
+             theme=gr.themes.Monochrome(),
+             examples = [example1, example2, example3, example4])
+  gr.Markdown(
+    """
+    ## Data dashboard
+    """)
+  gr.HTML("<img src='https://huggingface.co/spaces/Aldo07/chatbot_dashboard/resolve/main/data%20dashboard.png' width='500'>")
+
+demo.launch()
+

data/data.csv

@@ -0,0 +1,9 @@
+"Dashboard Financial KPIs 2023 merupakan dashboard berisi KPI Financial tahun 2023. Dashboard Financial KPIs 2023 berisi data terkait Expenses dan revenue di tahun 2023. 
+Link Dashboard Financial KPIs 2023 adalah https://dashboard.com/financial_kpi_2023/"
+Dashboard Operational KPIs 2023 merupakan dashboard KPI operational tahun 2023. Dashboard Operational KPIs 2023 berisi data terkait total deposit dan pinjaman di tahun 2023. Link Dashboard Operational KPIs 2023 adalah https://dashboard.com/operational_kpi_2023/
+"Dashboard Customer Satisfaction merupakan dashboard yang berisi terkait kepuasan pelanggan terhadap pelayanan. Skor kepuasan pelanggan dihitung berdasarkan tren, demografi,, tingkat respons, survei, dll. Link Dashboard Customer Satisfaction adalah https://dashboard.com/customer_satisfactione/"
+"Dashboard Teller Performance merupakan dashboard yang berisi informasi terkait performa teller. Dashboard Teller Performance berisi data terkait jumlah total customer yang dilayani, jumlah tiket yang diterbitkan per jam, dan rata-rata waktu yang dihabiskan customer di cabang, Link Dashboard Teller Performance adalah https://dashboard.com/teller_performance/"
+Dashboard Branch Performance merupakan dashboard yang berisi informasi terkait performa cabang. Link Dashboard Branch Performance adalah https://dashboard.com/branch_performance/
+Dashboard Transactions merupakan dashboard yang berisi informasi terkait seluruh transaksi yang terjadi. Dashboard Transactions berisi data terkait seluruh transaksi secara tahunan maupun historis. Link Dashboard Transactions adalah https://dashboard.com/transactions_report/
+Dashboard Marketing merupakan dashboard yang berisi informasi terkait marketing tahun ini dan secara historikal.Link Dashboard Marketing adalah https://dashboard.com/marketing_report/
+Dashboard Hardware Monitoring merupakan dashboard yang berisi informasi terkait performa hardware atau perangkat keras.Link Dashboard Hardware Monitoring adalah https://dashboard.com/hardware_monitoring/

model/config/config.py

@@ -0,0 +1,3 @@
+modelPath = 'firqaaa/indo-sentence-bert-base'
+model_kwargs = {'device':'cpu'}
+encode_kwargs = {'normalize_embeddings': False}

model/model.py

@@ -0,0 +1,39 @@
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain import HuggingFacePipeline
+from langchain.chains import RetrievalQA
+from sentence_transformers import SentenceTransformer
+import pandas as pd
+
+
+from model.config.config import modelPath, model_kwargs, encode_kwargs
+from model.processing.data_manager import text_chunk
+from model.processing.pipeline_qa import pipeline
+
+
+def embedding():
+    embeddings = HuggingFaceEmbeddings(
+            model_name=modelPath,
+            model_kwargs=model_kwargs,
+            encode_kwargs=encode_kwargs
+        )
+    
+    return embeddings
+
+
+def model(question):
+    text_chunks = text_chunk()
+    embeddings =embedding()
+
+    db = FAISS.from_documents(text_chunks, embeddings)
+    
+    pipe = pipeline()
+    
+    retriever = db.as_retriever(search_kwargs={"k": 1})
+    docs = retriever.get_relevant_documents(question)
+
+    answer = pipe({
+        'context': docs[0],
+        'question': question})
+
+    return answer

model/processing/data_manager.py

@@ -0,0 +1,26 @@
+import os
+from pathlib import Path
+
+import pandas as pd
+from langchain.document_loaders import DataFrameLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+def load_data():
+    
+    data_folder = Path("data")
+    data_path = os.path.join(data_folder, 'data.csv')
+    data = pd.read_csv(data_path, header=None, names=['description'])
+    data = pd.DataFrame(data)
+
+    return data
+
+def text_chunk():
+    data = load_data()
+    text_chunks = DataFrameLoader(
+        data, page_content_column="description").load_and_split(
+        text_splitter=RecursiveCharacterTextSplitter(
+            chunk_size=1000, chunk_overlap=0, length_function=len
+            ))
+    
+    return text_chunks

model/processing/pipeline_qa.py

@@ -0,0 +1,79 @@
+import itertools
+from typing import Dict, Union
+
+from nltk import sent_tokenize
+import nltk
+nltk.download('punkt')
+import torch
+from transformers import(
+    AutoModelForSeq2SeqLM, 
+    AutoTokenizer
+)
+
+class QAPipeline:
+
+    def __init__(
+        self
+    ):
+      
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("muchad/idt5-qa-qg")
+        self.tokenizer = AutoTokenizer.from_pretrained("muchad/idt5-qa-qg")
+        self.qg_format = "highlight"
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.model.to(self.device)
+        assert self.model.__class__.__name__ in ["T5ForConditionalGeneration"]
+        self.model_type = "t5"
+
+    def __call__(self, inputs: str):
+        inputs = " ".join(inputs.split())
+        answers = self._extract_answers(inputs)
+        flat_answers = list(itertools.chain(*answers))
+
+        if len(flat_answers) == 0:
+          return []
+    
+    def _tokenize(self,
+        inputs,
+        padding=True,
+        truncation=True,
+        add_special_tokens=True, 
+        max_length=512
+    ):
+        inputs = self.tokenizer.batch_encode_plus(
+            inputs, 
+            max_length=max_length,
+            add_special_tokens=add_special_tokens,
+            truncation=truncation,
+            padding="max_length" if padding else False,
+            pad_to_max_length=padding,
+            return_tensors="pt"
+        )
+        return inputs
+    
+class TaskPipeline(QAPipeline):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+    
+    def __call__(self, inputs: Union[Dict, str]):
+        return self._extract_answer(inputs["question"], inputs["context"])
+    
+    def _prepare_inputs(self, question, context):
+        source_text = f"question: {question}  context: {context}"
+        source_text = source_text + " </s>"
+        return  source_text
+    
+    def _extract_answer(self, question, context):
+        source_text = self._prepare_inputs(question, context)
+        inputs = self._tokenize([source_text], padding=False)
+    
+        outs = self.model.generate(
+            input_ids=inputs['input_ids'].to(self.device), 
+            attention_mask=inputs['attention_mask'].to(self.device), 
+            max_length=80,
+        )
+        answer = self.tokenizer.decode(outs[0], skip_special_tokens=True)
+        return answer
+
+def pipeline():    
+    task = TaskPipeline    
+    return task()

pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "chatbot data"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+readme = "README.md"
+packages = [{include = "model"}]
+
+
+[tool.poetry.dependencies]
+python = "^3.9"
+transformers = "^4.38.1"
+langchain = "^0.1.9"
+sentence-transformers = "^2.4.0"
+faiss-cpu = "^1.7.4"
+gradio = "^4.19.2"
+pandas = "^2.2.1"
+torch = "^2.2.1"
+nltk = "^3.8.1"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,8 @@
+transformers
+langchain
+sentence-transformers
+faiss-cpu
+gradio
+pandas
+torch
+nltk