Phase3/QuangDT: Add question_rag module

app/modules/question_rag/models/question_rag_logic.py

@@ -1,16 +1,12 @@
-''' 
-Input : str of job description
-Output : str of quiz
-'''
-
 from langchain_google_genai import ChatGoogleGenerativeAI
-from langchain_community.document_loaders import Docx2txtLoader
 from langchain_google_genai import GoogleGenerativeAIEmbeddings
-from langchain_community.vectorstores import FAISS
-from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_core.prompts import ChatPromptTemplate
 from langchain.chains.combine_documents import create_stuff_documents_chain
 from langchain.chains import create_retrieval_chain
+from langchain_core.output_parsers import JsonOutputParser
+
+from langchain_community.vectorstores import Qdrant
+import qdrant_client
 
 import os
 from dotenv import load_dotenv
@@ -20,37 +16,46 @@ load_dotenv()
 
 # Define the google api key
 os.environ['GOOGLE_API_KEY'] = os.getenv('GOOGLE_API_KEY')
-GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
-
-# JOB_TEXT = "Job Title is Senior Python Software Engineer, Level is Senior, and Brief summary of required skills is 5+ years of professional Python development experience, Expertise in Python and its frameworks."
+os.environ['QDRANT_API_KEY'] = os.getenv('QDRANT_API_KEY')
+os.environ['QDRANT_URL'] = os.getenv('QDRANT_URL')
 
-def question_rag(jobtext):
-    llm = ChatGoogleGenerativeAI(model="gemini-pro")
-
-    loader = Docx2txtLoader("data/w3school_data.docx")
+GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
+QDRANT_API_KEY = os.environ.get("QDRANT_API_KEY")
+QDRANT_URL = os.environ.get("QDRANT_URL")
 
-    docs = loader.load()
+def question_rag(jobtext: str):
+    llm = ChatGoogleGenerativeAI(model="gemini-pro", google_api_key=GOOGLE_API_KEY, request_timeout=120)
 
     embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
 
-    text_splitter = RecursiveCharacterTextSplitter()
-    documents = text_splitter.split_documents(docs)
-    vector = FAISS.from_documents(documents, embeddings)
-
-    prompt = ChatPromptTemplate.from_template("""Answer the question based only on the following context:                                
+    client = qdrant_client.QdrantClient(
+        url=QDRANT_URL,
+        api_key=QDRANT_API_KEY,
+    )
+    
+    doc_store = Qdrant(
+        client=client,
+        collection_name="rag_documents_test", 
+        embeddings=embeddings,
+    )
+
+    json_parser = JsonOutputParser()
+
+    prompt = ChatPromptTemplate.from_template("""
+    Answer the question based only on the following context:                                
     <context>
     {context}
     </context>
                                                                             
     Generate a 10 quiz suitable for the given job description "{input}". Do not include "All of the above" answers.
     Output format is JSON:
-    ("count": 10, "data": ( "id": "", "question": "", "choices": [ "A. ", "B. ", "C.", "D. " ], "explanation": "", "answer": "", "level": "", "domain": "" )).
+    ("__count__": 10, "data": ( "id": "", "question": "", "choices": [ "A. ", "B. ", "C.", "D. " ], "explanation": "", "answer": "", "level": "", "domain": "" )).
     About level help me three levels: "Fresher, Junior, Senior".
-                                            """)
+    """)
 
-    document_chain = create_stuff_documents_chain(llm, prompt)
+    document_chain = create_stuff_documents_chain(llm, prompt, output_parser=json_parser)
 
-    retriever = vector.as_retriever()
+    retriever = doc_store.as_retriever()
     retrieval_chain = create_retrieval_chain(retriever, document_chain)
     response = retrieval_chain.invoke({"input": jobtext})
 

requirements.txt

@@ -13,7 +13,6 @@ colorama==0.4.6
 cryptography==42.0.5
 dataclasses-json==0.6.4
 dnspython==2.6.1
-docx2txt==0.8
 email_validator==2.1.1
 fastapi==0.110.0
 firebase-admin==6.4.0