Phase2/HuyDN: Optimization speed and fix bugs

app/configs/database.py

@@ -58,4 +58,10 @@ if "cvs" not in [collection.id for collection in firebase_db.collections()]:
 if "jds" not in [collection.id for collection in firebase_db.collections()]:
     # Create a collection
     firebase_db.collection("jds").add({"position_applied_for": "test","jd_url": "test"})
-    print("Collection jds created")
+    print("Collection jds created")
+
+# check if have rag_question_tests collection
+if "rag_question_tests" not in [collection.id for collection in firebase_db.collections()]:
+    # Create a collection
+    firebase_db.collection("rag_question_tests").add({"id_jd": "test","question_generator_tests_url": "test"})
+    print("Collection rag_question_tests created")

app/configs/qdrant_db.py

@@ -10,14 +10,21 @@ qdrant_client = QdrantClient(
     url = os.getenv("QDRANT_URL"), 
     api_key = os.getenv("QDRANT_API_KEY"),
 )
+print("Qdrant Database connected")
 
-try:
-    collection_info = qdrant_client.get_collection("question_tests")
-except Exception as e:
+# 2. Check if the question_tests exists
+if qdrant_client.collection_exists('question_tests') == False:
     qdrant_client.create_collection(
     collection_name="question_tests",
     vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
-)
-    
-print("Qdrant Database connected")
-
+    )
+    print("Collection question_tests created")
+# 3. Check if the rag_documents_test exists
+elif qdrant_client.collection_exists('rag_documents_test') == False:
+    qdrant_client.create_collection(
+    collection_name="rag_documents_test",
+    vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
+    )
+    print("Collection rag_documents_test created")
+else:
+    print("Collections already exist")

app/modules/__init__.py

@@ -6,6 +6,7 @@ from app.modules.matching_cv import cvmatching_router
 from app.modules.crud_question_test import crud_question_tests_router
 from app.modules.crud_cvs import crud_cvs_router
 from app.modules.crud_jds import crud_jds_router
+from app.modules.crud_rag_question_tests import crud_rag_question_tests_router
 
 modules_router = APIRouter(prefix="/modules", tags=["modules"])
 modules_router.include_router(qtretrieval_router)
@@ -13,6 +14,7 @@ modules_router.include_router(cvmatching_router)
 modules_router.include_router(crud_question_tests_router)
 modules_router.include_router(crud_cvs_router)
 modules_router.include_router(crud_jds_router)
+modules_router.include_router(crud_rag_question_tests_router)
 
 @modules_router.get("/")
 async def index():

app/modules/crud_cvs/__init__.py

@@ -21,7 +21,7 @@ async def add_cv(name_candidate: str, apply_position: str, file_cv: Annotated[Up
         file_cv_type = file_cv.filename.split(".")[-1]
         if file_cv_type in ["pdf", "docx"]:
             # create a new document
-            if create_cv({"name_candidate": name_candidate, "apply_position":apply_position, "cv_url": file_cv}):
+            if create_cv({"name_candidate": name_candidate, "apply_position":apply_position, "cv_content": file_cv}):
                 return {"message": "CV added successfully"}
             else:
                 return {"message": "Error while adding CV file to database"}

app/modules/crud_cvs/models/crud_cvs.py

@@ -1,16 +1,22 @@
 import uuid
-from app.configs.database import firebase_bucket, firebase_db
+import pytz
 import io
+import os
+
+from app.configs.database import firebase_bucket, firebase_db
 from docx import Document
+from datetime import datetime
+
+from langchain_community.document_loaders import UnstructuredPDFLoader
 
 # CRUD operation
-def upload_file_cvs(file):
-    re_name_file = str(uuid.uuid4()).replace("-","_") + "_" + file.filename
-    # upload file to firebase storage
-    blob = firebase_bucket.blob(re_name_file)
-    blob.upload_from_file(file.file)
+def upload_file_cvs(file_path):
+    # upload file to firebase storage from file_path
+    name_file = file_path.split("/")[-1]
+    blob = firebase_bucket.blob(name_file)
+    blob.upload_from_filename(file_path)
     # return gs link
-    return f"gs://{firebase_bucket.name}/{re_name_file}"
+    return f"gs://{firebase_bucket.name}/{name_file}"
 
 def remove_file_cvs(file_url):
     # remove file from firebase storage using "gs://" link
@@ -18,22 +24,28 @@ def remove_file_cvs(file_url):
     blob.delete()
     return True
 
-def file_cv_doc2text(file_url):
-    # download file from firebase storage using "gs://" link
-    blob = firebase_bucket.blob(file_url.split(f"gs://{firebase_bucket.name}/")[1])
-    # download file and return string in file
-    file_bytes = blob.download_as_bytes()
-    # Create a BytesIO object from the file bytes
-    file_stream = io.BytesIO(file_bytes)
-    # Read the .docx file from the BytesIO object
-    doc = Document(file_stream)
+def file_cv_doc2text(file_path):
+    # Read the .docx file from file
+    doc = Document(file_path)
     # Extract text from the .docx file
     text = ""
     for paragraph in doc.paragraphs:
         text += paragraph.text + "\n"
-    
     return text
 
+# def load cv from docx file
+def file_cv_pdf2text(file_path):
+    # Read the .pdf file from the BytesIO object
+    loader = UnstructuredPDFLoader(file_path)
+    json_result = loader.load()
+    # take page_content from json_result
+    page_content  = json_result[0].page_content
+    return page_content
+
+def get_cv_content_by_id(id_cv):
+    # Get a document by id
+    doc = firebase_db.collection("cvs").document(id_cv).get()
+    return doc.to_dict()["cv_content"]
 
 def get_all_cvs():
     # Get all documents from the collection
@@ -51,15 +63,45 @@ def get_cv_by_id(id):
     return doc.to_dict()
 
 def create_cv(data):
-    # get file_cvs
-    file_cvs = data["cv_url"]
+    # get file_cv
+    file_cv = data["cv_content"]
+    # rename file name to uuid
+    re_name_file = str(uuid.uuid4()).replace("-","_") + "_" + file_cv.filename
+    # save uploaded file to tmp folder
+    cache_path = f"tmp/{re_name_file}"
+    with open(cache_path, "wb") as buffer:
+        buffer.write(file_cv.file.read())
+    
+    # take file_cv and cv_upload type file
+    file_cv_type = file_cv.filename.split(".")[-1]
+    cv_text = ""
+    if file_cv_type == "pdf":
+        cv_text = file_cv_pdf2text(cache_path)
+    elif file_cv_type == "docx":
+        cv_text = file_cv_doc2text(cache_path)
+    else:
+        return False
+
     # upload file to firebase storage
-    file_url = upload_file_cvs(file_cvs)
+    cv_uploaded_url = upload_file_cvs(cache_path)
+    # delete file in tmp folder
+    os.remove(cache_path)
+
+    # Get the current time in UTC
+    utc_now = datetime.utcnow()
+    # Specify the Vietnam time zone
+    vietnam_timezone = pytz.timezone('Asia/Ho_Chi_Minh')
+    # Convert the current time to Vietnam time zone
+    vietnam_now = utc_now.replace(tzinfo=pytz.utc).astimezone(vietnam_timezone).strftime("%Y-%m-%d %H:%M:%S")
+
     # add file url to data
-    data["cv_url"] = file_url
+    data["cv_url"] = cv_uploaded_url
+    # add cv_content
+    data["cv_content"] = cv_text
+    # add created_at
+    data["created_at"] = vietnam_now
     # Create a new document
-    document_ref = firebase_db.collection("cvs").add(data)
-    # document_id = document_ref[1].id
+    firebase_db.collection("cvs").add(data)
     return True
 
 def delete_cv(id):

app/modules/crud_jds/__init__.py

@@ -14,13 +14,13 @@ async def index():
 
 # [POST] add JD
 @crud_jds_router.post("/")
-# only upload pdf or json file
+# only upload txt file
 async def add_jd(position_applied_for: str, file_jd: Annotated[UploadFile, File(..., description="Upload jd file (upload .txt)")]):
     try:
         file_jd_type = file_jd.filename.split(".")[-1]
         if file_jd_type in ["txt"]:
             # create a new document
-            if create_jd({"position_applied_for": position_applied_for,"jd_url": file_jd}):
+            if create_jd({"position_applied_for": position_applied_for,"jd_text": file_jd}):
                 return {"message": "JD added successfully"}
             else:
                 return {"message": "Error while adding JD file to database"}

app/modules/crud_jds/models/crud_jds.py

@@ -1,6 +1,8 @@
 import uuid
 from app.configs.database import firebase_bucket, firebase_db
-
+from datetime import datetime
+import pytz
+import os
 
 # CRUD operation
 def upload_file_jds(file):
@@ -17,11 +19,10 @@ def remove_file_jds(file_url):
     blob.delete()
     return True
 
-def file_jd_txt2text(file_url):
-    # download file from firebase storage using "gs://" link
-    blob = firebase_bucket.blob(file_url.split(f"gs://{firebase_bucket.name}/")[1])
-    # download file and return string in file
-    return blob.download_as_text()
+def get_jd_text_by_id(id_jd):
+    # Get a document by id
+    doc = firebase_db.collection("jds").document(id_jd).get()
+    return doc.to_dict()["jd_text"]
 
 def get_all_jds():
     # Get all documents from the collection
@@ -40,20 +41,40 @@ def get_jd_by_id(id):
 
 def create_jd(data):
     # get file_jds
-    file_jds = data["jd_url"]
-    # upload file to firebase storage
-    file_url = upload_file_jds(file_jds)
+    file_jds = data["jd_text"]
+    # change file name to uuid
+    re_name_file = str(uuid.uuid4()).replace("-","_") + "_" + file_jds.filename
+    # save uploaded file to tmp folder
+    with open(f"tmp/{re_name_file}", "wb") as buffer:
+        buffer.write(file_jds.file.read())
+    # read file
+    with open(f"tmp/{re_name_file}", "r", encoding="utf8") as file:
+        jd_text = file.read()
+    # delete file in tmp folder
+    os.remove(f"tmp/{re_name_file}")
+
+    # # upload file to firebase storage
+    # file_url = upload_file_jds(file_jds)
+
+    # Get the current time in UTC
+    utc_now = datetime.utcnow()
+    # Specify the Vietnam time zone
+    vietnam_timezone = pytz.timezone('Asia/Ho_Chi_Minh')
+    # Convert the current time to Vietnam time zone
+    vietnam_now = utc_now.replace(tzinfo=pytz.utc).astimezone(vietnam_timezone).strftime("%Y-%m-%d %H:%M:%S")
+
     # add file url to data
-    data["jd_url"] = file_url
+    data["jd_text"] = jd_text
+    # add created_at
+    data["created_at"] = vietnam_now
     # Create a new document
-    document_ref = firebase_db.collection("jds").add(data)
-    # document_id = document_ref[1].id
+    firebase_db.collection("jds").add(data)
     return True
 
 def delete_jd(id):
-    # Delete a file from firebase storage
-    file_url = get_jd_by_id(id)["jd_url"]
-    remove_file_jds(file_url)
+    # # Delete a file from firebase storage
+    # file_url = get_jd_by_id(id)["jd_url"]
+    # remove_file_jds(file_url)
     # Delete a document by id
     firebase_db.collection("jds").document(id).delete()
     return True

app/modules/crud_question_test/__init__.py

@@ -1,7 +1,7 @@
 from fastapi import APIRouter, UploadFile, File
 from typing import Annotated
 
-from app.modules.crud_question_test.models.crud_question_tests import get_all_question_tests, get_question_test_by_id, create_question_test, update_question_test, delete_question_test
+from app.modules.crud_question_test.models.crud_question_tests import get_all_question_tests, create_question_test, delete_question_test
 
 crud_question_tests_router = APIRouter(prefix="/crud_question_tests_router", tags=["crud_question_tests_router"])
 
@@ -15,21 +15,22 @@ async def index():
 # [POST] add question test
 @crud_question_tests_router.post("/")
 # only upload pdf or json file
-async def add_question_test(description: str, role: str, file_question_tests: Annotated[UploadFile, File(..., description="The question test file", media_type=["application/pdf", "application/json"])]):
+async def add_question_test(description: str, role: str, file_question_tests: Annotated[UploadFile, File(..., description="The question test file (Upload .pdf or .json)", media_type=["application/pdf", "application/json"])]):
     try:
+        question_tests_upload_type = file_question_tests.filename.split(".")[-1]
         # check if file is pdf or json
-        if file_question_tests.content_type == "application/pdf":
+        if question_tests_upload_type == "pdf":
             # create a new document
             if create_question_test({"question_tests_description": description, "question_tests_role": role, "question_tests_url": file_question_tests}):
                 return {"message": "Question test added successfully"}
             else:
-                return {"message": "Error"}
-        elif file_question_tests.content_type == "application/json":
+                return {"message": "Error", "error": str(e)}
+        elif question_tests_upload_type == "json":
             # create a new document
             if create_question_test({"question_tests_description": description, "question_tests_role": role, "question_tests_url": file_question_tests}):
                 return {"message": "Question test added successfully"}
             else:
-                return {"message": "Error"}
+                return {"message": "Error", "error": str(e)}
         else:
             return {"message": "File type not supported"}
     except Exception as e:

app/modules/crud_question_test/models/crud_question_tests.py

@@ -4,6 +4,8 @@ from app.configs.qdrant_db import qdrant_client
 from app.configs.qdrant_db import models
 from app.modules.question_tests_retrieval.models.text2vector import text2vector
 
+from datetime import datetime
+import pytz
 
 # CRUD operation
 def upload_file_question_tests(file):
@@ -48,6 +50,16 @@ def create_question_test(data):
     file_question_tests = data["question_tests_url"]
     # upload file to firebase storage
     file_url = upload_file_question_tests(file_question_tests)
+
+    # Get the current time in UTC
+    utc_now = datetime.utcnow()
+    # Specify the Vietnam time zone
+    vietnam_timezone = pytz.timezone('Asia/Ho_Chi_Minh')
+    # Convert the current time to Vietnam time zone
+    vietnam_now = utc_now.replace(tzinfo=pytz.utc).astimezone(vietnam_timezone).strftime("%Y-%m-%d %H:%M:%S")
+    # add created_at
+    data["created_at"] = vietnam_now
+
     # add file url to data
     data["question_tests_url"] = file_url
     question_tests_des = data["question_tests_description"]
@@ -65,13 +77,6 @@ def create_question_test(data):
 
     return True
 
-def update_question_test(id, data):
-    # Update a document by id
-    firebase_db.collection("question_tests").document(id).update(data)
-    # Update corrensponding vector in Qdrant
-    
-    return True
-
 def delete_question_test(id):
     # Delete a file from firebase storage
     file_url = get_question_test_by_id(id)["question_tests_url"]

app/modules/crud_rag_question_tests/__init__.py

@@ -0,0 +1,40 @@
+from fastapi import APIRouter, UploadFile, File
+from typing import Annotated
+
+from app.modules.crud_rag_question_tests.models.crud_rag_question_tests import get_all_rag_question_tests, create_rag_question_test, delete_question_test
+
+crud_rag_question_tests_router = APIRouter(prefix="/crud_rag_question_tests_router", tags=["crud_rag_question_tests_router"])
+
+# [GET] all question tests
+@crud_rag_question_tests_router.get("/")
+async def index():
+    # Get all documents from the collection with id document
+    data = get_all_rag_question_tests()
+    return data
+
+# [POST] add question test
+@crud_rag_question_tests_router.post("/")
+# only upload pdf or json file
+async def add_question_generator(id_jd: str, file_question_generator_tests: Annotated[UploadFile, File(..., description="The question generator test file", media_type=["application/json"])]):
+    try:
+        question_tests_upload_type = file_question_generator_tests.filename.split(".")[-1]
+        # check if file is json
+        if question_tests_upload_type == "json":
+            # create a new document
+            if create_rag_question_test({"id_jd": id_jd, "question_generator_tests_url": file_question_generator_tests}):
+                return {"message": "Question test added successfully"}
+            else:
+                return {"error": str(e)}
+        else:
+            return {"message": "File type not supported"}
+    except Exception as e:
+        return {"message": "Error", "error": str(e)}
+
+# [DELETE] question test by id
+@crud_rag_question_tests_router.delete("/{id}")
+async def delete_question_test_by_id(id: str):
+    # Delete a document by id
+    if delete_question_test(id):
+        return {"message": f"Question test have id {id} deleted successfully"}
+    else:
+        return {"message": "Error"}

app/modules/crud_rag_question_tests/models/crud_rag_question_tests.py

@@ -0,0 +1,64 @@
+import uuid
+from app.configs.database import firebase_bucket, firebase_db
+
+from datetime import datetime
+import pytz
+
+# CRUD operation
+def upload_file_rag_question_tests(file):
+    re_name_file = str(uuid.uuid4()).replace("-","_") + "_" + file.filename
+    # upload file to firebase storage
+    blob = firebase_bucket.blob(re_name_file)
+    blob.upload_from_file(file.file)
+    # return gs link
+    return f"gs://{firebase_bucket.name}/{re_name_file}"
+
+def remove_file_rag_question_tests(file_url):
+    # remove file from firebase storage using "gs://" link
+    blob = firebase_bucket.blob(file_url.split(f"gs://{firebase_bucket.name}/")[1])
+    blob.delete()
+    return True
+
+def get_all_rag_question_tests():
+    # Get all documents from the collection
+    docs = firebase_db.collection("rag_question_tests").stream()
+    data = []
+    for doc in docs:
+        doc_data = doc.to_dict()
+        doc_data["id"] = doc.id
+        data.append(doc_data)
+    return data
+
+def get_question_test_by_id(id):
+    # Get a document by id
+    doc = firebase_db.collection("rag_question_tests").document(id).get()
+    return doc.to_dict()
+
+def create_rag_question_test(data):
+    # get file_rag_question_tests
+    file_rag_question_tests = data["question_generator_tests_url"]
+    # upload file to firebase storage
+    file_url = upload_file_rag_question_tests(file_rag_question_tests)
+
+    # Get the current time in UTC
+    utc_now = datetime.utcnow()
+    # Specify the Vietnam time zone
+    vietnam_timezone = pytz.timezone('Asia/Ho_Chi_Minh')
+    # Convert the current time to Vietnam time zone
+    vietnam_now = utc_now.replace(tzinfo=pytz.utc).astimezone(vietnam_timezone).strftime("%Y-%m-%d %H:%M:%S")
+    # add created_at
+    data["created_at"] = vietnam_now
+
+    # add file url to data
+    data["question_generator_tests_url"] = file_url
+    # Create a new document
+    document_ref = firebase_db.collection("rag_question_tests").add(data)
+    return True
+
+def delete_question_test(id):
+    # Delete a file from firebase storage
+    file_url = get_question_test_by_id(id)["question_generator_tests_url"]
+    remove_file_rag_question_tests(file_url)
+    # Delete a document by id
+    firebase_db.collection("rag_question_tests").document(id).delete()
+    return True

app/modules/matching_cv/__init__.py

@@ -4,8 +4,8 @@ from fastapi import APIRouter
 # from app.modules.matching_cv.models.match_cv_jd_model import Match_JD_CV_Model
 
 from app.modules.matching_cv.models.matching_cv_logic import result_matching_cv_jd
-from app.modules.crud_jds.models.crud_jds import get_jd_by_id, file_jd_txt2text
-from app.modules.crud_cvs.models.crud_cvs import get_cv_by_id, file_cv_doc2text
+from app.modules.crud_jds.models.crud_jds import get_jd_by_id, get_jd_text_by_id
+from app.modules.crud_cvs.models.crud_cvs import get_cv_by_id, file_cv_doc2text, file_cv_pdf2text
 
 cvmatching_router = APIRouter(prefix="/cvmatching", tags=["cvmatching"])
 
@@ -17,42 +17,10 @@ async def index():
 # only upload .pdf or .docx file
 async def matching_cv_jd(id_jd: str, id_cv:str):
     try:
-        # get jd and cv by id
-        jd_document = get_jd_by_id(id_jd)
-        cv_document = get_cv_by_id(id_cv)
+        cv_content = get_cv_by_id(id_cv)
+        jd_text = get_jd_text_by_id(id_jd)
 
-        # download file from firebase storage
-        jd_url = jd_document["jd_url"]
-        cv_url = cv_document["cv_url"]
-
-        # get type file cv from cv_url "gs://bucket_name/file_name"
-        cv_type = cv_url.split(".")[-1]
-        if cv_type == "pdf":
-            return {"message": "This feature is not available yet"}
-        elif cv_type == "docx":
-            cv_text = file_cv_doc2text(cv_url)
-        else:
-            return {"message": "Please upload only .pdf or .docx file for CV"}
-
-        # get jd_text from jd_url "gs://bucket_name/file_name"
-        jd_text = file_jd_txt2text(jd_url)
-
-        result = result_matching_cv_jd(cv_text, jd_text)
+        result = result_matching_cv_jd(cv_text=cv_content,jd_text=jd_text)
         return {"result": result}
-        # # take jd_upload and cv_upload type file
-        # jd_upload_type = jd_upload.filename.split(".")[-1]
-        # cv_upload_type = cv_upload.filename.split(".")[-1]
-        # if jd_upload_type in ["txt"] and cv_upload_type in ["pdf", "docx"]:
-        #     jd_text =  jd_upload.file.read().decode("utf-8")
-        #     if cv_upload_type == "docx":
-        #         cv_text = docx.Document(cv_upload.file).paragraphs
-        #         cv_text = "\n".join([i.text for i in cv_text])
-        #     elif cv_upload_type == "pdf":
-        #         return {"message": "This feature is not available yet"}
-        #     # check matching cv and jd
-        #     result = result_matching_cv_jd(cv_text, jd_text)
-        #     return {"result": result}
-        # else:
-        #     return {"message": "Please upload only .txt for JD. And .pdf or .docx file for CV"}
     except Exception as e:
         return {"Error": str(e)}

app/modules/matching_cv/models/match_cv_jd_model.py

@@ -1,7 +0,0 @@
-from fastapi import APIRouter, UploadFile, File
-
-class Match_JD_CV_Model:
-    jd = UploadFile
-    jd_default = File(..., description="Upload JD file (only .txt file)", media_type=["text/plain"])
-    cv = UploadFile
-    cv_default = File(..., description="Upload CV file (only .pdf and .docx)", media_type=["application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"])

app/modules/matching_cv/models/matching_cv_logic.py

@@ -6,6 +6,8 @@ from dotenv import load_dotenv
 from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
 from langchain_core.messages import SystemMessage
 from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_anthropic import ChatAnthropic
+from langchain_openai import OpenAI
 
 # import the json oupput parser from the langchain core
 from langchain_core.output_parsers import JsonOutputParser
@@ -18,32 +20,38 @@ load_dotenv()
 
 # Define the google api key
 os.environ['GOOGLE_API_KEY'] = os.getenv('GOOGLE_API_KEY')
+os.environ['CLAUDE_API_KEY'] = os.getenv('CLAUDE_API_KEY')
+os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY')
 GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
+CLAUDE_API_KEY = os.environ.get("CLAUDE_API_KEY")
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+
+llm = ChatGoogleGenerativeAI(model="gemini-pro", temperature=0.3, convert_system_message_to_human=True, api_key=GOOGLE_API_KEY, request_timeout=120)
+# llm = ChatAnthropic(temperature=0.3, model_name="claude-3-opus-20240229", anthropic_api_key=CLAUDE_API_KEY, default_request_timeout=120)
+# llm = OpenAI(model_name="gpt-3.5-turbo-0125", openai_api_key=OPENAI_API_KEY)
+chain = llm | parser
+
+# create the prompt template
+chat_template = ChatPromptTemplate.from_messages(
+    [
+        SystemMessage(
+            content=(
+                """
+                Given the following CV and JD, calculate the percentage match between the candidate's qualifications and the job requirements:
+                CV: {cv}
+                JD: {jd}
+                To determine the match percentage, analyze the skills and experience in the CV and compare them to the requirements outlined in the JD. Provide the final match percentage as a numeric value between 0-100%, along with a brief explanation of your analysis. Follow this json format: {"Skills Match": {"Required Skills": "","Candidate  Skills": "","Match Percentage": "",}, "Experience Match": {"Required Experience": "","Candidate Experience": "","Match Percentage": "",}, "Overall Match Percentage:": "", "Explanation": ""}
+                """
+            )
+        ),
+        HumanMessagePromptTemplate.from_template(["{cv}", "{jd}"]),
+    ]
+)
 
 # def matching cv and jd return percentage of matching using prompt template
-def result_matching_cv_jd(cv_text, jd_text):
-    # create the prompt template
-    chat_template = ChatPromptTemplate.from_messages(
-        [
-            SystemMessage(
-                content=(
-                    """
-                    Given the following CV and JD, calculate the percentage match between the candidate's qualifications and the job requirements:
-                    CV: {cv}
-                    JD: {jd}
-                    To determine the match percentage, analyze the skills and experience in the CV and compare them to the requirements outlined in the JD. Provide the final match percentage as a numeric value between 0-100%, along with a brief explanation of your analysis. Follow this json format: {"Skills Match": {"Required Skills": "","Candidate  Skills": "","Match Percentage": "",}, "Experience Match": {"Required Experience": "","Candidate Experience": "","Match Percentage": "",}, "Overall Match Percentage:": "", "Explanation": ""}
-                    """
-                )
-            ),
-            HumanMessagePromptTemplate.from_template(["{cv}", "{jd}"]),
-        ]
-    )
-
+def result_matching_cv_jd(cv_text:str, jd_text:str):
     # create the chat message
     chat_message =  chat_template.format_messages(cv=cv_text, jd=jd_text)
-
-    llm = ChatGoogleGenerativeAI(model="gemini-pro", temperature=0.3, convert_system_message_to_human=True, api_key=GOOGLE_API_KEY, request_timeout=120)
-    chain = llm | parser
     result = chain.invoke(chat_message)
 
     return result

app/modules/question_tests_retrieval/__init__.py

@@ -1,9 +1,8 @@
-from fastapi import APIRouter, UploadFile, File
-from typing import Annotated
+from fastapi import APIRouter
 
 from app.modules.question_tests_retrieval.models.jd2text import jobdes2text
 from app.modules.question_tests_retrieval.models.question_tests_logic import get_question_tests
-from app.modules.crud_jds.models.crud_jds import get_jd_by_id, file_jd_txt2text
+from app.modules.crud_jds.models.crud_jds import get_jd_by_id, get_jd_text_by_id
 
 qtretrieval_router = APIRouter(prefix="/qtretrieval", tags=["qtretrieval"])
 
@@ -15,15 +14,13 @@ async def index():
 # only upload .txt file
 async def send_jd_to_get_question(id_jd: str):
     try:
-        jd_document = get_jd_by_id(id_jd)
-        # download jd file from firebase storage
-        jd_file_string = file_jd_txt2text(jd_document["jd_url"])
-        sumaryjd_text = jobdes2text(jd_file_string)
+        # get jd_text by id
+        jd_text = get_jd_text_by_id(id_jd)
+        sumaryjd_text = jobdes2text(jd_text)
         if get_question_tests(sumaryjd_text):
             return {"message": "Send JD successfully and get question test successfully",
-                    "sumaryjd_text": sumaryjd_text}
+                    "sumary JD": sumaryjd_text}
         else:
-            return {"message": "Please upload only .txt file", "error": str(e)}
+            return {"error": str(e)}
     except Exception as e:
         return {"message": "Have error when find JD in database", "error": str(e)}
-

app/modules/question_tests_retrieval/models/jd2text.py

@@ -2,6 +2,8 @@ from langchain_google_genai import ChatGoogleGenerativeAI
 from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
 from langchain_core.messages import SystemMessage
 from langchain_core.output_parsers import JsonOutputParser
+from langchain_anthropic import ChatAnthropic
+
 import os
 from dotenv import load_dotenv
 
@@ -10,33 +12,33 @@ load_dotenv()
 
 # Define the google api key
 os.environ['GOOGLE_API_KEY'] = os.getenv('GOOGLE_API_KEY')
+os.environ['CLAUDE_API_KEY'] = os.getenv('CLAUDE_API_KEY')
 GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
+CLAUDE_API_KEY = os.environ.get("CLAUDE_API_KEY")
 
 # define the parser object
 parser = JsonOutputParser()
 
-def jobdes2text(jobdes):
-    # setup the gemini pro
-    llm = ChatGoogleGenerativeAI(model="gemini-pro", temperature=0.3, convert_system_message_to_human=True, api_key=GOOGLE_API_KEY, request_timeout=120)
-
-    # create the prompt template
-    finnal_jd_chat_template = ChatPromptTemplate.from_messages(
-        [
-            SystemMessage(
-                content=(
-                    """Return Job title, level(Fresher, Junior, Senior, ...) and Brief summary of required skills about 20 words from the job description. Use the following format: Job Title is {job title}, Level is {level}, and Brief summary of required skills is {brief summary of required skills}."""
-                )
-            ),
-            HumanMessagePromptTemplate.from_template("{text}"),
-        ]
-    )
-
+# setup the gemini pro
+llm = ChatGoogleGenerativeAI(model="gemini-pro", temperature=0.3, convert_system_message_to_human=True, api_key=GOOGLE_API_KEY, request_timeout=120)
+# llm = ChatAnthropic(temperature=0, anthropic_api_key=CLAUDE_API_KEY, model_name="claude-3-opus-20240229")
+# create the prompt template
+finnal_jd_chat_template = ChatPromptTemplate.from_messages(
+    [
+        SystemMessage(
+            content=(
+                """Return Job title, level(Fresher, Junior, Senior, ...) and Brief summary of required skills about 20 words from the job description. Use the following format: Job Title is {job title}, Level is {level}, and Brief summary of required skills is {brief summary of required skills}."""
+            )
+        ),
+        HumanMessagePromptTemplate.from_template("{text}"),
+    ]
+)
+
+def jobdes2text(jobdes: str):
     # create the chat message
     chat_message =  finnal_jd_chat_template.format_messages(text=jobdes)
-
     # create a chain 
     chain =  llm
-
     result = chain.invoke(chat_message)
-
+    
     return result.content

app/modules/question_tests_retrieval/models/question_tests_logic.py

@@ -20,21 +20,6 @@ GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
 # Setting model embedding
 embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=GOOGLE_API_KEY, request_timeout=120)
 gemini_evaluator = load_evaluator("embedding_distance", distance_metric=EmbeddingDistance.COSINE, embeddings=embedding_model)
-
-# def compare_vector(vector_extract, vector_des):
-#     maxnimun_value = 2
-#     for item in vector_des:
-#         two_object = (vector_extract, item)
-#         x = gemini_evaluator.evaluate_strings(prediction=two_object[0], reference=two_object[1])
-#         if x.get('score') < maxnimun_value: 
-#             maxnimun_value = x.get('score')
-#             des_item_choose = item
-#     if maxnimun_value == 2:
-#         return False
-#     elif maxnimun_value < 0.3:
-#         return des_item_choose
-#     else:
-#         return False
     
 def compare_vector(description_vector, max_number_of_points=3):
     similarity_list = qdrant_client.search(
@@ -51,17 +36,7 @@ def compare_vector(description_vector, max_number_of_points=3):
 
     return formatted_similarity_list
 
-# def download_question_test(question_test_url):
-#     # check folder exist
-#     if not os.path.exists('data/question_tests'):
-#         os.makedirs('data/question_tests')
-#     # download file from firebase storage using "gs://" link
-#     name_bucket = question_test_url.split(f"gs://{firebase_bucket.name}/")[1]
-#     blob = firebase_bucket.blob(name_bucket)
-#     blob.download_to_filename(f'data/question_tests/{name_bucket}')
-#     return True
-
-def download_question_test(question_test_url_list):
+def download_question_test(question_test_url_list: list):
     # check folder exist
     if not os.path.exists('data/question_tests'):
         os.makedirs('data/question_tests')
@@ -73,23 +48,7 @@ def download_question_test(question_test_url_list):
 
     return True
 
-
-# def get_question_test(text):
-#     all_question_tests = get_all_question_tests()
-#     value_in_des = []
-#     for item in all_question_tests:
-#         value_in_des.append(item['question_tests_description'])
-#     des_item_choose = compare_vector(text, value_in_des)
-#     if des_item_choose == False:
-#         return "No question test found"
-#     else:
-#         question_test_url = get_question_test_url_by_description(des_item_choose)
-#         if download_question_test(question_test_url):
-#             return True
-#         else:
-#             return False
-
-def get_question_tests(text):
+def get_question_tests(text: str):
     # Get formatted similarity list
     formatted_similarity_list = compare_vector(text2vector(text))
     # Get corresponding document url in Firebase and download them

app/modules/question_tests_retrieval/models/text2vector.py

@@ -8,8 +8,8 @@ load_dotenv()
 # Define the google api key
 os.environ['GOOGLE_API_KEY'] = os.getenv('GOOGLE_API_KEY')
 GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
+embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=GOOGLE_API_KEY, request_timeout=120)
 
-def text2vector(text):
-    embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=GOOGLE_API_KEY, request_timeout=120)
+def text2vector(text: str):
     vector = embeddings.embed_query(text)
     return vector

scrapping.py

@@ -1,8 +0,0 @@
-from bs4 import BeautifulSoup
-import requests
-
-url = "https://en.wikipedia.org/wiki/List_of_largest_companies_by_revenue"
-page = requests.get(url)
-soup = BeautifulSoup(page.text, 'html')
-
-print(soup)