Merge pull request #16 from fsa-simpleqt/TienTT

Phase2/TienTT: Add Query vector database and Delete tests

app/modules/crud_question_test/models/crud_question_tests.py

@@ -68,6 +68,8 @@ def create_question_test(data):
 def update_question_test(id, data):
     # Update a document by id
     firebase_db.collection("question_tests").document(id).update(data)
+    # Update corrensponding vector in Qdrant
+    
     return True
 
 def delete_question_test(id):
@@ -76,4 +78,19 @@ def delete_question_test(id):
     remove_file_question_tests(file_url)
     # Delete a document by id
     firebase_db.collection("question_tests").document(id).delete()
+
+    # Delete corresponding vector from Qdrant
+    qdrant_client.delete(
+        collection_name="question_tests",
+        points_selector=models.FilterSelector(
+            filter=models.Filter(
+                must=[
+                    models.FieldCondition(
+                        key="id",
+                        match=models.MatchValue(value=id),
+                    ),
+                ],
+            )
+        ),
+    )
     return True

app/modules/question_tests_retrieval/__init__.py

@@ -2,8 +2,8 @@ from fastapi import APIRouter, UploadFile, File
 from typing import Annotated
 
 from app.modules.question_tests_retrieval.models.jd2text import jobdes2text
-# from app.modules.question_tests_retrieval.models.text2tector import text2vector
-from app.modules.question_tests_retrieval.models.question_tests_logic import get_question_test
+# from app.modules.question_tests_retrieval.models.text2vector import text2vector
+from app.modules.question_tests_retrieval.models.question_tests_logic import get_question_tests
 
 qtretrieval_router = APIRouter(prefix="/qtretrieval", tags=["qtretrieval"])
 
@@ -18,10 +18,11 @@ async def send_jd(txt_file: Annotated[UploadFile, File(..., description="The JD
         # read the txt file with format
         jobdes = txt_file.file.read().decode("utf-8")
         sumaryjd_text = jobdes2text(jobdes)
-        if get_question_test(sumaryjd_text):
+        if get_question_tests(sumaryjd_text):
             return {"message": "Send JD successfully and get question test successfully",
                     "sumaryjd_text": sumaryjd_text}
         else:
             return {"message": "Please upload only .txt file", "error": str(e)}
     except Exception as e:
         return {"message": "Please upload only .txt file", "error": str(e)}
+

app/modules/question_tests_retrieval/models/question_tests_logic.py

@@ -5,8 +5,10 @@ from langchain_google_genai import GoogleGenerativeAIEmbeddings
 from langchain.evaluation import load_evaluator
 from langchain.evaluation import EmbeddingDistance
 
-from app.modules.crud_question_test.models.crud_question_tests import get_all_question_tests, get_question_test_url_by_description
+from app.modules.crud_question_test.models.crud_question_tests import get_question_test_by_id
+from app.modules.question_tests_retrieval.models.text2vector import text2vector
 from app.configs.database import firebase_bucket
+from app.configs.qdrant_db import qdrant_client
 
 # Import API key
 load_dotenv()
@@ -19,45 +21,84 @@ GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
 embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=GOOGLE_API_KEY)
 gemini_evaluator = load_evaluator("embedding_distance", distance_metric=EmbeddingDistance.COSINE, embeddings=embedding_model)
 
+# def compare_vector(vector_extract, vector_des):
+#     maxnimun_value = 2
+#     for item in vector_des:
+#         two_object = (vector_extract, item)
+#         x = gemini_evaluator.evaluate_strings(prediction=two_object[0], reference=two_object[1])
+#         if x.get('score') < maxnimun_value: 
+#             maxnimun_value = x.get('score')
+#             des_item_choose = item
+#     if maxnimun_value == 2:
+#         return False
+#     elif maxnimun_value < 0.3:
+#         return des_item_choose
+#     else:
+#         return False
+    
+def compare_vector(description_vector, max_number_of_points=3):
+    similarity_list = qdrant_client.search(
+        collection_name="question_tests",
+        query_vector=description_vector,
+        limit=max_number_of_points,
+        with_vectors=False,
+        with_payload=True,
+    )
 
-def compare_vector(vector_extract, vector_des):
-    maxnimun_value = 2
-    for item in vector_des:
-        two_object = (vector_extract, item)
-        x = gemini_evaluator.evaluate_strings(prediction=two_object[0], reference=two_object[1])
-        if x.get('score') < maxnimun_value: 
-            maxnimun_value = x.get('score')
-            des_item_choose = item
-    if maxnimun_value == 2:
-        return False
-    elif maxnimun_value < 0.3:
-        return des_item_choose
-    else:
-        return False
+    formatted_similarity_list = []
+    for point in similarity_list:
+        formatted_similarity_list.append({"id": point.payload.get("id"), "score": point.score})
+
+    return formatted_similarity_list
+
+# def download_question_test(question_test_url):
+#     # check folder exist
+#     if not os.path.exists('data/question_tests'):
+#         os.makedirs('data/question_tests')
+#     # download file from firebase storage using "gs://" link
+#     name_bucket = question_test_url.split(f"gs://{firebase_bucket.name}/")[1]
+#     blob = firebase_bucket.blob(name_bucket)
+#     blob.download_to_filename(f'data/question_tests/{name_bucket}')
+#     return True
 
-def download_question_test(question_test_url):
+def download_question_test(question_test_url_list):
     # check folder exist
     if not os.path.exists('data/question_tests'):
         os.makedirs('data/question_tests')
     # download file from firebase storage using "gs://" link
-    name_bucket = question_test_url.split(f"gs://{firebase_bucket.name}/")[1]
-    blob = firebase_bucket.blob(name_bucket)
-    blob.download_to_filename(f'data/question_tests/{name_bucket}')
+    for url in question_test_url_list:
+        name_bucket = url.split(f"gs://{firebase_bucket.name}/")[1]
+        blob = firebase_bucket.blob(name_bucket)
+        blob.download_to_filename(f'data/question_tests/{name_bucket}')
+
     return True
-    
 
 
-def get_question_test(text):
-    all_question_tests = get_all_question_tests()
-    value_in_des = []
-    for item in all_question_tests:
-        value_in_des.append(item['question_tests_description'])
-    des_item_choose = compare_vector(text, value_in_des)
-    if des_item_choose == False:
-        return "No question test found"
+# def get_question_test(text):
+#     all_question_tests = get_all_question_tests()
+#     value_in_des = []
+#     for item in all_question_tests:
+#         value_in_des.append(item['question_tests_description'])
+#     des_item_choose = compare_vector(text, value_in_des)
+#     if des_item_choose == False:
+#         return "No question test found"
+#     else:
+#         question_test_url = get_question_test_url_by_description(des_item_choose)
+#         if download_question_test(question_test_url):
+#             return True
+#         else:
+#             return False
+
+def get_question_tests(text):
+    # Get formatted similarity list
+    formatted_similarity_list = compare_vector(text2vector(text))
+    # Get corresponding document url in Firebase and download them
+    question_test_url_list = []
+    for point in formatted_similarity_list:
+        id = point.get("id")
+        question_test_url_list.append(get_question_test_by_id(id).get("question_tests_url"))
+
+    if download_question_test(question_test_url_list):
+        return True
     else:
-        question_test_url = get_question_test_url_by_description(des_item_choose)
-        if download_question_test(question_test_url):
-            return True
-        else:
-            return False
+        return False