Phase1/HuyDN: Done Base Phase 1

.gitignore

@@ -1,3 +1,4 @@
 venv/
 .env
 __pycache__/
+credentials/

app/configs/database.py

@@ -1,86 +1,43 @@
 import firebase_admin
-from firebase_admin import credentials, storage, db
-
-storageBucket = "fsa-firebase-tutorial.appspot.com"
-databaseURL = "https://fsa-firebase-tutorial-default-rtdb.asia-southeast1.firebasedatabase.app/"
-
-cred = credentials.Certificate('credentials\\fsa-firebase-database.json')
-firebase_admin.initialize_app(cred, {
-    'storageBucket': storageBucket,
-    'databaseURL': databaseURL
-})
-
-
-# Upload file to storage
-def upload_file(file_path, destination_path):
-    bucket = storage.bucket(storageBucket)
-    ref = bucket.blob(destination_path)
-    ref.upload_from_filename(file_path)
-    return True
-
-# Add file's url and metadata to realtime database
-def store_metadata(file_url, description):
-    database = db.reference('exam_data')  # Or your desired path
-    new_file_ref = database.push()
-    new_file_ref.set({
-        'file_url': file_url,
-        'description': description
+from firebase_admin import credentials
+from firebase_admin import firestore
+from firebase_admin import storage
+
+import os
+from dotenv import load_dotenv
+
+# load the environment variables
+load_dotenv()
+
+firebase_url_storageBucket = os.getenv("FIREBASE_URL_STORAGEBUCKET")
+
+# get credentials from .env
+credential_firebase = {
+    "type": os.getenv("FIREBASE_TYPE"),
+    "project_id": os.getenv("FIREBASE_PROJECT_ID"),
+    "private_key_id": os.getenv("FIREBASE_PRIVATE_KEY_ID"),
+    "private_key": os.getenv("FIREBASE_PRIVATE_KEY").replace('\\n', '\n'),
+    "client_email": os.getenv("FIREBASE_CLIENT_EMAIL"),
+    "client_id": os.getenv("FIREBASE_CLIENT_ID"),
+    "auth_uri": os.getenv("FIREBASE_AUTH_URI"),
+    "token_uri": os.getenv("FIREBASE_TOKEN_URI"),
+    "auth_provider_x509_cert_url": os.getenv("FIREBASE_AUTH_PROVIDER_X509_CERT_URL"),
+    "client_x509_cert_url": os.getenv("FIREBASE_CLIENT_X509_CERT_URL"),
+    "universe_domain": os.getenv("FIREBASE_UNIVERSE_DOMAIN")
+}
+
+# check if firebase is not initialized
+if not firebase_admin._apps:
+    # Initialize the app with a service account, granting admin privileges
+    cred = credentials.Certificate(credential_firebase)
+    firebase_admin.initialize_app(cred, {
+        'storageBucket': firebase_url_storageBucket
     })
-    return True
-
-# Query file's url and metadata from file's metadata
-def query_file_from_metadata(target_description):
-    temp_file_dict = {}
-    database_json = db.reference('exam_data').get()
-    for key, value in database_json.items():
-        if (target_description in value.get('description')):
-            temp_file_dict.update({key: value})
-    return temp_file_dict
-
-# Query database path
-def query_file(path):
-    database = db.reference(path)
-    ref = database.get()
-    return ref
-
-# Extract file's url from a bunch of things
-def extract_file_url(file_dict):
-    file_urls = []
-    for key, value in file_dict.items():
-        file_urls.append(value['file_url'])
-    return file_urls
-
-# Extract file's url with target description from a bunch of things
-def extract_file_url_by_description(file_dict, target_description):
-    file_urls = []
-    for key, value in file_dict.items():
-        if value.get('description') == target_description:
-            file_urls.append(value.get('file_url'))
-    return file_urls
 
-# Read file content, given its url
-def read_from_file_url(file_url_list):
-    temp_content_list = []
-    bucket = storage.bucket(storageBucket)
-    for file_url in file_url_list:
-        parts = []
-        parts = file_url.split('/')
-        legal_file_url = parts[3:][0]
-        ref = bucket.blob(legal_file_url)
-        file_content = ref.download_as_string().decode('utf-8')
-        temp_content_list.append(file_content)
-    return temp_content_list
+# Initialize Firestore
+firebase_db = firestore.client()
+print("Firestore connected")
 
-# Download file content, given its url
-def download_from_file_url(file_url_list, local_directory):
-    temp_content_list = []
-    bucket = storage.bucket(storageBucket)
-    for file_url in file_url_list:
-        parts = []
-        parts = file_url.split('/')
-        legal_file_url = parts[3:][0]
-        ref = bucket.blob(legal_file_url)
-        local_filename = f"{local_directory}/{legal_file_url}"
-        ref.download_to_filename(local_filename)
-        print(f"Downloaded {legal_file_url} to {local_filename}")
-    return True
+# Initialize Storage
+firebase_bucket = storage.bucket(app=firebase_admin.get_app())
+print("Storage connected")

app/modules/__init__.py

@@ -1,8 +1,10 @@
 from fastapi import APIRouter
-from app.modules.question_retrieval import qtretrieval_router
+from app.modules.question_tests_retrieval import qtretrieval_router
+from app.modules.crud_question_test import crud_question_tests_router
 
 modules_router = APIRouter(prefix="/modules", tags=["modules"])
 modules_router.include_router(qtretrieval_router)
+modules_router.include_router(crud_question_tests_router)
 
 @modules_router.get("/")
 async def index():

app/modules/crud_question_test/__init__.py

@@ -0,0 +1,45 @@
+from fastapi import APIRouter, UploadFile, File
+from typing import Annotated
+
+from app.modules.crud_question_test.models.crud_question_tests import get_all_question_tests, get_question_test_by_id, create_question_test, update_question_test, delete_question_test
+
+crud_question_tests_router = APIRouter(prefix="/crud_question_tests_router", tags=["crud_question_tests_router"])
+
+# [GET] all question tests
+@crud_question_tests_router.get("/")
+async def index():
+    # Get all documents from the collection with id document
+    data = get_all_question_tests()
+    return data
+
+# [POST] add question test
+@crud_question_tests_router.post("/")
+# only upload pdf or json file
+async def add_question_test(description: str, role: str, file_question_tests: Annotated[UploadFile, File(..., description="The question test file", media_type=["application/pdf", "application/json"])]):
+    try:
+        # check if file is pdf or json
+        if file_question_tests.content_type == "application/pdf":
+            # create a new document
+            if create_question_test({"description": description, "role": role, "question_tests": file_question_tests}):
+                return {"message": "Question test added successfully"}
+            else:
+                return {"message": "Error"}
+        elif file_question_tests.content_type == "application/json":
+            # create a new document
+            if create_question_test({"question_tests_description": description, "question_tests_role": role, "question_tests_url": file_question_tests}):
+                return {"message": "Question test added successfully"}
+            else:
+                return {"message": "Error"}
+        else:
+            return {"message": "File type not supported"}
+    except Exception as e:
+        return {"message": "Error", "error": str(e)}
+
+# [DELETE] question test by id
+@crud_question_tests_router.delete("/{id}")
+async def delete_question_test_by_id(id: str):
+    # Delete a document by id
+    if delete_question_test(id):
+        return {"message": f"Question test have id {id} deleted successfully"}
+    else:
+        return {"message": "Error"}

app/modules/crud_question_test/models/crud_question_tests.py

@@ -0,0 +1,62 @@
+from app.configs.database import firebase_bucket, firebase_db
+
+# CRUD operation
+def upload_file_question_tests(file):
+    # upload file to firebase storage
+    blob = firebase_bucket.blob(file.filename)
+    blob.upload_from_file(file.file)
+    # return gs link
+    return f"gs://{firebase_bucket.name}/{file.filename}"
+
+def remove_file_question_tests(file_url):
+    # remove file from firebase storage using "gs://" link
+    blob = firebase_bucket.blob(file_url.split(f"gs://{firebase_bucket.name}/")[1])
+    blob.delete()
+    return True
+
+def get_all_question_tests():
+    # Get all documents from the collection
+    docs = firebase_db.collection("question_tests").stream()
+    data = []
+    for doc in docs:
+        doc_data = doc.to_dict()
+        doc_data["id"] = doc.id
+        data.append(doc_data)
+    return data
+
+def get_question_test_by_id(id):
+    # Get a document by id
+    doc = firebase_db.collection("question_tests").document(id).get()
+    return doc.to_dict()
+
+def get_question_test_url_by_description(description):
+    # Get a question_tests_url where question_tests_description is equal to description
+    docs = firebase_db.collection("question_tests").where("question_tests_description", "==", description).stream()
+    for doc in docs:
+        return doc.to_dict()["question_tests_url"]
+    return False
+
+
+def create_question_test(data):
+    # get file_question_tests
+    file_question_tests = data["question_tests_url"]
+    # upload file to firebase storage
+    file_url = upload_file_question_tests(file_question_tests)
+    # add file url to data
+    data["question_tests_url"] = file_url
+    # Create a new document
+    firebase_db.collection("question_tests").add(data)
+    return True
+
+def update_question_test(id, data):
+    # Update a document by id
+    firebase_db.collection("question_tests").document(id).update(data)
+    return True
+
+def delete_question_test(id):
+    # Delete a file from firebase storage
+    file_url = get_question_test_by_id(id)["question_tests"]
+    remove_file_question_tests(file_url)
+    # Delete a document by id
+    firebase_db.collection("question_tests").document(id).delete()
+    return True

app/modules/{question_retrieval → question_tests_retrieval}/__init__.py

@@ -1,8 +1,9 @@
 from fastapi import APIRouter, UploadFile, File
 from typing import Annotated
 
-from app.modules.question_retrieval.models.jd2text import jobdes2text
-from app.modules.question_retrieval.models.text2tector import text2vector
+from app.modules.question_tests_retrieval.models.jd2text import jobdes2text
+from app.modules.question_tests_retrieval.models.text2tector import text2vector
+from app.modules.question_tests_retrieval.models.question_tests_logic import get_question_test
 
 qtretrieval_router = APIRouter(prefix="/qtretrieval", tags=["qtretrieval"])
 
@@ -14,12 +15,16 @@ async def index():
 # only upload .txt file
 async def send_jd(txt_file: Annotated[UploadFile, File(..., description="The JD file", media_type=["text/plain"])]):
     try:
-        # read the txt file with format
-        jobdes = txt_file.file.read().decode("utf-8")
-        sumaryjd_text = jobdes2text(jobdes)
-        print("sumaryjd_text: ", sumaryjd_text)
-        sumaryjd_vector = text2vector(sumaryjd_text)
-        print("sumaryjd_vector: ", sumaryjd_vector)
-        return {"message": "Send JD successfully"}
+        # # read the txt file with format
+        # jobdes = txt_file.file.read().decode("utf-8")
+        # sumaryjd_text = jobdes2text(jobdes)
+        # print("sumaryjd_text: ", sumaryjd_text)
+        # sumaryjd_vector = text2vector(sumaryjd_text)
+        # print("sumaryjd_vector: ", sumaryjd_vector)
+        text = "Job Title is Senior AI Engineer, Level is Senior, and Brief summary of required skills is NLP, experiencing in using containers"
+        if get_question_test(text):
+            return {"message": "Send JD successfully and get question test successfully"}
+        else:
+            return {"message": "Error"}
     except Exception as e:
         return {"message": "Error", "error": str(e)}

app/modules/question_tests_retrieval/models/question_tests_logic.py

@@ -0,0 +1,65 @@
+import os
+from dotenv import load_dotenv
+
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from langchain.evaluation import load_evaluator
+from langchain.evaluation import EmbeddingDistance
+
+from app.modules.crud_question_test.models.crud_question_tests import get_all_question_tests, get_question_test_url_by_description
+from app.configs.database import firebase_bucket
+
+# Import API key
+load_dotenv()
+
+# Define the google api key
+os.environ['GOOGLE_API_KEY'] = os.getenv('GOOGLE_API_KEY')
+GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
+
+# Setting model embedding
+embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=GOOGLE_API_KEY)
+gemini_evaluator = load_evaluator("embedding_distance", distance_metric=EmbeddingDistance.COSINE, embeddings=embedding_model)
+
+
+def compare_vector(vector_extract, vector_des):
+    maxnimun_value = 2
+    for item in vector_des:
+        two_object = (vector_extract, item)
+        x = gemini_evaluator.evaluate_strings(prediction=two_object[0], reference=two_object[1])
+        # print(f"item and score: {item} - {x.get('score')}")
+        if x.get('score') < maxnimun_value: 
+            maxnimun_value = x.get('score')
+            print("maxnimun_value: ", maxnimun_value)
+            des_item_choose = item
+    if maxnimun_value == 2:
+        return False
+    elif maxnimun_value < 0.3:
+        return des_item_choose
+    else:
+        return False
+
+def download_question_test(question_test_url):
+    # check folder exist
+    if not os.path.exists('data/question_tests'):
+        os.makedirs('data/question_tests')
+    # download file from firebase storage using "gs://" link
+    name_bucket = question_test_url.split(f"gs://{firebase_bucket.name}/")[1]
+    blob = firebase_bucket.blob(name_bucket)
+    blob.download_to_filename(f'data/question_tests/{name_bucket}')
+    return True
+    
+
+
+def get_question_test(text):
+    all_question_tests = get_all_question_tests()
+    value_in_des = []
+    for item in all_question_tests:
+        value_in_des.append(item['question_tests_description'])
+    des_item_choose = compare_vector(text, value_in_des)
+    if des_item_choose == False:
+        return "No question test found"
+    else:
+        question_test_url = get_question_test_url_by_description(des_item_choose)
+        if download_question_test(question_test_url):
+            return True
+        else:
+            return False

app/modules/{question_retrieval → question_tests_retrieval}/models/text2tector.py

@@ -1,7 +1,10 @@
 import os
 
 from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from dotenv import load_dotenv
 
+# load the environment variables
+load_dotenv()
 # Define the google api key
 os.environ['GOOGLE_API_KEY'] = os.getenv('GOOGLE_API_KEY')
 GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")

credentials/fsa-firebase-database.json

@@ -1,13 +0,0 @@
-{
-  "type": "service_account",
-  "project_id": "fsa-firebase-tutorial",
-  "private_key_id": "89175f3dd111886b549bda9d5a8163ed1fe83900",
-  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDDUXR7JQkfiZRe\nkAFNutX4eIeJxrb00Hpkkbe9Doj+M+aYRrPyXsjEn9o0ZGeomSUwVqDHQ2Tby0lS\nlarvYnML7RaE5bIcIMuZ8CDyKsu9ZyxMIlOm4deJnOP9IfN4ascHxiiMHJBsHsLG\nVHQqmx3XJrfRVGELsxLF/0+fwQOdSiPPhU4MKxqmOjHmCLC9IJLnhrAKyQ1LgV+l\n+6fXc7EGtRVdjUCwQGZMv11UaJbjQqZSCFRpabsyw0/RenPmdyYa35R8F48YaiDR\n/WdxqqMTNbZrbhigiz3rIZmropyUdoPT/fKKZstP2vpFj8PbmtgThqlFEAo5l/d8\n5dgdHXCJAgMBAAECggEADeLexooKqOG3yRTRKOjzgTW1gi288HEZgIitIN11jyga\nuQEO1liXisYoJfEs/vs7XgFMrzoLA8bjTMIsC9aqogKORy7K783Dl+tO9lSlVU8O\nYs/OdXcJQ0UCnwqwxAMIaTS1CYgW2v8BX0ceFrDeSV988l6ewy2NNEM7cprPAlnE\n65uO5Es3/R3pLyKORwxkvfY3AVWksXR8FtUdxyMSLjdLqNZ0jxdltMttzyHSvTmS\nLBvXoEznI8bEvvrQwZryOYPvM0sHQSlzyFpJbx/WhBfZ/isZ5AwXZEO82g8zQE2c\nTnK50Wu8AjtynlZhRPEEvu6//2yE9ufCK64/kxcT0QKBgQD6eUn80r4UEWWJUCNU\noLjsuvJAS3usQwwLCrq/FZSx4b53RdA0BADy4n1u0+6ps1EmnN3bdSiuFCZ9DltL\neP4uhwUtz4pJEnFeVUx1SuzzqPn64LKukrGkYqSma8XXx2ZmUp54RsRo8cTU8oAw\nkuipIOj/ynDXbiGN/X0w827seQKBgQDHoKOx/K/sg1nBUHESJudAgpTrSKwUIMgL\nc/qHLanuh2pnK320tqJtkt3hsmC6fiwIU8kX/hSXiB81aZT02rAlZS8eQOftReg5\nlHF/m2B5PPQvamSZ4XUAon1XliXB1z3q/uw72Qc+KzYxUFdb/mBugXYEgo2md7eZ\nCJlaUDWAkQKBgQDERaT3q8BtA1DYb+avmx4tyzSnsz6fNw7J3EeIEwYo+eeBmUBU\nckq4CUkXPpWHf094/XVQUhdlib67QjPCCXbMyNsDEd2f8wbJT/gQ9XyGVwREVi1O\nZP80/GNfUZZ/xu30AovRXkrsnX+LCtfuuD/+wB40ytLvzYoXtOqUID3sqQKBgAym\ncFkiEx/gvvpSF6g30OW0oK08QLNR1HttUQ3p5d/YsvRwvBmMjUR2TxszS4/l2PJ6\nJL8hebqUhK3E0AyoIwtrmw3GvSu7B1lGv95/QSDNrQeoLVHXCKwOn/TB7giXlDPl\nDaFJQn09DcUYO0D4ILI///OPNly7L2ZCNj2T4aARAoGANEfSQY9v2X21VcuKf5Gl\nIZ0682yvtUmkX0EF8mNSz11GIscWGjQWtEi1v+BSYB0By4mHkg67lOjKcYfaFwqT\nYCPsJgl9pfreraT7hDt8CjJo6QfIcgNhbseCyTF3qqUxQ47aN9c9n0YCd/p8lCvB\ndcbch0+stt5cRoZwXcxGSEo=\n-----END PRIVATE KEY-----\n",
-  "client_email": "firebase-adminsdk-yn0hp@fsa-firebase-tutorial.iam.gserviceaccount.com",
-  "client_id": "104598090077479952875",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-yn0hp%40fsa-firebase-tutorial.iam.gserviceaccount.com",
-  "universe_domain": "googleapis.com"
-}

data/question_tests/Question_AI_Senior_1.json

@@ -0,0 +1,355 @@
+{
+    "__count__": 30,
+    "data": [
+        {
+            "id": "1",
+            "question": "What is the concept of 'Curse of Dimensionality' in Machine Learning?",
+            "choices": [
+                "A. It refers to the difficulty in visualizing high-dimensional data.",
+                "B. It refers to the phenomenon where the feature space becomes increasingly sparse for an increasing number of dimensions of a fixed-size training dataset.",
+                "C. It refers to the computational complexity of mathematical operations in high-dimensional spaces.",
+                "D. It refers to the overfitting that occurs when the model is too complex."
+            ],
+            "explanation": "The 'Curse of Dimensionality' refers to various phenomena that arise when analyzing and organizing data in high-dimensional spaces (often with hundreds or thousands of dimensions) that do not occur in low-dimensional settings such as the three-dimensional physical space of everyday experience.",
+            "answer": "B",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "2",
+            "question": "What is the difference between 'one-hot encoding' and 'label encoding'?",
+            "choices": [
+                "A. 'One-hot encoding' is used for ordinal variables, while 'label encoding' is used for nominal variables.",
+                "B. 'Label encoding' is used for ordinal variables, while 'one-hot encoding' is used for nominal variables.",
+                "C. Both 'one-hot encoding' and 'label encoding' are used for ordinal variables.",
+                "D. Both 'one-hot encoding' and 'label encoding' are used for nominal variables."
+            ],
+            "explanation": "'Label encoding' and 'one-hot encoding' are two common methods used to convert categorical data into numerical data. 'Label encoding' assigns each unique category in a categorical variable with an integer. No new columns are created. On the other hand, 'one-hot encoding' creates new (binary) columns, indicating the presence of each possible value from the original data.",
+            "answer": "B",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "3",
+            "question": "What is 'Early Stopping' in model training?",
+            "choices": [
+                "A. It's a form of regularization used to avoid overfitting when training a learner with an iterative method.",
+                "B. It's a technique used to speed up training by stopping the training process before it completes all iterations.",
+                "C. It's a method used to stop training when the model's performance starts to decrease.",
+                "D. It's a technique used to stop training when the model's performance does not improve on an held-out validation set."
+            ],
+            "explanation": "'Early stopping' is a form of regularization used to avoid overfitting when training a machine learning model with an iterative method, such as gradient descent. Such methods update the model's parameters (such as the weights in a neural network) iteratively, and early stopping halts this iterative process when the model's performance stops improving on a held-out validation dataset.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "4",
+            "question": "What is the difference between 'Random Forest' and 'Gradient Boosting' algorithms?",
+            "choices": [
+                "A. 'Random Forest' is a bagging algorithm, while 'Gradient Boosting' is a boosting algorithm.",
+                "B. 'Gradient Boosting' is a bagging algorithm, while 'Random Forest' is a boosting algorithm.",
+                "C. Both 'Random Forest' and 'Gradient Boosting' are bagging algorithms.",
+                "D. Both 'Random Forest' and 'Gradient Boosting' are boosting algorithms."
+            ],
+            "explanation": "'Random Forest' is a bagging algorithm and 'Gradient Boosting' is a boosting algorithm. Both are ensemble methods, but they combine the models in different ways. In 'Random Forest', each tree in the ensemble is built from a sample drawn with replacement (i.e., a bootstrap sample) from the training set. In 'Gradient Boosting', each new tree is fit on a modified version of the original data set.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "5",
+            "question": "What is 'Transfer Learning' in the context of Machine Learning?",
+            "choices": [
+                "A. It's a technique where a pre-trained model is used on a new problem.",
+                "B. It's a technique to transfer knowledge from one model to another.",
+                "C. It's a technique to transfer knowledge from one problem domain to another.",
+                "D. All of the above."
+            ],
+            "explanation": "'Transfer Learning' is a research problem in machine learning that focuses on storing knowledge gained while solving one problem and applying it to a different but related problem. For example, knowledge gained while learning to recognize cars could apply when trying to recognize trucks.",
+            "answer": "D",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "6",
+            "question": "What is the purpose of 'ReLU' in a neural network?",
+            "choices": [
+                "A. To introduce non-linearity in the neural network.",
+                "B. To normalize the output of the neural network.",
+                "C. To speed up the training process of the neural network.",
+                "D. To prevent overfitting in the neural network."
+            ],
+            "explanation": "'ReLU' stands for 'Rectified Linear Unit'. It is the most commonly used activation function in neural networks and deep learning models. The function returns 0 if it receives any negative input, but for any positive value 'x' it returns that value back. It's used to introduce non-linearity in the neural network.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "7",
+            "question": "What is 'Batch Normalization' in the context of Neural Networks?",
+            "choices": [
+                "A. It's a technique to provide any layer in a neural network with inputs that are zero mean/unit variance.",
+                "B. It's a technique to normalize the output of a previous activation layer by subtracting the batch mean and dividing by the batch standard deviation.",
+                "C. It's a technique to make the weights of a neural network have zero mean and unit variance.",
+                "D. Both A and B."
+            ],
+            "explanation": "'Batch Normalization' is a technique to provide any layer in a neural network with inputs that are zero mean/unit variance, and it is a technique to normalize the output of the previous activation layer by subtracting the batch mean and dividing by the batch standard deviation. This technique helps to speed up learning in deep neural networks by reducing internal covariate shift, and it has become a standard component of most state-of-the-art neural networks.",
+            "answer": "D",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "8",
+            "question": "What is the purpose of 'L1' and 'L2' regularization?",
+            "choices": [
+                "A. They are techniques to prevent overfitting in a machine learning model.",
+                "B. They are techniques to increase the speed of training a machine learning model.",
+                "C. They are techniques to increase the complexity of a machine learning model.",
+                "D. They are techniques to decrease the complexity of a machine learning model."
+            ],
+            "explanation": "'L1' and 'L2' are regularization techniques used to prevent overfitting in a machine learning model by adding a penalty term to the loss function. The penalty term encourages the model to have smaller weights, which makes the model simpler and thus less likely to overfit.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "9",
+            "question": "What is 'Ensemble Learning' in the context of Machine Learning?",
+            "choices": [
+                "A. It's a technique where multiple models are trained to solve the same problem and combined to get better results.",
+                "B. It's a technique where one model is trained to solve multiple problems.",
+                "C. It's a technique where the model is trained on an ensemble of different datasets.",
+                "D. It's a technique where the model is trained multiple times on the same dataset."
+            ],
+            "explanation": "'Ensemble Learning' is a machine learning paradigm where multiple models (often called 'weak learners') are trained to solve the same problem and combined to get better results. The main hypothesis is that when weak models are correctly combined we can obtain more accurate and/or robust models.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "10",
+            "question": "What is the difference between 'Ridge' and 'Lasso' regression?",
+            "choices": [
+                "A. 'Ridge' regression uses L1 regularization while 'Lasso' regression uses L2 regularization.",
+                "B. 'Lasso' regression uses L1 regularization while 'Ridge' regression uses L2 regularization.",
+                "C. Both 'Ridge' and 'Lasso' regression use L1 regularization.",
+                "D. Both 'Ridge' and 'Lasso' regression use L2 regularization."
+            ],
+            "explanation": "'Ridge' and 'Lasso' regression are two types of linear regression models that use different types of regularization. 'Ridge' regression uses L2 regularization, which adds a penalty equal to the square of the magnitude of coefficients. On the other hand, 'Lasso' regression uses L1 regularization, which adds a penalty equal to the absolute value of the magnitude of coefficients.",
+            "answer": "B",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "11",
+            "question": "What is 'Data Augmentation' in the context of Machine Learning?",
+            "choices": [
+                "A. It's a technique to artificially create new training data from existing training data.",
+                "B. It's a technique to increase the size of the dataset by collecting more data.",
+                "C. It's a technique to clean the training data.",
+                "D. It's a technique to reduce the size of the dataset."
+            ],
+            "explanation": "'Data Augmentation' is a strategy that enables practitioners to significantly increase the diversity of data available for training models, without actually collecting new data. Data augmentation techniques such as cropping, padding, and horizontal flipping are commonly used to train large neural networks.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "12",
+            "question": "What is the purpose of 'Max Pooling' in a Convolutional Neural Network (CNN)?",
+            "choices": [
+                "A. To reduce the spatial dimensions of the output volume.",
+                "B. To increase the spatial dimensions of the output volume.",
+                "C. To normalize the output of the previous activation layer.",
+                "D. To introduce non-linearity in the neural network."
+            ],
+            "explanation": "'Max Pooling' is a pooling operation that is typically added to CNNs following individual convolutional layers. When added to a model, max pooling reduces the dimensionality of images by reducing the number of pixels in the output from the previous convolutional layer.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "13",
+            "question": "What is the difference between 'Batch Gradient Descent' and 'Mini-Batch Gradient Descent'?",
+            "choices": [
+                "A. 'Batch Gradient Descent' uses the entire training set to compute the gradient of the cost function, while 'Mini-Batch Gradient Descent' uses a subset of the training set.",
+                "B. 'Mini-Batch Gradient Descent' uses the entire training set to compute the gradient of the cost function, while 'Batch Gradient Descent' uses a subset of the training set.",
+                "C. Both 'Batch Gradient Descent' and 'Mini-Batch Gradient Descent' use the entire training set to compute the gradient of the cost function.",
+                "D. Both 'Batch Gradient Descent' and 'Mini-Batch Gradient Descent' use a subset of the training set to compute the gradient of the cost function."
+            ],
+            "explanation": "'Batch Gradient Descent' uses the entire training set to compute the gradient of the cost function, while 'Mini-Batch Gradient Descent' uses a subset of the training set. With 'Mini-Batch Gradient Descent', you can replace the actual gradient (calculated from the entire data set) with an estimate of the gradient (calculated from a randomly selected subset of the data). Especially in big data applications, this can help to speed up gradient-based optimization algorithms significantly.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "14",
+            "question": "What is 'Principal Component Analysis' (PCA) used for?",
+            "choices": [
+                "A. PCA is used to compress the data by reducing the number of dimensions.",
+                "B. PCA is used to decompress the data by increasing the number of dimensions.",
+                "C. PCA is used to classify the data into different categories.",
+                "D. PCA is used to cluster the data into different groups."
+            ],
+            "explanation": "'Principal Component Analysis' (PCA) is a dimensionality reduction technique that is commonly used in machine learning and data visualization. It can be thought of as a projection method where data with 'm' columns (features) is projected into a subspace with 'm' or fewer columns, whilst retaining the essence of the original data.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "15",
+            "question": "What is the purpose of 'Word Embeddings' in Natural Language Processing (NLP)?",
+            "choices": [
+                "A. To map words or phrases from the vocabulary to vectors of real numbers.",
+                "B. To map words or phrases from the vocabulary to a dictionary of words.",
+                "C. To convert the words in the vocabulary to lower case.",
+                "D. To remove stop words from the vocabulary."
+            ],
+            "explanation": "'Word Embeddings' are a type of word representation that allows words with similar meaning to have a similar representation. They are a distributed representation for text that is perhaps one of the key breakthroughs for the impressive performance of deep learning methods on challenging natural language processing problems.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "16",
+            "question": "What is the difference between 'Long Short Term Memory' (LSTM) and 'Gated Recurrent Unit' (GRU)?",
+            "choices": [
+                "A. LSTM has three gates (input, output, forget) while GRU has two gates (reset, update).",
+                "B. GRU has three gates (input, output, forget) while LSTM has two gates (reset, update).",
+                "C. Both LSTM and GRU have three gates (input, output, forget).",
+                "D. Both LSTM and GRU have two gates (reset, update)."
+            ],
+            "explanation": "Both LSTM (Long Short Term Memory) and GRU (Gated Recurrent Unit) are types of recurrent neural network (RNN) architecture used in deep learning. The key difference between them is that LSTM has three gates (input, output, forget), while GRU has two gates (reset, update). This makes GRUs a simpler and more efficient model for certain tasks.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "17",
+            "question": "What is 'Autoencoder' in the context of Machine Learning?",
+            "choices": [
+                "A. It's a type of artificial neural network used for learning efficient codings of input data.",
+                "B. It's a type of artificial neural network used for generating new data that is similar to the input data.",
+                "C. It's a type of artificial neural network used for classifying input data into different categories.",
+                "D. It's a type of artificial neural network used for clustering input data into different groups."
+            ],
+            "explanation": "An 'Autoencoder' is a type of artificial neural network used for learning efficient codings of input data. It's typically used for the purpose of dimensionality reduction and feature learning.",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "18",
+            "question": "What is the purpose of 'Attention Mechanism' in the context of Machine Learning?",
+            "choices": [
+                "A. It's used to focus on certain parts of the input data that are more relevant to the task at hand.",
+                "B. It's used to pay equal attention to all parts of the input data.",
+                "C. It's used to ignore certain parts of the input data that are not relevant to the task at hand.",
+                "D. Both A and C."
+            ],
+            "explanation": "The 'Attention Mechanism' is a technique used in machine learning models, especially in deep learning models, to focus on certain parts of the input data that are more relevant to the task at hand, and to ignore other parts. It's particularly useful in tasks such as machine translation, where it's important to focus on the right words in the input sequence when generating the output sequence.",
+            "answer": "D",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "19",
+            "question": "What is 'Reinforcement Learning' in the context of Machine Learning?",
+            "choices": [
+                "A. It's a type of machine learning where an agent learns to make decisions by taking actions in an environment to maximize some notion of cumulative reward.",
+                "B. It's a type of machine learning where an agent learns to make decisions based on a fixed set of rules.",
+                "C. It's a type of machine learning where an agent learns to make decisions based on a predefined set of actions.",
+                "D. It's a type of machine learning where an agent learns to make decisions based on the actions taken by other agents."
+            ],
+            "explanation": "'Reinforcement Learning' is a type of machine learning where an agent learns to make decisions by taking actions in an environment to maximize some notion of cumulative reward. The agent learns from the consequences of its actions, rather than from being explicitly taught and it selects its actions on basis of its past experiences (exploitation) and also by new choices (exploration).",
+            "answer": "A",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "20",
+            "question": "What is 'Generative Adversarial Network' (GAN) in the context of Machine Learning?",
+            "choices": [
+                "A. It's a class of machine learning systems invented by Ian Goodfellow and his colleagues in 2014.",
+                "B. It's a class of machine learning systems where two neural networks contest with each other in a game.",
+                "C. It's a class of machine learning systems where one neural network, called the generator, generates new data instances, while the other, the discriminator, evaluates them for authenticity.",
+                "D. All of the above."
+            ],
+            "explanation": "'Generative Adversarial Network' (GAN) is a class of machine learning systems invented by Ian Goodfellow and his colleagues in 2014. Two neural networks contest with each other in a game. Given a training set, this technique learns to generate new data with the same statistics as the training set. For example, a GAN trained on photographs can generate new photographs that look at least superficially authentic to human observers, having many realistic characteristics.",
+            "answer": "D",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "21",
+            "question": "Write a Python function to implement a basic 'K-Nearest Neighbors' (KNN) model.",
+            "explanation": "The function should take a dataset and a value for 'K' as arguments and return a trained KNN model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "22",
+            "question": "Write a Python function to implement a basic 'Naive Bayes' model.",
+            "explanation": "The function should take a dataset as an argument and return a trained Naive Bayes model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "23",
+            "question": "Write a Python function to implement a basic 'Random Forest' model.",
+            "explanation": "The function should take a dataset as an argument and return a trained Random Forest model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "24",
+            "question": "Write a Python function to implement a basic 'Gradient Boosting' model.",
+            "explanation": "The function should take a dataset as an argument and return a trained Gradient Boosting model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "25",
+            "question": "Write a Python function to implement a basic 'Deep Neural Network' (DNN) model.",
+            "explanation": "The function should take a dataset as an argument and return a trained DNN model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "26",
+            "question": "Write a Python function to implement a basic 'Convolutional Neural Network' (CNN) model.",
+            "explanation": "The function should take a dataset as an argument and return a trained CNN model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "27",
+            "question": "Write a Python function to implement a basic 'Decision Tree' model.",
+            "explanation": "The function should take a dataset as an argument and return a trained decision tree model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "28",
+            "question": "Write a Python function to implement a basic 'Support Vector Machine' (SVM) model.",
+            "explanation": "The function should take a dataset as an argument and return a trained SVM model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "29",
+            "question": "Write a Python function to implement a basic 'Linear Regression' model.",
+            "explanation": "The function should take a dataset as an argument and return a trained linear regression model.",
+            "level": "Senior",
+            "domain": "AI"
+        },
+        {
+            "id": "30",
+            "question": "Write a Python function to implement a basic 'Logistic Regression' model.",
+            "explanation": "The function should take a dataset as an argument and return a trained logistic regression model.",
+            "level": "Senior",
+            "domain": "AI"
+        }
+    ]
+}

data/test_data/jd_1.txt

@@ -1,11 +0,0 @@
-Senior AI Engineer
-
-This position focuses on developing algorithms for multilingual conversational systems and improving chatbots using advanced Natural Language Processing (NLP) techniques. Responsibilities include collaborating on NLP applications, conducting research, and optimizing AI models.
-
-Responsibilities:
-• Develop algorithms in multilingual conversational systems
-• Implement NLP techniques to enhance text analysis & chatbots understanding of user intent, sentiment analysis, and context-aware responses
-• Team up with software engineers to build end-to-end NLP applications
-• Conduct research and stay up-to-date with the latest advancements in NLP
-• Monitor and analyze the performance of AI models in production, identify and troubleshoot any issues or bottlenecks, and propose solutions for optimization
-• Communicating complex analytical findings and recommendations to non-technical stakeholders, including senior management.