Spaces:
Running
Running
[NOTICKET] [AI] Calculate Score - filtering profiles first before calculate => DONE
Browse files[NOTICKET] [AI] Endpoint score card jumlah extracted & processed data => DONE
[NOTICKET] [AI] Update end point get profile, tambahkan link untuk view blob => DONE
- .env.example +2 -8
- config/constant.py +1 -0
- externals/databases/pg_crud.py +55 -3
- interfaces/api/profile.py +0 -6
- services/agentic/profile_scoring.py +10 -9
- services/knowledge/get_profile.py +13 -3
.env.example
CHANGED
|
@@ -1,13 +1,6 @@
|
|
| 1 |
## -------------------- DATABASE CONFIGURATION -------------------- ##
|
| 2 |
|
| 3 |
-
#
|
| 4 |
-
causalogy--pg--name = ""
|
| 5 |
-
causalogy--pg--user = ""
|
| 6 |
-
causalogy--pg--password = ""
|
| 7 |
-
causalogy--pg--host = ""
|
| 8 |
-
causalogy--pg--port = ""
|
| 9 |
-
|
| 10 |
-
# Local Postgresql
|
| 11 |
ss__postgresql__url = ""
|
| 12 |
|
| 13 |
|
|
@@ -55,6 +48,7 @@ azureai__embedmodel__name = ""
|
|
| 55 |
|
| 56 |
# AZURE BLOB
|
| 57 |
azureai__search__sas = ""
|
|
|
|
| 58 |
azureai__container__endpoint = ""
|
| 59 |
azureai__container__name = ""
|
| 60 |
azureai__container__account__name = ""
|
|
|
|
| 1 |
## -------------------- DATABASE CONFIGURATION -------------------- ##
|
| 2 |
|
| 3 |
+
# Postgresql
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
ss__postgresql__url = ""
|
| 5 |
|
| 6 |
|
|
|
|
| 48 |
|
| 49 |
# AZURE BLOB
|
| 50 |
azureai__search__sas = ""
|
| 51 |
+
azureai__blob__sas = ""
|
| 52 |
azureai__container__endpoint = ""
|
| 53 |
azureai__container__name = ""
|
| 54 |
azureai__container__account__name = ""
|
config/constant.py
CHANGED
|
@@ -23,6 +23,7 @@ class AzureBlobConstants:
|
|
| 23 |
ENDPOINT: str = os.environ.get("azureai__container__endpoint")
|
| 24 |
CONTAINER_NAME: str = os.environ.get("azureai__container__name")
|
| 25 |
SAS_KEY: str = os.environ.get("azureai__search__sas")
|
|
|
|
| 26 |
CONTAINER_KEY: str = os.environ.get("azureai__container__key")
|
| 27 |
MAX_FILE_SIZE_MB: int = int(os.getenv("azureai__max_file_size_mb", "5"))
|
| 28 |
CHUNK_SIZE: int = 4 * 1024 * 1024
|
|
|
|
| 23 |
ENDPOINT: str = os.environ.get("azureai__container__endpoint")
|
| 24 |
CONTAINER_NAME: str = os.environ.get("azureai__container__name")
|
| 25 |
SAS_KEY: str = os.environ.get("azureai__search__sas")
|
| 26 |
+
BLOB_SAS_KEY: str = os.environ.get("azureai__blob__sas")
|
| 27 |
CONTAINER_KEY: str = os.environ.get("azureai__container__key")
|
| 28 |
MAX_FILE_SIZE_MB: int = int(os.getenv("azureai__max_file_size_mb", "5"))
|
| 29 |
CHUNK_SIZE: int = 4 * 1024 * 1024
|
externals/databases/pg_crud.py
CHANGED
|
@@ -433,13 +433,65 @@ async def get_profiles_by_criteria_id(
|
|
| 433 |
return result.scalars().all()
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
async def get_profile_by_id(
|
| 437 |
db: AsyncSession,
|
| 438 |
profile_id: str,
|
| 439 |
-
) -> Optional[
|
| 440 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 441 |
result = await db.execute(stmt)
|
| 442 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 443 |
|
| 444 |
|
| 445 |
async def list_profiles(
|
|
|
|
| 433 |
return result.scalars().all()
|
| 434 |
|
| 435 |
|
| 436 |
+
# async def get_profile_by_id(
|
| 437 |
+
# db: AsyncSession,
|
| 438 |
+
# profile_id: str,
|
| 439 |
+
# ) -> Optional[CVProfile]:
|
| 440 |
+
# stmt = select(CVProfile).where(CVProfile.profile_id == profile_id)
|
| 441 |
+
# result = await db.execute(stmt)
|
| 442 |
+
# return result.scalar_one_or_none()
|
| 443 |
+
|
| 444 |
+
from typing import Tuple
|
| 445 |
+
|
| 446 |
+
# async def get_profile_by_id(
|
| 447 |
+
# db: AsyncSession,
|
| 448 |
+
# profile_id: str,
|
| 449 |
+
# ) -> Optional[Tuple[CVProfile, str]]:
|
| 450 |
+
|
| 451 |
+
# stmt = (
|
| 452 |
+
# select(CVProfile, CVFile.url)
|
| 453 |
+
# .join(CVFile, CVProfile.file_id == CVFile.file_id)
|
| 454 |
+
# .where(CVProfile.profile_id == profile_id)
|
| 455 |
+
# )
|
| 456 |
+
|
| 457 |
+
# result = await db.execute(stmt)
|
| 458 |
+
# result = result.scalar_one_or_none()
|
| 459 |
+
|
| 460 |
+
# if not result:
|
| 461 |
+
# return None
|
| 462 |
+
|
| 463 |
+
# return result
|
| 464 |
+
|
| 465 |
+
|
| 466 |
async def get_profile_by_id(
|
| 467 |
db: AsyncSession,
|
| 468 |
profile_id: str,
|
| 469 |
+
) -> Optional[dict]:
|
| 470 |
+
|
| 471 |
+
stmt = (
|
| 472 |
+
select(CVProfile, CVFile.url)
|
| 473 |
+
.join(CVFile, CVProfile.file_id == CVFile.file_id)
|
| 474 |
+
.where(CVProfile.profile_id == profile_id)
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
result = await db.execute(stmt)
|
| 478 |
+
row = result.first()
|
| 479 |
+
|
| 480 |
+
if not row:
|
| 481 |
+
return None
|
| 482 |
+
|
| 483 |
+
profile, url = row
|
| 484 |
+
|
| 485 |
+
# Convert SQLAlchemy model to dict safely
|
| 486 |
+
profile_dict = {
|
| 487 |
+
column.name: getattr(profile, column.name)
|
| 488 |
+
for column in CVProfile.__table__.columns
|
| 489 |
+
}
|
| 490 |
+
|
| 491 |
+
# Add file URL
|
| 492 |
+
profile_dict["url"] = url
|
| 493 |
+
|
| 494 |
+
return profile_dict
|
| 495 |
|
| 496 |
|
| 497 |
async def list_profiles(
|
interfaces/api/profile.py
CHANGED
|
@@ -51,12 +51,6 @@ async def get_profile(
|
|
| 51 |
detail="Profile not found",
|
| 52 |
)
|
| 53 |
|
| 54 |
-
# π Tenant isolation (IMPORTANT)
|
| 55 |
-
if profile.tenant_id != current_user.tenant_id:
|
| 56 |
-
raise HTTPException(
|
| 57 |
-
status_code=status.HTTP_403_FORBIDDEN,
|
| 58 |
-
detail="Not authorized to access this profile",
|
| 59 |
-
)
|
| 60 |
return profile
|
| 61 |
|
| 62 |
|
|
|
|
| 51 |
detail="Profile not found",
|
| 52 |
)
|
| 53 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
return profile
|
| 55 |
|
| 56 |
|
services/agentic/profile_scoring.py
CHANGED
|
@@ -26,7 +26,7 @@ from services.agentic.weight import AgenticWeightService
|
|
| 26 |
from services.agentic.filter import AgenticFilterService
|
| 27 |
from services.knowledge.get_profile import KnowledgeGetProfileService
|
| 28 |
from sqlalchemy.ext.asyncio import AsyncSession
|
| 29 |
-
from externals.databases.pg_crud import get_profiles, create_matchings, create_scores
|
| 30 |
from utils.logger import get_logger
|
| 31 |
|
| 32 |
|
|
@@ -423,12 +423,12 @@ class AgenticScoringService:
|
|
| 423 |
try:
|
| 424 |
results = []
|
| 425 |
for i, p in enumerate(profiles):
|
| 426 |
-
print(f"==> {i+1} profile: {p} vs criteria: {criteria}")
|
| 427 |
tmp_matching:AIMatchProfile = await self._ai_matching(profile=p, criteria=criteria)
|
| 428 |
results.append(tmp_matching)
|
| 429 |
return results
|
| 430 |
except Exception as E:
|
| 431 |
-
|
| 432 |
raise
|
| 433 |
|
| 434 |
def _calculate_score(self, match_result:AIMatchProfile, weight_data:CriteriaWeight) -> float:
|
|
@@ -450,7 +450,7 @@ class AgenticScoringService:
|
|
| 450 |
for k, v in weight_data.items():
|
| 451 |
total_weight += v
|
| 452 |
|
| 453 |
-
logger.info(f"ποΈ helper_calculate_score/total_weight: {total_weight}")
|
| 454 |
|
| 455 |
if total_weight > 1.0:
|
| 456 |
# normalized weight
|
|
@@ -489,13 +489,14 @@ class AgenticScoringService:
|
|
| 489 |
async def scoring(self, weight_id: str):
|
| 490 |
try:
|
| 491 |
# Get profile data all
|
| 492 |
-
all_profiles = await get_profiles(self.db)
|
| 493 |
-
|
| 494 |
-
|
|
|
|
| 495 |
|
| 496 |
_weight:CVWeight = await self.weight_service.get_weight_by_weight_id(weight_id=weight_id)
|
| 497 |
-
print(f"Found weight: {_weight}")
|
| 498 |
-
print(f"--> criteria id: {_weight.criteria_id}")
|
| 499 |
|
| 500 |
_criteria:CVFilter = await self.criteria_service.get_filter_by_id(criteria_id=_weight.criteria_id)
|
| 501 |
|
|
|
|
| 26 |
from services.agentic.filter import AgenticFilterService
|
| 27 |
from services.knowledge.get_profile import KnowledgeGetProfileService
|
| 28 |
from sqlalchemy.ext.asyncio import AsyncSession
|
| 29 |
+
from externals.databases.pg_crud import get_profiles, create_matchings, create_scores, get_profiles_by_criteria_id, get_weight_by_id
|
| 30 |
from utils.logger import get_logger
|
| 31 |
|
| 32 |
|
|
|
|
| 423 |
try:
|
| 424 |
results = []
|
| 425 |
for i, p in enumerate(profiles):
|
| 426 |
+
# print(f"==> {i+1} profile: {p} vs criteria: {criteria}")
|
| 427 |
tmp_matching:AIMatchProfile = await self._ai_matching(profile=p, criteria=criteria)
|
| 428 |
results.append(tmp_matching)
|
| 429 |
return results
|
| 430 |
except Exception as E:
|
| 431 |
+
logger.error(f"β error in _ai_matching_bulk: {E}")
|
| 432 |
raise
|
| 433 |
|
| 434 |
def _calculate_score(self, match_result:AIMatchProfile, weight_data:CriteriaWeight) -> float:
|
|
|
|
| 450 |
for k, v in weight_data.items():
|
| 451 |
total_weight += v
|
| 452 |
|
| 453 |
+
# logger.info(f"ποΈ helper_calculate_score/total_weight: {total_weight}")
|
| 454 |
|
| 455 |
if total_weight > 1.0:
|
| 456 |
# normalized weight
|
|
|
|
| 489 |
async def scoring(self, weight_id: str):
|
| 490 |
try:
|
| 491 |
# Get profile data all
|
| 492 |
+
# all_profiles = await get_profiles(self.db)
|
| 493 |
+
weight = await get_weight_by_id(self.db, weight_id=weight_id)
|
| 494 |
+
all_profiles = await get_profiles_by_criteria_id(db=self.db, criteria_id=weight.criteria_id, current_user=self.user)
|
| 495 |
+
print(f"π«‘ Found {len(all_profiles)} profiles to be scored")
|
| 496 |
|
| 497 |
_weight:CVWeight = await self.weight_service.get_weight_by_weight_id(weight_id=weight_id)
|
| 498 |
+
# print(f"Found weight: {_weight}")
|
| 499 |
+
# print(f"--> criteria id: {_weight.criteria_id}")
|
| 500 |
|
| 501 |
_criteria:CVFilter = await self.criteria_service.get_filter_by_id(criteria_id=_weight.criteria_id)
|
| 502 |
|
services/knowledge/get_profile.py
CHANGED
|
@@ -3,6 +3,7 @@ from externals.databases.pg_models import CVUser, CVProfile
|
|
| 3 |
from services.models.data_model import Profile
|
| 4 |
from sqlalchemy.ext.asyncio import AsyncSession
|
| 5 |
from typing import List, Dict, Generator, Union, Any
|
|
|
|
| 6 |
from utils.logger import get_logger
|
| 7 |
|
| 8 |
logger = get_logger("get profile")
|
|
@@ -27,10 +28,19 @@ class KnowledgeGetProfileService:
|
|
| 27 |
|
| 28 |
return profile
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
async def get_profile(self, profile_id: str) -> CVProfile:
|
| 32 |
try:
|
| 33 |
-
profile = await get_profile_by_id(profile_id=profile_id)
|
|
|
|
| 34 |
return profile
|
| 35 |
except Exception as E:
|
| 36 |
logger.error(f"β get profile error for profile_id {profile_id}, {E}")
|
|
@@ -39,10 +49,10 @@ class KnowledgeGetProfileService:
|
|
| 39 |
|
| 40 |
async def get_profile_generator(self, profile_id: str):
|
| 41 |
try:
|
| 42 |
-
profile = await get_profile_by_id(profile_id=profile_id)
|
| 43 |
yield profile
|
| 44 |
except Exception as E:
|
| 45 |
-
logger.error(f"β get profile error for profile_id {profile_id}, {E}")
|
| 46 |
return
|
| 47 |
|
| 48 |
|
|
|
|
| 3 |
from services.models.data_model import Profile
|
| 4 |
from sqlalchemy.ext.asyncio import AsyncSession
|
| 5 |
from typing import List, Dict, Generator, Union, Any
|
| 6 |
+
from config.constant import AzureBlobConstants
|
| 7 |
from utils.logger import get_logger
|
| 8 |
|
| 9 |
logger = get_logger("get profile")
|
|
|
|
| 28 |
|
| 29 |
return profile
|
| 30 |
|
| 31 |
+
|
| 32 |
+
def add_sas_blob(self, url: str) -> str:
|
| 33 |
+
sas_token = AzureBlobConstants.BLOB_SAS_KEY
|
| 34 |
+
if '?' in url:
|
| 35 |
+
return f"{url}&{sas_token}"
|
| 36 |
+
else:
|
| 37 |
+
return f"{url}?{sas_token}"
|
| 38 |
+
|
| 39 |
|
| 40 |
async def get_profile(self, profile_id: str) -> CVProfile:
|
| 41 |
try:
|
| 42 |
+
profile = await get_profile_by_id(db=self.db, profile_id=profile_id)
|
| 43 |
+
profile["url"] = self.add_sas_blob(profile["url"])
|
| 44 |
return profile
|
| 45 |
except Exception as E:
|
| 46 |
logger.error(f"β get profile error for profile_id {profile_id}, {E}")
|
|
|
|
| 49 |
|
| 50 |
async def get_profile_generator(self, profile_id: str):
|
| 51 |
try:
|
| 52 |
+
profile = await get_profile_by_id(db=self.db, profile_id=profile_id)
|
| 53 |
yield profile
|
| 54 |
except Exception as E:
|
| 55 |
+
logger.error(f"β get profile generator error for profile_id {profile_id}, {E}")
|
| 56 |
return
|
| 57 |
|
| 58 |
|