Upload 6 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Data/Domain-A_Dataset_Clean.csv filter=lfs diff=lfs merge=lfs -text

Data/Domain-A_Dataset_Clean.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a75145c90f0d6dad33433e7fa996dae9941da0cda2065b2015b327e368a19c91
+size 20014738

utils/calibration_utils.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import joblib
+import numpy as np
+
+try:
+    from scipy.optimize import minimize_scalar
+except Exception:  # pragma: no cover - scipy may be unavailable in some runtimes
+    minimize_scalar = None
+
+
+DEFAULT_EPS = 1e-6
+
+
+def identity_temperature_scaler(eps: float = DEFAULT_EPS) -> dict:
+    return {
+        "temperature": 1.0,
+        "eps": float(eps),
+        "method": "logit_temperature_scaling",
+        "fit_objective": "binary_nll",
+        "fit_split": "validation",
+        "base_calibration": "identity",
+    }
+
+
+def _ensure_2d(array_like) -> tuple[np.ndarray, bool]:
+    arr = np.asarray(array_like, dtype=float)
+    was_1d = arr.ndim == 1
+    if was_1d:
+        arr = arr.reshape(1, -1)
+    return arr, was_1d
+
+
+def _restore_shape(arr: np.ndarray, was_1d: bool) -> np.ndarray:
+    if was_1d:
+        return arr.reshape(-1)
+    return arr
+
+
+def _binary_nll(y_true, probs, eps: float = DEFAULT_EPS) -> float:
+    y = np.asarray(y_true, dtype=float)
+    p = np.clip(np.asarray(probs, dtype=float), eps, 1.0 - eps)
+    return float(-np.mean(y * np.log(p) + (1.0 - y) * np.log(1.0 - p)))
+
+
+def apply_per_class_calibration(raw_probs, calibrators=None):
+    probs, was_1d = _ensure_2d(raw_probs)
+    calibrated = probs.copy()
+
+    if not calibrators:
+        return _restore_shape(calibrated, was_1d)
+
+    for idx, calibrator in enumerate(calibrators):
+        if calibrator is None:
+            continue
+
+        values = calibrated[:, idx]
+        if hasattr(calibrator, "predict"):
+            calibrated[:, idx] = calibrator.predict(values)
+        else:
+            calibrated[:, idx] = calibrator.transform(values)
+
+    return _restore_shape(calibrated, was_1d)
+
+
+def apply_temperature_scaling(probabilities, temperature_scaler=None):
+    probs, was_1d = _ensure_2d(probabilities)
+    scaler = temperature_scaler or identity_temperature_scaler()
+
+    temperature = float(scaler.get("temperature", 1.0))
+    eps = float(scaler.get("eps", DEFAULT_EPS))
+    temperature = max(temperature, eps)
+
+    clipped = np.clip(probs, eps, 1.0 - eps)
+    logits = np.log(clipped / (1.0 - clipped))
+    scaled_logits = np.clip(logits / temperature, -50.0, 50.0)
+    scaled = 1.0 / (1.0 + np.exp(-scaled_logits))
+
+    return _restore_shape(scaled, was_1d)
+
+
+def calibrate_probabilities(raw_probs, tag_calibrators=None, temperature_scaler=None):
+    per_class = apply_per_class_calibration(raw_probs, tag_calibrators)
+    return apply_temperature_scaling(per_class, temperature_scaler)
+
+
+def max_confidence(probabilities) -> float:
+    probs = np.asarray(probabilities, dtype=float)
+    if probs.size == 0:
+        return 0.0
+    return float(np.max(probs))
+
+
+def fit_temperature_scaler(validation_probs, y_true, bounds=(0.5, 5.0), eps: float = DEFAULT_EPS):
+    probs, _ = _ensure_2d(validation_probs)
+    y = np.asarray(y_true, dtype=float)
+    if probs.shape != y.shape:
+        raise ValueError(
+            f"Shape mismatch for temperature scaling: probs={probs.shape}, y_true={y.shape}"
+        )
+
+    def objective(temp: float) -> float:
+        scaled = apply_temperature_scaling(
+            probs,
+            {"temperature": temp, "eps": eps},
+        )
+        return _binary_nll(y, scaled, eps=eps)
+
+    if minimize_scalar is not None:
+        result = minimize_scalar(objective, bounds=bounds, method="bounded")
+        best_temperature = float(result.x) if result.success else 1.0
+    else:
+        grid = np.exp(np.linspace(np.log(bounds[0]), np.log(bounds[1]), 256))
+        losses = np.array([objective(float(temp)) for temp in grid], dtype=float)
+        best_temperature = float(grid[int(losses.argmin())])
+
+    best_temperature = max(best_temperature, eps)
+    scaled = apply_temperature_scaling(
+        probs,
+        {"temperature": best_temperature, "eps": eps},
+    )
+
+    return {
+        "temperature": round(best_temperature, 6),
+        "eps": float(eps),
+        "method": "logit_temperature_scaling",
+        "fit_objective": "binary_nll",
+        "fit_split": "validation",
+        "base_calibration": "per_class_calibrator_then_temperature",
+        "nll_before": _binary_nll(y, probs, eps=eps),
+        "nll_after": _binary_nll(y, scaled, eps=eps),
+        "mean_conf_before": float(np.mean(np.max(probs, axis=1))),
+        "mean_conf_after": float(np.mean(np.max(scaled, axis=1))),
+    }
+
+
+def load_temperature_scaler(path, default=None):
+    scaler_path = Path(path)
+    if scaler_path.exists():
+        loaded = joblib.load(scaler_path)
+        if isinstance(loaded, dict):
+            return loaded
+        return {"temperature": float(loaded), "eps": DEFAULT_EPS}
+    return default or identity_temperature_scaler()

utils/duplicate_detection_utils.py

@@ -0,0 +1,315 @@
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+import faiss
+from huggingface_hub import hf_hub_download
+import joblib
+import numpy as np
+import pandas as pd
+from sentence_transformers import SentenceTransformer
+
+try:
+    from .runtime_utils import (
+        load_duplicate_threshold,
+        load_model_config,
+        resolve_dataset_file,
+        resolve_model_dir,
+        resolve_model_reference,
+    )
+except ImportError:  # pragma: no cover
+    from runtime_utils import (
+        load_duplicate_threshold,
+        load_model_config,
+        resolve_dataset_file,
+        resolve_model_dir,
+        resolve_model_reference,
+    )
+
+
+class CachedDuplicateDetectionEngine:
+    def __init__(self, base_dir: str | Path | None = None):
+        self.base_dir = Path(base_dir).resolve() if base_dir is not None else Path(__file__).resolve().parent
+        self.model_dir = resolve_model_dir(self.base_dir)
+        self.model_config = load_model_config(self.base_dir)
+        self.duplicate_threshold = load_duplicate_threshold(self.base_dir)
+        from huggingface_hub import hf_hub_download
+        dataset_path = hf_hub_download(
+            repo_id="Eklavya73/ticket-duplicate-assets",
+            filename="Domain-A_Dataset_Clean.csv"
+        )
+
+        self.dataset = pd.read_csv(dataset_path)
+
+        dataset_path = hf_hub_download(
+            repo_id="Eklavya73/ticket-duplicate-assets",
+            filename="Domain-A_Dataset_Clean.csv"
+        )
+        self.dataset = pd.read_csv(dataset_path)
+        self.db_texts = self.dataset["text"].astype(str).tolist()
+
+        ticket_id_path = hf_hub_download(
+            repo_id="Eklavya73/ticket-duplicate-assets",
+            filename="ticket_ids.pkl"
+        )
+        self.db_ids = joblib.load(ticket_id_path)
+
+        if ticket_id_path.exists():
+            self.db_ids = [str(ticket_id) for ticket_id in joblib.load(ticket_id_path)]
+        else:
+            self.db_ids = self.dataset["ticket_id"].astype(str).tolist()
+
+        from huggingface_hub import hf_hub_download
+    
+        file_path = hf_hub_download(
+            repo_id="Eklavya73/ticket-duplicate-assets",
+            filename="db_embeddings.npy"
+        )
+        self.db_embeddings = np.load(file_path).astype("float32").astype("float32")
+
+        if self.db_embeddings.ndim != 2:
+            raise ValueError(
+                f"Expected 2D duplicate embedding matrix, got shape={self.db_embeddings.shape}"
+            )
+        faiss.normalize_L2(self.db_embeddings)
+
+        self.embedding_dim = int(self.db_embeddings.shape[1])
+        self.faiss_meta = self._load_faiss_meta()
+        self.index = self._build_index(self.db_embeddings)
+        self.index.add(self.db_embeddings)
+        self._encoder: SentenceTransformer | None = None
+
+    def _load_faiss_meta(self) -> dict:
+        meta_path = self.model_dir / "faiss_index_meta.pkl"
+        if meta_path.exists():
+            loaded = joblib.load(meta_path)
+            if isinstance(loaded, dict):
+                return loaded
+        return {
+            "dimension": self.embedding_dim,
+            "index_type": "flat",
+            "size": len(self.db_texts),
+        }
+
+    def _build_index(self, embeddings: np.ndarray):
+        index_type = str(self.faiss_meta.get("index_type", "flat")).lower()
+        nlist = max(1, int(self.faiss_meta.get("nlist", 256)))
+        nprobe = max(1, int(self.faiss_meta.get("nprobe", 48)))
+
+        if index_type == "ivf" and len(embeddings) >= max(64, nlist):
+            quantizer = faiss.IndexFlatIP(self.embedding_dim)
+            index = faiss.IndexIVFFlat(
+                quantizer,
+                self.embedding_dim,
+                nlist,
+                faiss.METRIC_INNER_PRODUCT,
+            )
+            index.train(embeddings)
+            index.nprobe = min(nprobe, nlist)
+            return index
+
+        return faiss.IndexFlatIP(self.embedding_dim)
+
+    @property
+    def index_size(self) -> int:
+        return int(self.index.ntotal)
+
+    def _get_encoder(self) -> SentenceTransformer:
+        if self._encoder is None:
+            model_ref = resolve_model_reference(
+                self.model_config.get("duplicate_sbert_model", "Eklavya73/duplicate_sbert"),
+                base_dir=self.base_dir,
+                model_dir=self.model_dir,
+                default="all-mpnet-base-v2",
+            )
+            self._encoder = SentenceTransformer(model_ref)
+        return self._encoder
+
+    def _encode(
+        self,
+        texts,
+        *,
+        batch_size: int = 64,
+        show_progress_bar: bool = False,
+    ) -> np.ndarray:
+        encoder = self._get_encoder()
+        embeddings = encoder.encode(
+            list(texts),
+            batch_size=batch_size,
+            show_progress_bar=show_progress_bar,
+            normalize_embeddings=True,
+        )
+        return np.asarray(embeddings, dtype="float32")
+
+    def _normalize_query(self, embedding) -> np.ndarray:
+        query = np.asarray(embedding, dtype="float32").reshape(1, -1).copy()
+        faiss.normalize_L2(query)
+        return query
+
+    def _search(self, embedding, *, k: int = 20):
+        if self.index_size == 0:
+            return np.empty((1, 0), dtype="float32"), np.empty((1, 0), dtype=int)
+        query = self._normalize_query(embedding)
+        return self.index.search(query, min(max(1, int(k)), self.index_size))
+
+    def find_best_match(
+        self,
+        embedding,
+        *,
+        k: int = 20,
+        exclude_indices=None,
+    ) -> dict | None:
+        scores, indices = self._search(embedding, k=k)
+        excluded = set(int(idx) for idx in (exclude_indices or []))
+
+        for score, idx in zip(scores[0], indices[0]):
+            idx = int(idx)
+            if idx < 0 or idx in excluded:
+                continue
+            return {
+                "index": idx,
+                "ticket_id": self.db_ids[idx] if idx < len(self.db_ids) else None,
+                "duplicate_of": self.db_ids[idx] if idx < len(self.db_ids) else None,
+                "matched_text": self.db_texts[idx] if idx < len(self.db_texts) else None,
+                "similarity": float(score),
+            }
+        return None
+
+    def detect_duplicate(
+        self,
+        text: str | None = None,
+        *,
+        embedding=None,
+        k: int = 20,
+        exclude_indices=None,
+    ) -> dict | None:
+        if embedding is None:
+            if text is None:
+                raise ValueError("Either text or embedding must be provided.")
+            embedding = self._encode([str(text)])[0]
+
+        match = self.find_best_match(
+            embedding,
+            k=k,
+            exclude_indices=exclude_indices,
+        )
+        if match is None:
+            return None
+        if float(match["similarity"]) < float(self.duplicate_threshold):
+            return None
+        return match
+
+    def add_ticket(
+        self,
+        ticket_id: str,
+        text: str,
+        *,
+        embedding=None,
+    ) -> None:
+        if embedding is None:
+            embedding = self._encode([str(text)])[0]
+
+        query = self._normalize_query(embedding)
+        self.index.add(query)
+        self.db_ids.append(str(ticket_id))
+        self.db_texts.append(str(text))
+        self.db_embeddings = np.vstack([self.db_embeddings, query]).astype("float32")
+        self.faiss_meta["size"] = int(self.index.ntotal)
+
+    def benchmark_duplicate_detection(self, *, num_queries: int = 200, k: int = 5) -> dict:
+        if self.index_size <= 1:
+            return {
+                "exact_latency_ms": 0.0,
+                "faiss_latency_ms": 0.0,
+                "speedup_vs_exact": 0.0,
+                "recall_at_k": 0.0,
+                "duplicate_precision": 0.0,
+                "duplicate_recall": 0.0,
+                "duplicate_f1": 0.0,
+                "duplicate_eval_pairs": 0,
+            }
+
+        k = max(1, int(k))
+        rng = np.random.default_rng(42)
+        query_count = min(int(num_queries), self.index_size)
+        sampled_indices = rng.choice(self.index_size, size=query_count, replace=False)
+
+        exact_hits = 0
+        tp = 0
+        fp = 0
+        fn = 0
+        exact_latencies = []
+        faiss_latencies = []
+
+        for query_idx in sampled_indices:
+            query_embedding = self.db_embeddings[query_idx]
+
+            exact_start = time.perf_counter()
+            similarities = self.db_embeddings @ query_embedding
+            similarities[int(query_idx)] = -np.inf
+            exact_top = np.argsort(-similarities)[:k]
+            exact_score = float(similarities[int(exact_top[0])]) if exact_top.size else 0.0
+            exact_is_duplicate = exact_score >= float(self.duplicate_threshold)
+            exact_latencies.append((time.perf_counter() - exact_start) * 1000.0)
+
+            faiss_start = time.perf_counter()
+            distances, neighbors = self._search(query_embedding, k=k + 1)
+            faiss_latencies.append((time.perf_counter() - faiss_start) * 1000.0)
+
+            faiss_candidates = []
+            faiss_best_score = 0.0
+            for score, neighbor_idx in zip(distances[0], neighbors[0]):
+                neighbor_idx = int(neighbor_idx)
+                if neighbor_idx < 0 or neighbor_idx == int(query_idx):
+                    continue
+                faiss_candidates.append(neighbor_idx)
+                if len(faiss_candidates) == 1:
+                    faiss_best_score = float(score)
+                if len(faiss_candidates) >= k:
+                    break
+
+            if exact_top.size and int(exact_top[0]) in set(faiss_candidates):
+                exact_hits += 1
+
+            pred_is_duplicate = bool(faiss_candidates) and faiss_best_score >= float(self.duplicate_threshold)
+            if pred_is_duplicate and exact_is_duplicate:
+                tp += 1
+            elif pred_is_duplicate and not exact_is_duplicate:
+                fp += 1
+            elif exact_is_duplicate and not pred_is_duplicate:
+                fn += 1
+
+        precision = tp / max(tp + fp, 1)
+        recall = tp / max(tp + fn, 1)
+        f1 = 0.0 if (precision + recall) == 0.0 else (2.0 * precision * recall) / (precision + recall)
+
+        exact_latency_ms = float(np.mean(exact_latencies)) if exact_latencies else 0.0
+        faiss_latency_ms = float(np.mean(faiss_latencies)) if faiss_latencies else 0.0
+
+        return {
+            "exact_latency_ms": exact_latency_ms,
+            "faiss_latency_ms": faiss_latency_ms,
+            "speedup_vs_exact": (
+                float(exact_latency_ms / faiss_latency_ms)
+                if faiss_latency_ms > 0.0
+                else 0.0
+            ),
+            "recall_at_k": float(exact_hits / max(query_count, 1)),
+            "duplicate_precision": float(precision),
+            "duplicate_recall": float(recall),
+            "duplicate_f1": float(f1),
+            "duplicate_eval_pairs": int(query_count),
+        }
+
+    def get_duplicate_metrics(self) -> dict:
+        return {
+            "duplicate_threshold": float(self.duplicate_threshold),
+            "faiss_meta": {
+                "dimension": int(self.embedding_dim),
+                "index_type": str(self.faiss_meta.get("index_type", "flat")),
+                "nlist": int(self.faiss_meta.get("nlist", 0)),
+                "nprobe": int(self.faiss_meta.get("nprobe", 0)),
+                "size": int(self.index_size),
+            },
+        }

utils/hybrid_routing_utils.py

@@ -0,0 +1,769 @@
+from __future__ import annotations
+
+import ast
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Mapping
+
+import joblib
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
+
+DEFAULT_TAG_TO_DEPARTMENT = {
+    "technical_issue": "Technical_Support",
+    "hardware_issue": "Technical_Support",
+    "software_issue": "Technical_Support",
+    "network_issue": "Technical_Support",
+    "performance_issue": "Technical_Support",
+    "system_issue": "Technical_Support",
+    "configuration_issue": "Technical_Support",
+    "compatibility_issue": "Technical_Support",
+    "maintenance_issue": "Technical_Support",
+    "update_request": "Technical_Support",
+    "scalability_issue": "Technical_Support",
+    "synchronization_issue": "Technical_Support",
+    "access_issue": "IT_Support",
+    "authentication_issue": "IT_Support",
+    "security_issue": "IT_Support",
+    "api_issue": "IT_Support",
+    "data_issue": "IT_Support",
+    "integration_issue": "IT_Support",
+    "incident": "Service_Outages_And_Maintenance",
+    "service_request": "Customer_Service",
+    "general_inquiry": "Customer_Service",
+    "billing_issue": "Billing_And_Payments",
+    "refund_request": "Billing_And_Payments",
+    "order_issue": "Returns_And_Exchanges",
+    "feature_request": "Product_Support",
+    "training_request": "Human_Resources",
+    "sales_inquiry": "Sales_And_Presales",
+    "digital_marketing": "Marketing",
+    "digital_strategy": "Marketing",
+}
+
+KNOWN_DEPARTMENT_ALIASES = {
+    "technical_support": "Technical_Support",
+    "it_support": "IT_Support",
+    "service_outages_and_maintenance": "Service_Outages_And_Maintenance",
+    "customer_service": "Customer_Service",
+    "billing_and_payments": "Billing_And_Payments",
+    "returns_and_exchanges": "Returns_And_Exchanges",
+    "product_support": "Product_Support",
+    "human_resources": "Human_Resources",
+    "sales_and_presales": "Sales_And_Presales",
+    "marketing": "Marketing",
+    "general_inquiry": "Customer_Service",
+}
+
+
+def _normalize_mapping(mapping) -> dict[str, str]:
+    if not isinstance(mapping, Mapping):
+        return {}
+    return {
+        str(tag): str(department)
+        for tag, department in mapping.items()
+    }
+
+
+def _extract_tag_mapping(policy_or_mapping) -> dict[str, str]:
+    if isinstance(policy_or_mapping, Mapping) and "tag_to_department" in policy_or_mapping:
+        return _normalize_mapping(policy_or_mapping.get("tag_to_department", {}))
+    return _normalize_mapping(policy_or_mapping)
+
+
+def _normalize_department_key(value: str) -> str:
+    return (
+        str(value)
+        .strip()
+        .lower()
+        .replace("&", "and")
+        .replace("-", "_")
+        .replace(" ", "_")
+    )
+
+
+def canonicalize_department_name(name, *, valid_departments=None):
+    if name is None:
+        return None
+
+    text = str(name).strip()
+    if not text or text.lower() == "nan":
+        return None
+
+    valid_departments = [str(department) for department in (valid_departments or [])]
+    normalized_valid = {
+        _normalize_department_key(department): department
+        for department in valid_departments
+    }
+    normalized_name = _normalize_department_key(text)
+
+    if normalized_name in normalized_valid:
+        return normalized_valid[normalized_name]
+
+    aliased = KNOWN_DEPARTMENT_ALIASES.get(normalized_name)
+    if aliased is not None:
+        return aliased
+
+    return text if not valid_departments else None
+
+
+def parse_tag_list(value) -> list[str]:
+    if isinstance(value, list):
+        return [str(tag) for tag in value]
+
+    if value is None:
+        return []
+
+    if isinstance(value, str):
+        stripped = value.strip()
+        if not stripped:
+            return []
+        try:
+            parsed = ast.literal_eval(stripped)
+            if isinstance(parsed, list):
+                return [str(tag) for tag in parsed]
+        except Exception:
+            return []
+
+    return []
+
+
+def validate_routing_label_mapping(
+    tag_to_department,
+    *,
+    valid_tags=None,
+    valid_departments=None,
+):
+    normalized_mapping = _normalize_mapping(tag_to_department)
+    valid_tag_list = (
+        [str(tag) for tag in valid_tags]
+        if valid_tags is not None
+        else None
+    )
+    valid_department_set = (
+        {str(department) for department in valid_departments}
+        if valid_departments is not None
+        else None
+    )
+
+    stale_tags = []
+    if valid_tag_list is not None:
+        stale_tags = [
+            tag for tag in normalized_mapping
+            if tag not in valid_tag_list
+        ]
+
+    invalid_departments = {}
+    if valid_department_set is not None:
+        invalid_departments = {
+            tag: department
+            for tag, department in normalized_mapping.items()
+            if department not in valid_department_set
+        }
+
+    cleaned_mapping = {
+        tag: department
+        for tag, department in normalized_mapping.items()
+        if tag not in stale_tags and tag not in invalid_departments
+    }
+
+    missing_tags = []
+    if valid_tag_list is not None:
+        missing_tags = [
+            tag for tag in valid_tag_list
+            if tag not in cleaned_mapping
+        ]
+
+    return cleaned_mapping, missing_tags, stale_tags, invalid_departments
+
+
+def build_routing_label_policy(
+    *,
+    valid_tags,
+    valid_departments,
+    base_mapping=None,
+    fallback_tag_to_department=None,
+    default_department="Human_Review",
+):
+    merged_mapping = _normalize_mapping(fallback_tag_to_department)
+    merged_mapping.update(_normalize_mapping(base_mapping))
+
+    cleaned_mapping, missing_tags, stale_tags, invalid_departments = (
+        validate_routing_label_mapping(
+            merged_mapping,
+            valid_tags=valid_tags,
+            valid_departments=valid_departments,
+        )
+    )
+
+    return {
+        "tag_to_department": cleaned_mapping,
+        "valid_tags": [str(tag) for tag in valid_tags],
+        "valid_departments": [str(department) for department in valid_departments],
+        "default_department": str(default_department),
+        "missing_tags": missing_tags,
+        "stale_tags": stale_tags,
+        "invalid_departments": invalid_departments,
+        "mapping_count": len(cleaned_mapping),
+    }
+
+
+def assert_valid_routing_label_policy(
+    policy_or_mapping,
+    *,
+    valid_tags,
+    valid_departments,
+):
+    cleaned_mapping, missing_tags, stale_tags, invalid_departments = (
+        validate_routing_label_mapping(
+            _extract_tag_mapping(policy_or_mapping),
+            valid_tags=valid_tags,
+            valid_departments=valid_departments,
+        )
+    )
+
+    if missing_tags or stale_tags or invalid_departments:
+        raise ValueError(
+            "Routing label policy validation failed: "
+            f"missing_tags={missing_tags}, "
+            f"stale_tags={stale_tags}, "
+            f"invalid_departments={invalid_departments}"
+        )
+
+    return cleaned_mapping
+
+
+def assert_predicted_tags_mapped(
+    predicted_tags,
+    policy_or_mapping,
+    *,
+    valid_departments,
+):
+    tag_to_department = _extract_tag_mapping(policy_or_mapping)
+    valid_department_set = {str(department) for department in valid_departments}
+
+    missing_tags = [
+        str(tag) for tag in predicted_tags
+        if str(tag) not in tag_to_department
+    ]
+    invalid_departments = {
+        str(tag): tag_to_department.get(str(tag))
+        for tag in predicted_tags
+        if tag_to_department.get(str(tag)) not in valid_department_set
+    }
+
+    if missing_tags or invalid_departments:
+        raise ValueError(
+            "Predicted tags are not fully covered by the routing label policy: "
+            f"missing_tags={missing_tags}, "
+            f"invalid_departments={invalid_departments}"
+        )
+
+
+def _normalize_vector(vector):
+    arr = np.asarray(vector, dtype=float).reshape(-1)
+    if arr.size == 0:
+        return arr
+    norm = np.linalg.norm(arr)
+    if norm == 0.0:
+        return arr
+    return arr / norm
+
+
+def _build_department_semantic_vectors(
+    valid_departments,
+    *,
+    dept_prototypes=None,
+    embed_text_fn=None,
+):
+    department_vectors = {}
+
+    for department in valid_departments:
+        weighted_parts = []
+        if dept_prototypes is not None and department in dept_prototypes:
+            proto = _normalize_vector(dept_prototypes[department])
+            if proto.size:
+                weighted_parts.append((0.25, proto))
+
+        if embed_text_fn is not None:
+            label_vector = _normalize_vector(
+                embed_text_fn(str(department).replace("_", " "))
+            )
+            if label_vector.size:
+                weighted_parts.append((0.75, label_vector))
+
+        if not weighted_parts:
+            continue
+
+        if len(weighted_parts) == 1:
+            department_vectors[str(department)] = weighted_parts[0][1]
+            continue
+
+        merged = sum(weight * vector for weight, vector in weighted_parts)
+        merged = _normalize_vector(merged)
+        department_vectors[str(department)] = merged
+
+    return department_vectors
+
+
+def _department_semantic_scores(tag, department_vectors, *, embed_text_fn=None):
+    if embed_text_fn is None or not department_vectors:
+        return {}
+
+    tag_vector = _normalize_vector(embed_text_fn(str(tag).replace("_", " ")))
+    if tag_vector.size == 0:
+        return {}
+
+    scores = {}
+    for department, department_vector in department_vectors.items():
+        if department_vector.shape != tag_vector.shape:
+            continue
+        raw_similarity = float(np.dot(tag_vector, department_vector))
+        scores[department] = normalize_semantic_similarity(raw_similarity)
+
+    return scores
+
+
+def build_department_prototypes_from_tag_map(
+    texts,
+    tag_lists,
+    *,
+    tag_to_department,
+    embed_texts_fn,
+    min_examples=5,
+):
+    if embed_texts_fn is None:
+        raise ValueError("embed_texts_fn is required to build department prototypes.")
+
+    normalized_mapping = _extract_tag_mapping(tag_to_department)
+    department_texts: dict[str, list[str]] = defaultdict(list)
+
+    for text, tags in zip(texts, tag_lists):
+        matched_departments = {
+            normalized_mapping[tag]
+            for tag in parse_tag_list(tags)
+            if tag in normalized_mapping
+        }
+        for department in matched_departments:
+            department_texts[department].append(str(text))
+
+    department_prototypes = {}
+    for department, department_examples in department_texts.items():
+        if len(department_examples) < int(min_examples):
+            continue
+
+        embeddings = np.asarray(embed_texts_fn(department_examples), dtype=float)
+        if embeddings.ndim == 1:
+            embeddings = embeddings.reshape(1, -1)
+
+        prototype = _normalize_vector(np.mean(embeddings, axis=0))
+        if prototype.size == 0:
+            continue
+        department_prototypes[department] = prototype
+
+    if not department_prototypes:
+        raise ValueError("No department prototypes could be built from the provided texts and tags.")
+
+    return department_prototypes
+
+
+def analyze_routing_label_policy(
+    dataset_paths,
+    *,
+    tag_classes,
+    valid_departments,
+    existing_mapping=None,
+    queue_column="queue",
+    tags_column="tags",
+):
+    import pandas as pd
+
+    valid_tags = [str(tag) for tag in tag_classes]
+    valid_departments = [str(department) for department in valid_departments]
+    existing_mapping = _extract_tag_mapping(existing_mapping)
+
+    tag_department_counts: dict[str, Counter] = defaultdict(Counter)
+    observed_departments = set()
+    total_rows = 0
+    used_rows = 0
+
+    for dataset_path in dataset_paths:
+        path = Path(dataset_path)
+        if not path.exists():
+            raise FileNotFoundError(f"Dataset not found for routing-label analysis: {path}")
+
+        frame = pd.read_csv(path, usecols=[queue_column, tags_column])
+        total_rows += len(frame)
+
+        for queue_value, tags_value in zip(frame[queue_column], frame[tags_column]):
+            department = canonicalize_department_name(
+                queue_value,
+                valid_departments=valid_departments,
+            )
+            tags = parse_tag_list(tags_value)
+            if department is None or not tags:
+                continue
+
+            used_rows += 1
+            observed_departments.add(department)
+
+            for tag in tags:
+                if tag in valid_tags:
+                    tag_department_counts[tag][department] += 1
+
+    tag_statistics = {}
+    missing_mappings = []
+    unused_tags = []
+    majority_mismatch_tags = []
+
+    for tag in valid_tags:
+        counts = tag_department_counts.get(tag, Counter())
+        ranked = counts.most_common()
+        total_examples = int(sum(counts.values()))
+        majority_department = ranked[0][0] if ranked else None
+        majority_count = int(ranked[0][1]) if ranked else 0
+        fallback_department = ranked[1][0] if len(ranked) > 1 else None
+        fallback_count = int(ranked[1][1]) if len(ranked) > 1 else 0
+        majority_share = float(majority_count / total_examples) if total_examples else 0.0
+        majority_margin_share = (
+            float((majority_count - fallback_count) / total_examples)
+            if total_examples
+            else 0.0
+        )
+
+        current_department = existing_mapping.get(tag)
+        if current_department is None:
+            missing_mappings.append(tag)
+        if total_examples == 0:
+            unused_tags.append(tag)
+        if current_department is not None and majority_department is not None and current_department != majority_department:
+            majority_mismatch_tags.append(tag)
+
+        tag_statistics[tag] = {
+            "current_department": current_department,
+            "department_counts": dict(counts),
+            "total_examples": total_examples,
+            "majority_department": majority_department,
+            "majority_share": majority_share,
+            "majority_margin_share": majority_margin_share,
+            "fallback_department": fallback_department,
+            "fallback_share": float(fallback_count / total_examples) if total_examples else 0.0,
+            "missing_mapping": current_department is None,
+            "unused_tag": total_examples == 0,
+        }
+
+    redundant_mappings = [
+        tag for tag in existing_mapping
+        if tag not in valid_tags
+    ]
+    invalid_departments = {
+        tag: department
+        for tag, department in existing_mapping.items()
+        if canonicalize_department_name(department, valid_departments=valid_departments) is None
+    }
+
+    return {
+        "dataset_paths": [str(Path(path)) for path in dataset_paths],
+        "valid_tags": valid_tags,
+        "valid_departments": valid_departments,
+        "tag_statistics": tag_statistics,
+        "missing_mappings": missing_mappings,
+        "redundant_mappings": redundant_mappings,
+        "invalid_departments": invalid_departments,
+        "unused_tags": unused_tags,
+        "majority_mismatch_tags": majority_mismatch_tags,
+        "observed_departments": sorted(observed_departments),
+        "row_count": int(total_rows),
+        "used_row_count": int(used_rows),
+    }
+
+
+def rebuild_routing_label_policy(
+    analysis,
+    *,
+    existing_mapping=None,
+    valid_departments=None,
+    dept_prototypes=None,
+    embed_text_fn=None,
+    rare_tag_threshold=500,
+    strong_majority_threshold=0.50,
+    distribution_weight=0.60,
+    semantic_weight=0.40,
+    default_department="Human_Review",
+    semantic_override_margin=0.05,
+):
+    tag_statistics = analysis.get("tag_statistics", {})
+    valid_tags = [str(tag) for tag in analysis.get("valid_tags", tag_statistics.keys())]
+    valid_departments = [
+        str(department)
+        for department in (valid_departments or analysis.get("valid_departments", []))
+    ]
+    existing_mapping = _extract_tag_mapping(existing_mapping)
+
+    department_vectors = _build_department_semantic_vectors(
+        valid_departments,
+        dept_prototypes=dept_prototypes,
+        embed_text_fn=embed_text_fn,
+    )
+
+    rebuilt_mapping = {}
+    tag_metadata = {}
+
+    for tag in valid_tags:
+        stats = tag_statistics.get(tag, {})
+        counts = Counter(stats.get("department_counts", {}))
+        total_examples = int(stats.get("total_examples", 0))
+        majority_department = stats.get("majority_department")
+        majority_share = float(stats.get("majority_share", 0.0))
+        semantic_scores = _department_semantic_scores(
+            tag,
+            department_vectors,
+            embed_text_fn=embed_text_fn,
+        )
+        semantic_department = (
+            max(semantic_scores, key=semantic_scores.get)
+            if semantic_scores
+            else None
+        )
+
+        if total_examples >= rare_tag_threshold and majority_department is not None:
+            primary_department = majority_department
+            selection_reason = (
+                "majority_distribution"
+                if majority_share >= strong_majority_threshold
+                else "majority_with_fallback"
+            )
+            ranked_combined = []
+        else:
+            combined_scores = {}
+            for department in valid_departments:
+                semantic_score = float(semantic_scores.get(department, 0.0))
+                combined_scores[department] = float(semantic_weight) * semantic_score
+
+            ranked_combined = sorted(
+                combined_scores.items(),
+                key=lambda item: item[1],
+                reverse=True,
+            )
+            primary_department = semantic_department or (ranked_combined[0][0] if ranked_combined else None)
+            current_department = existing_mapping.get(tag)
+            current_semantic_score = float(semantic_scores.get(current_department, 0.0))
+            winning_semantic_score = float(semantic_scores.get(primary_department, 0.0))
+            if (
+                current_department in valid_departments
+                and current_semantic_score + float(semantic_override_margin) >= winning_semantic_score
+            ):
+                primary_department = current_department
+            if total_examples == 0:
+                selection_reason = "semantic_fallback"
+            else:
+                selection_reason = "rare_tag_semantic"
+
+        if primary_department is None:
+            primary_department = existing_mapping.get(tag)
+            selection_reason = "existing_mapping_fallback"
+
+        if primary_department is None or primary_department not in valid_departments:
+            raise ValueError(f"Unable to determine a valid department for tag '{tag}'.")
+
+        observed_fallback = stats.get("fallback_department")
+        combined_fallback = (
+            ranked_combined[1][0]
+            if len(ranked_combined) > 1
+            else None
+        )
+        fallback_department = observed_fallback or combined_fallback
+        if fallback_department == primary_department:
+            fallback_department = None
+
+        rebuilt_mapping[tag] = primary_department
+        tag_metadata[tag] = {
+            "primary_department": primary_department,
+            "fallback_department": fallback_department,
+            "current_department": existing_mapping.get(tag),
+            "majority_department": majority_department,
+            "majority_share": majority_share,
+            "total_examples": total_examples,
+            "selection_reason": selection_reason,
+            "department_counts": dict(counts),
+            "semantic_department": semantic_department,
+        }
+
+    policy = build_routing_label_policy(
+        valid_tags=valid_tags,
+        valid_departments=valid_departments,
+        base_mapping=rebuilt_mapping,
+        fallback_tag_to_department=existing_mapping,
+        default_department=default_department,
+    )
+    policy.update(
+        {
+            "tag_metadata": tag_metadata,
+            "analysis_summary": {
+                "row_count": int(analysis.get("row_count", 0)),
+                "used_row_count": int(analysis.get("used_row_count", 0)),
+                "missing_mappings": list(analysis.get("missing_mappings", [])),
+                "redundant_mappings": list(analysis.get("redundant_mappings", [])),
+                "invalid_departments": dict(analysis.get("invalid_departments", {})),
+                "unused_tags": list(analysis.get("unused_tags", [])),
+                "majority_mismatch_tags": list(analysis.get("majority_mismatch_tags", [])),
+            },
+            "policy_name": "routing_label_policy",
+        }
+    )
+    return policy
+
+
+def load_routing_label_policy(
+    policy_path,
+    *,
+    fallback_tag_to_department=None,
+    valid_tags=None,
+    valid_departments=None,
+    default_department="Human_Review",
+):
+    path = Path(policy_path) if policy_path is not None else None
+    loaded_policy = {}
+    if path is not None and path.exists():
+        loaded_policy = joblib.load(path)
+
+    base_mapping = _extract_tag_mapping(loaded_policy)
+
+    valid_tags = (
+        [str(tag) for tag in valid_tags]
+        if valid_tags is not None
+        else list(base_mapping)
+    )
+    valid_departments = (
+        [str(department) for department in valid_departments]
+        if valid_departments is not None
+        else []
+    )
+
+    policy = build_routing_label_policy(
+        valid_tags=valid_tags,
+        valid_departments=valid_departments,
+        base_mapping=base_mapping,
+        fallback_tag_to_department=fallback_tag_to_department,
+        default_department=default_department,
+    )
+    if isinstance(loaded_policy, Mapping):
+        for key, value in loaded_policy.items():
+            if key == "tag_to_department":
+                continue
+            policy[key] = value
+    return policy
+
+
+def normalize_semantic_similarity(raw_similarity: float) -> float:
+    return float(np.clip((float(raw_similarity) + 1.0) / 2.0, 0.0, 1.0))
+
+
+def _to_tag_prob_dict(tag_prob_source, tag_names=None) -> dict[str, float]:
+    if isinstance(tag_prob_source, Mapping):
+        return {str(tag): float(prob) for tag, prob in tag_prob_source.items()}
+
+    if tag_names is None:
+        raise ValueError("tag_names are required when tag_prob_source is not a mapping.")
+
+    probs = np.asarray(tag_prob_source, dtype=float).reshape(-1)
+    return {
+        str(tag_names[idx]): float(probs[idx])
+        for idx in range(min(len(tag_names), probs.size))
+    }
+
+
+def _department_classifier_confidence(probabilities) -> float:
+    if len(probabilities) == 0:
+        return 0.0
+    probs = np.clip(np.asarray(probabilities, dtype=float), 0.0, 1.0)
+    return float(1.0 - np.prod(1.0 - probs))
+
+
+def compute_department_hybrid_scores(
+    tag_prob_source,
+    embedding,
+    dept_prototypes,
+    tag_to_department=None,
+    *,
+    tag_names=None,
+    classifier_weight=0.7,
+    similarity_weight=0.3,
+    top_k=5,
+):
+    tag_to_department = tag_to_department or DEFAULT_TAG_TO_DEPARTMENT
+    tag_prob_dict = _to_tag_prob_dict(tag_prob_source, tag_names=tag_names)
+    sorted_tags = sorted(
+        tag_prob_dict.items(),
+        key=lambda item: item[1],
+        reverse=True,
+    )
+    if top_k is not None:
+        sorted_tags = sorted_tags[:top_k]
+
+    assert_predicted_tags_mapped(
+        [tag for tag, _ in sorted_tags],
+        tag_to_department,
+        valid_departments=dept_prototypes.keys(),
+    )
+
+    department_prob_lists: dict[str, list[float]] = {}
+    top_tag_votes = []
+
+    for tag, prob in sorted_tags:
+        department = tag_to_department.get(tag)
+        if department is None:
+            continue
+        clipped_prob = float(np.clip(prob, 0.0, 1.0))
+        department_prob_lists.setdefault(department, []).append(clipped_prob)
+        top_tag_votes.append(
+            {
+                "tag": tag,
+                "score": clipped_prob,
+                "department": department,
+            }
+        )
+
+    details = {}
+    candidate_departments = set(dept_prototypes) | set(department_prob_lists)
+    if not candidate_departments:
+        return None, 0.0, {}, top_tag_votes
+
+    emb = np.asarray(embedding, dtype=float).reshape(1, -1)
+
+    for department in candidate_departments:
+        classifier_confidence = _department_classifier_confidence(
+            department_prob_lists.get(department, [])
+        )
+
+        proto = dept_prototypes.get(department)
+        raw_similarity = 0.0
+        if proto is not None:
+            raw_similarity = float(
+                cosine_similarity(
+                    emb,
+                    np.asarray(proto, dtype=float).reshape(1, -1),
+                )[0][0]
+            )
+
+        semantic_similarity = normalize_semantic_similarity(raw_similarity)
+        hybrid_confidence = (
+            float(classifier_weight) * classifier_confidence
+            + float(similarity_weight) * semantic_similarity
+        )
+
+        details[department] = {
+            "department": department,
+            "classifier_confidence": float(classifier_confidence),
+            "semantic_similarity": float(semantic_similarity),
+            "raw_semantic_similarity": float(raw_similarity),
+            "hybrid_confidence": float(hybrid_confidence),
+        }
+
+    best_department = max(
+        details,
+        key=lambda dept: details[dept]["hybrid_confidence"],
+    )
+    best_hybrid_confidence = float(details[best_department]["hybrid_confidence"])
+    return best_department, best_hybrid_confidence, details, top_tag_votes

utils/review_policy_utils.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import joblib
+import numpy as np
+
+
+DEFAULT_TARGET_REVIEW_FRACTION = 0.15
+DEFAULT_MIN_REVIEW_FRACTION = 0.10
+DEFAULT_FALLBACK_THRESHOLD = 0.55
+
+
+def build_default_review_policy(
+    *,
+    target_review_fraction: float = DEFAULT_TARGET_REVIEW_FRACTION,
+    min_review_fraction: float = DEFAULT_MIN_REVIEW_FRACTION,
+    fallback_threshold: float = DEFAULT_FALLBACK_THRESHOLD,
+):
+    target_review_fraction = float(
+        np.clip(
+            max(float(target_review_fraction), float(min_review_fraction)),
+            0.0,
+            1.0,
+        )
+    )
+    fallback_threshold = float(np.clip(fallback_threshold, 0.0, 1.0))
+    return {
+        "target_review_fraction": target_review_fraction,
+        "min_review_fraction": float(np.clip(min_review_fraction, 0.0, 1.0)),
+        "fallback_threshold": fallback_threshold,
+        "percentile_threshold": fallback_threshold,
+        "effective_threshold": fallback_threshold,
+        "percentile_review_fraction": target_review_fraction,
+        "effective_review_fraction": target_review_fraction,
+        "method": "percentile_plus_fixed_threshold",
+        "fit_source": "default",
+        "sample_size": 0,
+    }
+
+
+def select_review_indices_by_percentile(
+    hybrid_confidences,
+    *,
+    target_review_fraction: float = DEFAULT_TARGET_REVIEW_FRACTION,
+    min_review_fraction: float = DEFAULT_MIN_REVIEW_FRACTION,
+):
+    scores = np.asarray(hybrid_confidences, dtype=float).reshape(-1)
+    if scores.size == 0:
+        return np.zeros(0, dtype=bool)
+
+    target_review_fraction = float(
+        np.clip(
+            max(float(target_review_fraction), float(min_review_fraction)),
+            0.0,
+            1.0,
+        )
+    )
+    review_count = int(np.ceil(scores.size * target_review_fraction))
+    review_count = max(review_count, int(np.ceil(scores.size * float(min_review_fraction))))
+    review_count = min(review_count, scores.size)
+
+    review_mask = np.zeros(scores.size, dtype=bool)
+    if review_count == 0:
+        return review_mask
+
+    review_indices = np.argsort(scores, kind="mergesort")[:review_count]
+    review_mask[review_indices] = True
+    return review_mask
+
+
+def fit_review_policy(
+    hybrid_confidences,
+    *,
+    target_review_fraction: float = DEFAULT_TARGET_REVIEW_FRACTION,
+    min_review_fraction: float = DEFAULT_MIN_REVIEW_FRACTION,
+    fallback_threshold: float = DEFAULT_FALLBACK_THRESHOLD,
+):
+    scores = np.asarray(hybrid_confidences, dtype=float).reshape(-1)
+    if scores.size == 0:
+        return build_default_review_policy(
+            target_review_fraction=target_review_fraction,
+            min_review_fraction=min_review_fraction,
+            fallback_threshold=fallback_threshold,
+        )
+
+    review_mask = select_review_indices_by_percentile(
+        scores,
+        target_review_fraction=target_review_fraction,
+        min_review_fraction=min_review_fraction,
+    )
+    percentile_threshold = float(np.max(scores[review_mask])) if review_mask.any() else float(fallback_threshold)
+    fallback_threshold = float(np.clip(fallback_threshold, 0.0, 1.0))
+    effective_threshold = float(max(percentile_threshold, fallback_threshold))
+    effective_review_mask = (scores <= percentile_threshold) | (scores < fallback_threshold)
+
+    policy = build_default_review_policy(
+        target_review_fraction=target_review_fraction,
+        min_review_fraction=min_review_fraction,
+        fallback_threshold=fallback_threshold,
+    )
+    policy.update(
+        {
+            "percentile_threshold": percentile_threshold,
+            "effective_threshold": effective_threshold,
+            "percentile_review_fraction": float(review_mask.mean()),
+            "effective_review_fraction": float(effective_review_mask.mean()),
+            "fit_source": "validation",
+            "sample_size": int(scores.size),
+            "hybrid_confidence_mean": float(np.mean(scores)),
+            "hybrid_confidence_std": float(np.std(scores)),
+            "hybrid_confidence_min": float(np.min(scores)),
+            "hybrid_confidence_max": float(np.max(scores)),
+        }
+    )
+    return policy
+
+
+def load_review_policy(path, default=None):
+    path = Path(path)
+    if path.exists():
+        loaded = joblib.load(path)
+        if isinstance(loaded, dict):
+            merged = build_default_review_policy()
+            merged.update(loaded)
+            return merged
+    return default or build_default_review_policy()
+
+
+def apply_controlled_review(mode, hybrid_confidence, review_policy=None):
+    policy = review_policy or build_default_review_policy()
+    hybrid_confidence = float(np.clip(hybrid_confidence, 0.0, 1.0))
+
+    percentile_threshold = float(policy.get("percentile_threshold", DEFAULT_FALLBACK_THRESHOLD))
+    fallback_threshold = float(policy.get("fallback_threshold", DEFAULT_FALLBACK_THRESHOLD))
+    effective_threshold = float(policy.get("effective_threshold", max(percentile_threshold, fallback_threshold)))
+
+    percentile_trigger = hybrid_confidence <= percentile_threshold
+    fallback_trigger = hybrid_confidence < fallback_threshold
+    forced_human_review = mode != "HUMAN_REVIEW" and (percentile_trigger or fallback_trigger)
+
+    triggered_rules = []
+    if percentile_trigger:
+        triggered_rules.append("percentile")
+    if fallback_trigger:
+        triggered_rules.append("fixed_threshold")
+
+    final_mode = "HUMAN_REVIEW" if forced_human_review else mode
+    requires_review = final_mode != "AUTO_ROUTE"
+
+    if forced_human_review:
+        reason = (
+            f"Controlled review injection forced HUMAN_REVIEW: hybrid_confidence={hybrid_confidence:.4f}, "
+            f"percentile_threshold={percentile_threshold:.4f}, fallback_threshold={fallback_threshold:.4f}."
+        )
+    elif mode == "HUMAN_REVIEW":
+        reason = "Two-stage gate already routed the ticket to HUMAN_REVIEW."
+    elif mode == "AUTO_ROUTE_FLAGGED":
+        reason = "Ticket remains AUTO_ROUTE_FLAGGED after the controlled review check."
+    else:
+        reason = "Ticket passed the controlled review check and remains AUTO_ROUTE."
+
+    decision = {
+        "base_mode": mode,
+        "final_mode": final_mode,
+        "requires_review": requires_review,
+        "forced_human_review": forced_human_review,
+        "triggered_rules": triggered_rules,
+        "hybrid_confidence": hybrid_confidence,
+        "percentile_threshold": percentile_threshold,
+        "fallback_threshold": fallback_threshold,
+        "effective_threshold": effective_threshold,
+        "target_review_fraction": float(policy.get("target_review_fraction", DEFAULT_TARGET_REVIEW_FRACTION)),
+        "reason": reason,
+    }
+    return final_mode, requires_review, decision

utils/runtime_utils.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import joblib
+import yaml
+
+try:
+    from .hybrid_routing_utils import DEFAULT_TAG_TO_DEPARTMENT
+except ImportError:  # pragma: no cover
+    from hybrid_routing_utils import DEFAULT_TAG_TO_DEPARTMENT
+
+
+DEFAULT_DUPLICATE_THRESHOLD = 0.7623
+
+DEFAULT_ROUTING_CONFIG = {
+    "global_threshold": 0.35,
+    "confidence_threshold": 0.45,
+    "default_department": "Human_Review",
+    "departments": dict(DEFAULT_TAG_TO_DEPARTMENT),
+    "priority_escalation": {
+        "critical": "Escalation",
+        "high": None,
+    },
+}
+
+
+def _resolve_base_dir(base_dir: str | Path | None = None) -> Path:
+    if base_dir is None:
+        return Path(__file__).resolve().parent
+    return Path(base_dir).resolve()
+
+
+def resolve_model_dir(base_dir: str | Path | None = None) -> Path:
+    root = _resolve_base_dir(base_dir)
+    candidates = [
+        root / "Models",
+        root.parent / "Models",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return candidates[0]
+
+
+def resolve_data_root(base_dir: str | Path | None = None) -> Path:
+    root = _resolve_base_dir(base_dir)
+    candidates = [
+        root / "Datasets",
+        root.parent / "Datasets",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return candidates[0]
+
+
+def resolve_dataset_file(
+    base_dir: str | Path | None,
+    filename: str,
+    *,
+    prefer_processed: bool = True,
+) -> Path:
+    data_root = resolve_data_root(base_dir)
+    ordered = []
+    if prefer_processed:
+        ordered.extend(
+            [
+                data_root / "Processed" / filename,
+                data_root / filename,
+            ]
+        )
+    else:
+        ordered.extend(
+            [
+                data_root / filename,
+                data_root / "Processed" / filename,
+            ]
+        )
+
+    for candidate in ordered:
+        if candidate.exists():
+            return candidate
+
+    raise FileNotFoundError(
+        f"Dataset '{filename}' not found in deployment bundle. Checked: {ordered}"
+    )
+
+
+def load_model_config(base_dir: str | Path | None = None) -> dict[str, Any]:
+    model_dir = resolve_model_dir(base_dir)
+    config_path = model_dir / "model_config.pkl"
+    if not config_path.exists():
+        return {}
+    loaded = joblib.load(config_path)
+    return loaded if isinstance(loaded, dict) else {}
+
+
+def resolve_model_reference(
+    model_ref: str | Path | None,
+    *,
+    base_dir: str | Path | None = None,
+    model_dir: str | Path | None = None,
+    default: str | None = None,
+) -> str:
+    if model_ref in (None, ""):
+        if default is None:
+            raise FileNotFoundError("No model reference was provided.")
+        return str(default)
+
+    raw_value = str(model_ref)
+    raw_path = Path(raw_value)
+    base_path = _resolve_base_dir(base_dir)
+    model_path_root = Path(model_dir).resolve() if model_dir is not None else resolve_model_dir(base_path)
+
+    candidates: list[Path] = []
+    if raw_path.is_absolute():
+        candidates.append(raw_path)
+        if "Models" in raw_path.parts:
+            model_idx = raw_path.parts.index("Models")
+            suffix_parts = raw_path.parts[model_idx + 1 :]
+            if suffix_parts:
+                candidates.append(model_path_root.joinpath(*suffix_parts))
+        candidates.append(model_path_root / raw_path.name)
+        candidates.append(base_path / raw_path.name)
+    else:
+        candidates.extend(
+            [
+                raw_path,
+                base_path / raw_path,
+                model_path_root / raw_path,
+                model_path_root / raw_path.name,
+                base_path / raw_path.name,
+            ]
+        )
+
+    seen = set()
+    for candidate in candidates:
+        candidate = candidate.resolve() if candidate.exists() else candidate
+        normalized = str(candidate)
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        if candidate.exists():
+            return str(candidate)
+
+    if default is not None:
+        return str(default)
+    return raw_value
+
+
+def _merge_routing_config(loaded: dict[str, Any] | None) -> dict[str, Any]:
+    merged = {
+        "global_threshold": DEFAULT_ROUTING_CONFIG["global_threshold"],
+        "confidence_threshold": DEFAULT_ROUTING_CONFIG["confidence_threshold"],
+        "default_department": DEFAULT_ROUTING_CONFIG["default_department"],
+        "departments": dict(DEFAULT_ROUTING_CONFIG["departments"]),
+        "priority_escalation": dict(DEFAULT_ROUTING_CONFIG["priority_escalation"]),
+    }
+    if not isinstance(loaded, dict):
+        return merged
+
+    for key in ("global_threshold", "confidence_threshold", "default_department"):
+        if key in loaded:
+            merged[key] = loaded[key]
+
+    merged["departments"].update(loaded.get("departments") or {})
+    merged["priority_escalation"].update(loaded.get("priority_escalation") or {})
+    return merged
+
+
+def load_routing_config(
+    base_dir: str | Path | None = None,
+) -> tuple[dict[str, Any], Path | None]:
+    root = _resolve_base_dir(base_dir)
+    model_dir = resolve_model_dir(root)
+    candidates = [
+        root / "config" / "routing_config.yaml",
+        model_dir / "routing_config.yaml",
+        root.parent / "config" / "routing_config.yaml",
+    ]
+
+    for candidate in candidates:
+        if not candidate.exists():
+            continue
+        with candidate.open("r", encoding="utf-8") as handle:
+            return _merge_routing_config(yaml.safe_load(handle)), candidate
+
+    return _merge_routing_config(None), None
+
+
+def load_duplicate_threshold(base_dir: str | Path | None = None) -> float:
+    model_dir = resolve_model_dir(base_dir)
+    threshold_path = model_dir / "duplicate_thresholds.pkl"
+    if threshold_path.exists():
+        payload = joblib.load(threshold_path)
+        if isinstance(payload, dict):
+            try:
+                return float(payload.get("duplicate_threshold", DEFAULT_DUPLICATE_THRESHOLD))
+            except (TypeError, ValueError):
+                pass
+    return float(DEFAULT_DUPLICATE_THRESHOLD)
+
+
+def load_metric_artifact(
+    base_dir: str | Path | None,
+    filename: str,
+) -> dict[str, Any]:
+    model_dir = resolve_model_dir(base_dir)
+    artifact_path = model_dir / filename
+    if not artifact_path.exists():
+        raise FileNotFoundError(f"Metric artifact not found: {artifact_path}")
+    with artifact_path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)