Merge remote-tracking branch 'origin/feature/integration2.0' into hugging_face_final

config/__init__.py

@@ -3,6 +3,9 @@
 from __future__ import annotations
 
 import os
+
+# ClearML UI project name (must match the project in your ClearML workspace).
+CLEARML_PROJECT_NAME = "FocusGuards Large Group Project"
 from pathlib import Path
 from typing import Any
 

config/clearml_enrich.py

@@ -0,0 +1,87 @@
+"""Extra ClearML polish: env tags, config snapshot, output model metadata."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def project_root() -> Path:
+    return Path(__file__).resolve().parent.parent
+
+
+def active_config_path() -> Path:
+    env = os.environ.get("FOCUSGUARD_CONFIG")
+    if env:
+        return Path(env).expanduser()
+    return Path(__file__).resolve().parent / "default.yaml"
+
+
+def enrich_task(task, *, role: str) -> None:
+    """Tags for filtering in the UI (Python, OS, torch device, git revision)."""
+    tags = [
+        role,
+        f"py{sys.version_info.major}{sys.version_info.minor}",
+        sys.platform.replace(" ", "_"),
+    ]
+    try:
+        import torch
+
+        ver = torch.__version__.split("+")[0].replace(".", "_")
+        tags.append(f"torch_{ver}")
+        tags.append("cuda" if torch.cuda.is_available() else "cpu")
+    except ImportError:
+        tags.append("no_torch")
+    rev = _git_short_rev()
+    if rev:
+        tags.append(f"git_{rev}")
+    task.add_tags(tags)
+
+
+def _git_short_rev() -> str | None:
+    root = project_root()
+    try:
+        p = subprocess.run(
+            ["git", "rev-parse", "--short", "HEAD"],
+            cwd=str(root),
+            capture_output=True,
+            text=True,
+            timeout=6,
+            check=False,
+        )
+        if p.returncode == 0 and p.stdout:
+            return p.stdout.strip()
+    except (OSError, subprocess.TimeoutExpired):
+        pass
+    return None
+
+
+def upload_repro_artifacts(task) -> None:
+    """Pin the exact YAML + requirements file used for this run."""
+    cfg = active_config_path()
+    if cfg.is_file():
+        task.upload_artifact(name="config_yaml", artifact_object=str(cfg))
+    req = project_root() / "requirements.txt"
+    if req.is_file():
+        task.upload_artifact(name="requirements_txt", artifact_object=str(req))
+
+
+def attach_output_metrics(output_model, metrics: dict[str, float | str]) -> None:
+    """Surface headline metrics on the registered model card."""
+    for k, v in metrics.items():
+        key = str(k).replace("/", "_")
+        try:
+            output_model.set_metadata(key, str(v))
+        except Exception:
+            pass
+
+
+def task_done_summary(task, summary: str) -> None:
+    setter = getattr(task, "set_comment", None)
+    if callable(setter):
+        try:
+            setter(summary)
+        except Exception:
+            pass

evaluation/justify_thresholds.py

@@ -32,22 +32,26 @@ _logger = None
 if _USE_CLEARML:
     try:
         from clearml import Task
-        from config import flatten_for_clearml
+        from config import CLEARML_PROJECT_NAME, flatten_for_clearml
         _task = Task.init(
-            project_name="Focus Guard",
+            project_name=CLEARML_PROJECT_NAME,
             task_name="Threshold Justification",
             tags=["evaluation", "thresholds"],
         )
+        from config.clearml_enrich import enrich_task, upload_repro_artifacts
+
+        enrich_task(_task, role="eval_thresholds")
         flat = flatten_for_clearml()
         flat["evaluation/SEED"] = SEED
         flat["evaluation/n_participants"] = 9
         _task.connect(flat)
+        upload_repro_artifacts(_task)
         _logger = _task.get_logger()
         if _CLEARML_QUEUE:
             print(f"[ClearML] Enqueuing to queue '{_CLEARML_QUEUE}'.")
             _task.execute_remotely(queue_name=_CLEARML_QUEUE)
             sys.exit(0)
-        print("ClearML enabled — logging to project 'Focus Guard'")
+        print(f"ClearML enabled — logging to project '{CLEARML_PROJECT_NAME}'")
     except ImportError:
         print("WARNING: ClearML not installed. Continuing without logging.")
         _USE_CLEARML = False
@@ -548,6 +552,17 @@ def main():
 
     # Close ClearML task
     if _task:
+        from config.clearml_enrich import task_done_summary
+
+        if os.path.isfile(REPORT_PATH):
+            _task.upload_artifact(
+                name="threshold_justification_report",
+                artifact_object=REPORT_PATH,
+            )
+        task_done_summary(
+            _task,
+            "LOPO threshold / weight analysis; see artifact threshold_justification_report and plots in Debug samples.",
+        )
         _task.close()
         print("ClearML task closed.")
 

models/mlp/train.py

@@ -1,3 +1,4 @@
+import csv
 import json
 import os
 import random
@@ -9,10 +10,12 @@ import torch
 import torch.nn as nn
 import torch.optim as optim
 from sklearn.metrics import (
+    classification_report,
     confusion_matrix,
     f1_score,
     precision_recall_fscore_support,
     roc_auc_score,
+    roc_curve,
 )
 
 from data_preparation.prepare_dataset import get_dataloaders, SELECTED_FEATURES
@@ -62,12 +65,15 @@ task = None
 if USE_CLEARML:
     try:
         from clearml import Task
-        from config import flatten_for_clearml
+        from config import CLEARML_PROJECT_NAME, flatten_for_clearml
         task = Task.init(
-            project_name="Focus Guard",
+            project_name=CLEARML_PROJECT_NAME,
             task_name="MLP Model Training",
             tags=["training", "mlp_model"],
         )
+        from config.clearml_enrich import enrich_task, upload_repro_artifacts
+
+        enrich_task(task, role="train_mlp")
         flat = flatten_for_clearml()
         flat["mlp/model_name"] = CFG.get("model_name", "face_orientation")
         flat["mlp/epochs"] = CFG.get("epochs", 30)
@@ -77,6 +83,7 @@ if USE_CLEARML:
         flat["mlp/hidden_sizes"] = str(CFG.get("hidden_sizes", [64, 32]))
         flat["mlp/split_ratios"] = str(CFG.get("split_ratios", (0.7, 0.15, 0.15)))
         task.connect(flat)
+        upload_repro_artifacts(task)
         if CLEARML_QUEUE:
             print(f"[ClearML] Enqueuing to queue '{CLEARML_QUEUE}'. Agent will run training.")
             task.execute_remotely(queue_name=CLEARML_QUEUE)
@@ -313,18 +320,75 @@ def main() -> None:
     np.savez(meta_path, feature_names=np.array(SELECTED_FEATURES["face_orientation"]))
     print(f"[LOG] Scaler and meta saved to {ckpt_dir}")
 
-    # ClearML: artifacts, confusion matrix, per-class metrics
+    cm = confusion_matrix(test_labels_np, test_preds_np)
+    pred_csv = os.path.join(logs_dir, f"{CFG['model_name']}_test_predictions.csv")
+    with open(pred_csv, "w", newline="") as f:
+        w = csv.writer(f)
+        w.writerow(["y_true", "y_pred"] + [f"prob_{j}" for j in range(num_classes)])
+        for i in range(len(test_labels_np)):
+            w.writerow(
+                [int(test_labels_np[i]), int(test_preds_np[i])]
+                + [float(x) for x in test_probs[i]]
+            )
+    summary_path = os.path.join(logs_dir, f"{CFG['model_name']}_test_metrics_summary.json")
+    with open(summary_path, "w", encoding="utf-8") as f:
+        json.dump(
+            {
+                "model": "mlp",
+                "model_name": CFG["model_name"],
+                "checkpoint": os.path.basename(best_ckpt_path),
+                "test_loss": history["test_loss"],
+                "test_accuracy": history["test_acc"],
+                "test_f1_weighted": history["test_f1"],
+                "test_roc_auc": history["test_auc"],
+                "confusion_matrix": cm.tolist(),
+                "classification_report": classification_report(
+                    test_labels_np, test_preds_np, digits=4
+                ),
+            },
+            f,
+            indent=2,
+        )
+    print(f"[LOG] Test predictions → {pred_csv}")
+
+    # ClearML: artifacts, confusion matrix, per-class metrics, registered model
     if task is not None:
-        task.upload_artifact(name="mlp_best", artifact_object=best_ckpt_path)
+        from clearml import OutputModel
+        from config.clearml_enrich import attach_output_metrics, task_done_summary
+
+        task.upload_artifact(name="mlp_checkpoint", artifact_object=best_ckpt_path)
         task.upload_artifact(name="training_log", artifact_object=log_path)
+        task.upload_artifact(name="test_predictions", artifact_object=pred_csv)
+        task.upload_artifact(name="test_metrics_summary", artifact_object=summary_path)
+        task.upload_artifact(name="scaler_mlp", artifact_object=scaler_path)
+        task.upload_artifact(name="meta_mlp", artifact_object=meta_path)
+        out_model = OutputModel(
+            task=task, name=f"MLP_{CFG['model_name']}", framework="PyTorch"
+        )
+        out_model.update_weights(
+            weights_filename=best_ckpt_path, auto_delete_file=False
+        )
+        attach_output_metrics(
+            out_model,
+            {
+                "test_accuracy": round(float(test_acc), 6),
+                "test_f1_weighted": round(float(test_f1), 6),
+                "test_roc_auc": round(float(test_auc), 6),
+            },
+        )
+        task_done_summary(
+            task,
+            f"MLP {CFG['model_name']}: test acc={test_acc:.4f}, F1={test_f1:.4f}, ROC-AUC={test_auc:.4f}",
+        )
         task.logger.report_single_value("test/accuracy", test_acc)
         task.logger.report_single_value("test/f1_weighted", test_f1)
         task.logger.report_single_value("test/roc_auc", test_auc)
         for key, val in dataset_stats.items():
             if isinstance(val, list):
-                task.logger.report_single_value(f"dataset/{key}", str(val))
+                for i, v in enumerate(val):
+                    task.logger.report_single_value(f"dataset/{key}/{i}", float(v))
             else:
-                task.logger.report_single_value(f"dataset/{key}", val)
+                task.logger.report_single_value(f"dataset/{key}", float(val))
         prec, rec, f1_per_class, _ = precision_recall_fscore_support(
             test_labels_np, test_preds_np, average=None, zero_division=0
         )
@@ -332,7 +396,6 @@ def main() -> None:
             task.logger.report_single_value(f"test/class_{c}_precision", float(prec[c]))
             task.logger.report_single_value(f"test/class_{c}_recall", float(rec[c]))
             task.logger.report_single_value(f"test/class_{c}_f1", float(f1_per_class[c]))
-        cm = confusion_matrix(test_labels_np, test_preds_np)
         import matplotlib
         matplotlib.use("Agg")
         import matplotlib.pyplot as plt
@@ -351,6 +414,20 @@ def main() -> None:
         fig.tight_layout()
         task.logger.report_matplotlib_figure(title="Confusion Matrix", series="test", figure=fig, iteration=0)
         plt.close(fig)
+        if num_classes == 2:
+            fpr, tpr, _ = roc_curve(test_labels_np, test_probs[:, 1])
+            fig_r, ax_r = plt.subplots(figsize=(6, 5))
+            ax_r.plot(fpr, tpr, label=f"ROC-AUC = {test_auc:.4f}")
+            ax_r.plot([0, 1], [0, 1], "k--", lw=1)
+            ax_r.set_xlabel("False positive rate")
+            ax_r.set_ylabel("True positive rate")
+            ax_r.set_title("Test ROC (MLP)")
+            ax_r.legend(loc="lower right")
+            fig_r.tight_layout()
+            task.logger.report_matplotlib_figure(
+                title="ROC", series="test", figure=fig_r, iteration=0
+            )
+            plt.close(fig_r)
         task.logger.flush()
 
 

models/xgboost/sweep.py

@@ -18,7 +18,15 @@ Usage
     python models/xgboost/sweep.py
 """
 
+import os
+import sys
 import time
+
+_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+if _ROOT not in sys.path:
+    sys.path.insert(0, _ROOT)
+
+from config import CLEARML_PROJECT_NAME
 from clearml import Task
 from clearml.automation import (
     HyperParameterOptimizer,
@@ -73,7 +81,7 @@ OBJECTIVE_SIGN          = "min"   # minimize logloss
 def main():
     # Register this controller as its own ClearML Task so it is tracked too.
     controller_task = Task.init(
-        project_name="FocusGuards Large Group Project",
+        project_name=CLEARML_PROJECT_NAME,
         task_name="XGBoost HPO Sweep Controller",
         task_type=Task.TaskTypes.optimizer,
         tags=["sweep", "xgboost", "hpo"],

models/xgboost/sweep_local.py

@@ -13,11 +13,13 @@ import numpy as np
 from xgboost import XGBClassifier
 from sklearn.metrics import f1_score, roc_auc_score, accuracy_score
 
+from config import CLEARML_PROJECT_NAME
+
 # Import your own dataset loading logic
 from data_preparation.prepare_dataset import get_default_split_config, get_numpy_splits
 
 # ── General Settings ──────────────────────────────────────────────────────────
-PROJECT_NAME = "FocusGuards Large Group Project"
+PROJECT_NAME = CLEARML_PROJECT_NAME
 BASE_TASK_NAME = "XGBoost Sweep Trial"
 DATA_SPLITS, SEED = get_default_split_config()
 

models/xgboost/train.py

@@ -1,12 +1,20 @@
+import csv
 import json
 import os
 import random
 import sys
 
 import numpy as np
-from sklearn.metrics import confusion_matrix, f1_score, precision_recall_fscore_support, roc_auc_score
-
-from data_preparation.prepare_dataset import get_numpy_splits
+from sklearn.metrics import (
+    classification_report,
+    confusion_matrix,
+    f1_score,
+    precision_recall_fscore_support,
+    roc_auc_score,
+    roc_curve,
+)
+
+from data_preparation.prepare_dataset import get_numpy_splits, SELECTED_FEATURES
 from models.xgboost.config import XGB_BASE_PARAMS, build_xgb_classifier
 
 _PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
@@ -48,12 +56,15 @@ task = None
 if USE_CLEARML:
     try:
         from clearml import Task
-        from config import flatten_for_clearml
+        from config import CLEARML_PROJECT_NAME, flatten_for_clearml
         task = Task.init(
-            project_name="Focus Guard",
+            project_name=CLEARML_PROJECT_NAME,
             task_name="XGBoost Model Training",
             tags=["training", "xgboost"],
         )
+        from config.clearml_enrich import enrich_task, upload_repro_artifacts
+
+        enrich_task(task, role="train_xgboost")
         flat = flatten_for_clearml()
         for k, v in CFG.get("xgb_params", {}).items():
             flat[f"xgb_params/{k}"] = v
@@ -61,6 +72,7 @@ if USE_CLEARML:
         flat["seed"] = CFG["seed"]
         flat["split_ratios"] = str(CFG["split_ratios"])
         task.connect(flat)
+        upload_repro_artifacts(task)
         if CLEARML_QUEUE:
             print(f"[ClearML] Enqueuing to queue '{CLEARML_QUEUE}'.")
             task.execute_remotely(queue_name=CLEARML_QUEUE)
@@ -134,6 +146,49 @@ def main():
         "test_class_counts": np.bincount(y_test.astype(int), minlength=num_classes).tolist(),
     }
 
+    logs_dir = CFG["logs_dir"]
+    os.makedirs(logs_dir, exist_ok=True)
+    cm = confusion_matrix(y_test, test_preds)
+    y_test_i = y_test.astype(int)
+    pred_path = os.path.join(logs_dir, f"xgboost_{CFG['model_name']}_test_predictions.csv")
+    with open(pred_path, "w", newline="") as f:
+        w = csv.writer(f)
+        w.writerow(["y_true", "y_pred"] + [f"prob_{j}" for j in range(num_classes)])
+        for i in range(len(y_test_i)):
+            w.writerow(
+                [int(y_test_i[i]), int(test_preds[i])]
+                + [float(x) for x in test_probs[i]]
+            )
+    summary_path = os.path.join(logs_dir, f"xgboost_{CFG['model_name']}_test_metrics_summary.json")
+    with open(summary_path, "w", encoding="utf-8") as f:
+        json.dump(
+            {
+                "model": "xgboost",
+                "model_name": CFG["model_name"],
+                "test_accuracy": round(test_acc, 6),
+                "test_f1_weighted": round(test_f1, 6),
+                "test_roc_auc": round(test_auc, 6),
+                "confusion_matrix": cm.tolist(),
+                "classification_report": classification_report(
+                    y_test, test_preds, digits=4
+                ),
+            },
+            f,
+            indent=2,
+        )
+    feat_names = list(
+        SELECTED_FEATURES.get(CFG["model_name"], SELECTED_FEATURES["face_orientation"])
+    )
+    imp_vals = model.feature_importances_
+    imp_rows = [
+        {"feature": feat_names[i], "importance": float(imp_vals[i])}
+        for i in range(min(len(feat_names), len(imp_vals)))
+    ]
+    imp_path = os.path.join(logs_dir, f"xgboost_{CFG['model_name']}_feature_importance.json")
+    with open(imp_path, "w", encoding="utf-8") as f:
+        json.dump(imp_rows, f, indent=2)
+    print(f"[LOG] Test predictions → {pred_path}")
+
     if task is not None:
         for i, (tl, vl) in enumerate(zip(train_losses, val_losses)):
             task.logger.report_scalar("Loss", "Train", tl, iteration=i + 1)
@@ -142,9 +197,11 @@ def main():
         task.logger.report_single_value("test/f1_weighted", test_f1)
         task.logger.report_single_value("test/roc_auc", test_auc)
         for key, val in dataset_stats.items():
-            task.logger.report_single_value(
-                f"dataset/{key}", str(val) if isinstance(val, list) else val
-            )
+            if isinstance(val, list):
+                for i, v in enumerate(val):
+                    task.logger.report_single_value(f"dataset/{key}/{i}", float(v))
+            else:
+                task.logger.report_single_value(f"dataset/{key}", float(val))
         prec, rec, f1_per_class, _ = precision_recall_fscore_support(
             y_test, test_preds, average=None, zero_division=0
         )
@@ -152,7 +209,6 @@ def main():
             task.logger.report_single_value(f"test/class_{c}_precision", float(prec[c]))
             task.logger.report_single_value(f"test/class_{c}_recall", float(rec[c]))
             task.logger.report_single_value(f"test/class_{c}_f1", float(f1_per_class[c]))
-        cm = confusion_matrix(y_test, test_preds)
         import matplotlib
         matplotlib.use("Agg")
         import matplotlib.pyplot as plt
@@ -171,6 +227,20 @@ def main():
         fig.tight_layout()
         task.logger.report_matplotlib_figure(title="Confusion Matrix", series="test", figure=fig, iteration=0)
         plt.close(fig)
+        if num_classes == 2:
+            fpr, tpr, _ = roc_curve(y_test, test_probs[:, 1])
+            fig_r, ax_r = plt.subplots(figsize=(6, 5))
+            ax_r.plot(fpr, tpr, label=f"ROC-AUC = {test_auc:.4f}")
+            ax_r.plot([0, 1], [0, 1], "k--", lw=1)
+            ax_r.set_xlabel("False positive rate")
+            ax_r.set_ylabel("True positive rate")
+            ax_r.set_title("Test ROC (XGBoost)")
+            ax_r.legend(loc="lower right")
+            fig_r.tight_layout()
+            task.logger.report_matplotlib_figure(
+                title="ROC", series="test", figure=fig_r, iteration=0
+            )
+            plt.close(fig_r)
         task.logger.flush()
 
     # ── Save checkpoint ───────────────────────────────────────────
@@ -200,8 +270,6 @@ def main():
         "dataset_stats": dataset_stats,
     }
 
-    logs_dir = CFG["logs_dir"]
-    os.makedirs(logs_dir, exist_ok=True)
     log_path = os.path.join(logs_dir, f"xgboost_{CFG['model_name']}_training_log.json")
 
     with open(log_path, "w") as f:
@@ -210,8 +278,30 @@ def main():
     print(f"[LOG] Training history saved to: {log_path}")
 
     if task is not None:
+        from clearml import OutputModel
+        from config.clearml_enrich import attach_output_metrics, task_done_summary
+
         task.upload_artifact(name="xgboost_model", artifact_object=model_path)
         task.upload_artifact(name="training_log", artifact_object=log_path)
+        task.upload_artifact(name="test_predictions", artifact_object=pred_path)
+        task.upload_artifact(name="test_metrics_summary", artifact_object=summary_path)
+        task.upload_artifact(name="feature_importance", artifact_object=imp_path)
+        out_model = OutputModel(
+            task=task, name=f"XGBoost_{CFG['model_name']}", framework="XGBoost"
+        )
+        out_model.update_weights(weights_filename=model_path, auto_delete_file=False)
+        attach_output_metrics(
+            out_model,
+            {
+                "test_accuracy": round(test_acc, 6),
+                "test_f1_weighted": round(test_f1, 6),
+                "test_roc_auc": round(test_auc, 6),
+            },
+        )
+        task_done_summary(
+            task,
+            f"XGBoost {CFG['model_name']}: test acc={test_acc:.4f}, F1={test_f1:.4f}, ROC-AUC={test_auc:.4f}",
+        )
 
 
 if __name__ == "__main__":