Upload eval_framework source code

.gitignore

@@ -0,0 +1,5 @@
+results/
+converted/
+__pycache__/
+*.pyc
+*.jsonl

__init__.py

@@ -0,0 +1,5 @@
+"""Unified memory evaluation framework (package scaffold)."""
+
+from __future__ import annotations
+
+__all__: list[str] = []

cli.py

@@ -0,0 +1,471 @@
+"""CLI: dataset path, baseline, output dir, dry-run, smoke eval.
+
+Evaluation uses batch LLM judge: 2 calls/session + 2 calls/QA.
+Session and QA evaluations run in parallel via ThreadPoolExecutor.
+Pipeline results are checkpointed before eval so --eval-only can resume.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+try:
+    from openai import OpenAI
+except ImportError:
+    OpenAI = None  # type: ignore[assignment]
+
+from eval_framework.config import EvalConfig
+from eval_framework.datasets.domain_a_v2 import (
+    DomainAV2AcademicBundle,
+    NormalizedCheckpointQuestion,
+    load_domain_a_v2_academic,
+)
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.evaluators.aggregate import aggregate_metrics
+from eval_framework.evaluators.extraction import evaluate_extraction
+from eval_framework.evaluators.qa import evaluate_checkpoint_qa
+from eval_framework.memory_adapters.base import MemoryAdapter
+from eval_framework.openai_compat import (
+    patch_openai_chat_completions,
+    rewrite_chat_completion_kwargs,
+)
+from eval_framework.pipeline.gold_state import GoldMemoryPoint, SessionGoldState
+from eval_framework.pipeline.records import PipelineCheckpointQARecord, PipelineSessionRecord
+from eval_framework.pipeline.runner import run_domain_a_v2_sample
+
+
+_CHECKPOINT_SESSIONS = "pipeline_sessions.jsonl"
+_CHECKPOINT_QA = "pipeline_qa.jsonl"
+
+
+# ---------------------------------------------------------------------------
+# Checkpoint deserialization: dict -> frozen dataclass
+# ---------------------------------------------------------------------------
+
+def _gold_point_from_dict(d: dict[str, Any]) -> GoldMemoryPoint:
+    return GoldMemoryPoint(
+        memory_id=d["memory_id"],
+        memory_content=d["memory_content"],
+        memory_type=d["memory_type"],
+        memory_source=d["memory_source"],
+        is_update=bool(d["is_update"]),
+        original_memories=tuple(d.get("original_memories") or ()),
+        importance=float(d.get("importance", 0.0)),
+        timestamp=d.get("timestamp"),
+        update_type=d.get("update_type", ""),
+    )
+
+
+def _gold_state_from_dict(d: dict[str, Any]) -> SessionGoldState:
+    return SessionGoldState(
+        session_id=d["session_id"],
+        cumulative_gold_memories=tuple(_gold_point_from_dict(g) for g in d["cumulative_gold_memories"]),
+        session_new_memories=tuple(_gold_point_from_dict(g) for g in d["session_new_memories"]),
+        session_update_memories=tuple(_gold_point_from_dict(g) for g in d["session_update_memories"]),
+        session_interference_memories=tuple(_gold_point_from_dict(g) for g in d["session_interference_memories"]),
+    )
+
+
+def _snapshot_record_from_dict(d: dict[str, Any]) -> MemorySnapshotRecord:
+    return MemorySnapshotRecord(
+        memory_id=d["memory_id"],
+        text=d["text"],
+        session_id=d["session_id"],
+        status=d["status"],
+        source=d.get("source"),
+        raw_backend_id=d.get("raw_backend_id"),
+        raw_backend_type=d.get("raw_backend_type"),
+        metadata=d.get("metadata") or {},
+    )
+
+
+def _delta_record_from_dict(d: dict[str, Any]) -> MemoryDeltaRecord:
+    return MemoryDeltaRecord(
+        session_id=d["session_id"],
+        op=d["op"],
+        text=d["text"],
+        linked_previous=tuple(d.get("linked_previous") or ()),
+        raw_backend_id=d.get("raw_backend_id"),
+        metadata=d.get("metadata") or {},
+    )
+
+
+def _retrieval_item_from_dict(d: dict[str, Any]) -> RetrievalItem:
+    return RetrievalItem(
+        rank=int(d["rank"]),
+        memory_id=d["memory_id"],
+        text=d["text"],
+        score=float(d["score"]),
+        raw_backend_id=d.get("raw_backend_id"),
+    )
+
+
+def _retrieval_record_from_dict(d: dict[str, Any]) -> RetrievalRecord:
+    return RetrievalRecord(
+        query=d["query"],
+        top_k=int(d["top_k"]),
+        items=[_retrieval_item_from_dict(i) for i in d["items"]],
+        raw_trace=d.get("raw_trace") or {},
+    )
+
+
+def _session_record_from_dict(d: dict[str, Any]) -> PipelineSessionRecord:
+    return PipelineSessionRecord(
+        sample_id=d["sample_id"],
+        sample_uuid=d["sample_uuid"],
+        session_id=d["session_id"],
+        memory_snapshot=tuple(_snapshot_record_from_dict(s) for s in d["memory_snapshot"]),
+        memory_delta=tuple(_delta_record_from_dict(dl) for dl in d["memory_delta"]),
+        gold_state=_gold_state_from_dict(d["gold_state"]),
+    )
+
+
+def _qa_record_from_dict(d: dict[str, Any]) -> PipelineCheckpointQARecord:
+    return PipelineCheckpointQARecord(
+        sample_id=d["sample_id"],
+        sample_uuid=d["sample_uuid"],
+        checkpoint_id=d["checkpoint_id"],
+        question=d["question"],
+        gold_answer=d["gold_answer"],
+        gold_evidence_memory_ids=tuple(d.get("gold_evidence_memory_ids") or ()),
+        gold_evidence_contents=tuple(d.get("gold_evidence_contents") or ()),
+        question_type=d["question_type"],
+        question_type_abbrev=d["question_type_abbrev"],
+        difficulty=d["difficulty"],
+        retrieval=_retrieval_record_from_dict(d["retrieval"]),
+        generated_answer=d["generated_answer"],
+        cited_memories=tuple(d.get("cited_memories") or ()),
+    )
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def _load_pipeline_checkpoint(
+    output_dir: Path,
+) -> tuple[list[PipelineSessionRecord], list[PipelineCheckpointQARecord]]:
+    """Restore pipeline records from checkpoint JSONL files."""
+    sess_path = output_dir / _CHECKPOINT_SESSIONS
+    qa_path = output_dir / _CHECKPOINT_QA
+    if not sess_path.exists() or not qa_path.exists():
+        raise SystemExit(
+            f"Checkpoint files not found in {output_dir}. "
+            f"Run without --eval-only first to generate them."
+        )
+    session_records = [_session_record_from_dict(d) for d in _read_jsonl(sess_path)]
+    qa_records = [_qa_record_from_dict(d) for d in _read_jsonl(qa_path)]
+    return session_records, qa_records
+
+
+def _default_create_adapter(baseline_name: str) -> MemoryAdapter:
+    from eval_framework.memory_adapters import registry as reg
+
+    if baseline_name in reg.MEMGALLERY_NATIVE_REGISTRY:
+        return reg.MEMGALLERY_NATIVE_REGISTRY[baseline_name]()
+    if baseline_name in reg.EXTERNAL_ADAPTER_REGISTRY:
+        return reg.EXTERNAL_ADAPTER_REGISTRY[baseline_name]()
+    known = sorted(
+        reg.MEMGALLERY_NATIVE_BASELINES | reg.EXTERNAL_ADAPTER_KEYS
+    )
+    raise SystemExit(
+        f"Unknown baseline {baseline_name!r}. "
+        f"Expected one of: {', '.join(known)}"
+    )
+
+
+def _gold_echo_answer(
+    q: NormalizedCheckpointQuestion, _retrieval: RetrievalRecord
+) -> tuple[str, list[str]]:
+    return q.gold_answer, []
+
+
+def _parse_answer_json(raw: str) -> tuple[str, list[str]]:
+    """Extract answer and cited_memories from the model's JSON response."""
+    # Try to parse as JSON first
+    try:
+        data = json.loads(raw)
+        answer = str(data.get("answer", ""))
+        cited = data.get("cited_memories", [])
+        if isinstance(cited, list):
+            return answer, [str(c) for c in cited]
+        return answer, []
+    except (json.JSONDecodeError, TypeError):
+        pass
+    # Fallback: try to find JSON block in the response
+    import re
+    m = re.search(r"\{[\s\S]*\}", raw)
+    if m:
+        try:
+            data = json.loads(m.group())
+            answer = str(data.get("answer", ""))
+            cited = data.get("cited_memories", [])
+            if isinstance(cited, list):
+                return answer, [str(c) for c in cited]
+        except (json.JSONDecodeError, TypeError):
+            pass
+    # Final fallback: treat entire response as the answer, no citations
+    return raw.strip(), []
+
+
+def build_default_answer_fn() -> Callable[
+    [NormalizedCheckpointQuestion, RetrievalRecord], tuple[str, list[str]]
+]:
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key or OpenAI is None:
+        return _gold_echo_answer
+
+    client = OpenAI(
+        api_key=api_key,
+        base_url=os.getenv("OPENAI_BASE_URL"),
+    )
+    model = os.getenv("OPENAI_MODEL") or "gpt-4o"
+    temperature = float(os.getenv("OPENAI_TEMPERATURE", "0.0"))
+    max_tokens = int(os.getenv("OPENAI_MAX_TOKENS", "1024"))
+
+    def _answer(
+        q: NormalizedCheckpointQuestion, retrieval: RetrievalRecord
+    ) -> tuple[str, list[str]]:
+        context_lines = [
+            f"[{item.rank}] {item.text}" for item in retrieval.items[: retrieval.top_k]
+        ]
+        context = "\n".join(context_lines) if context_lines else "No retrieved memories."
+        prompt = (
+            "Answer the user's question using only the retrieved memories below. "
+            "If the memories are insufficient, answer exactly: Not mentioned in memory.\n\n"
+            "You MUST also list the specific memory passages you relied on to produce "
+            "the answer. Copy the relevant text verbatim from the retrieved memories.\n\n"
+            f"Question: {q.question}\n\n"
+            f"Retrieved memories:\n{context}\n\n"
+            'Respond in JSON:\n'
+            '{\n'
+            '  "answer": "your concise answer",\n'
+            '  "cited_memories": ["verbatim passage 1", "verbatim passage 2"]\n'
+            '}\n'
+        )
+        request_kwargs = rewrite_chat_completion_kwargs(
+            {
+                "model": model,
+                "messages": [
+                    {
+                        "role": "system",
+                        "content": (
+                            "You answer benchmark questions using only supplied memory context. "
+                            "Be concise and do not invent missing facts. "
+                            "Always respond in the requested JSON format."
+                        ),
+                    },
+                    {"role": "user", "content": prompt},
+                ],
+                "temperature": temperature,
+                "max_tokens": max_tokens,
+            }
+        )
+        response = client.chat.completions.create(**request_kwargs)
+        raw = response.choices[0].message.content or ""
+        return _parse_answer_json(raw)
+
+    return _answer
+
+
+def config_from_namespace(ns: argparse.Namespace) -> EvalConfig:
+    return EvalConfig(
+        dataset_path=Path(ns.dataset).expanduser().resolve(),
+        output_dir=Path(ns.output_dir).expanduser().resolve(),
+        baseline=str(ns.baseline),
+        smoke=bool(ns.smoke),
+        dry_run=bool(ns.dry_run),
+    )
+
+
+def _record_to_json_obj(obj: Any) -> dict[str, Any]:
+    return asdict(obj)
+
+
+def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as fh:
+        for row in rows:
+            fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+
+def run_eval(
+    config: EvalConfig,
+    *,
+    load_domain_bundle: Callable[[Path], DomainAV2AcademicBundle] = load_domain_a_v2_academic,
+    create_adapter: Callable[[str], MemoryAdapter] | None = None,
+    answer_fn: Callable | None = None,
+    max_eval_workers: int = 5,
+    eval_only: bool = False,
+) -> None:
+    """Load data, run pipeline (serial) + LLM eval (parallel)."""
+    patch_openai_chat_completions()
+    if config.dry_run:
+        return
+
+    out = config.output_dir
+    out.mkdir(parents=True, exist_ok=True)
+
+    if eval_only:
+        # --- Resume from checkpoint ---
+        print(f"[Eval-only] Loading pipeline checkpoint from {out}")
+        session_records, qa_records = _load_pipeline_checkpoint(out)
+        print(f"[Eval-only] Loaded {len(session_records)} sessions + {len(qa_records)} QA records")
+    else:
+        # --- Stage 1: Pipeline (serial — adapter is stateful) ---
+        adapter_factory = create_adapter or _default_create_adapter
+        bundle = load_domain_bundle(config.dataset_path)
+        samples = bundle.samples[:1] if config.smoke else bundle.samples
+        _answer = answer_fn if answer_fn is not None else build_default_answer_fn()
+
+        session_records: list[PipelineSessionRecord] = []
+        qa_records: list[PipelineCheckpointQARecord] = []
+
+        print(f"[Pipeline] Running {len(samples)} sample(s) with baseline={config.baseline}")
+        for i, sample in enumerate(samples):
+            print(f"  Sample {i + 1}/{len(samples)}: {sample.sample_id}")
+            adapter = adapter_factory(config.baseline)
+            sess, qa = run_domain_a_v2_sample(
+                adapter,
+                sample,
+                answer_fn=_answer,
+            )
+            session_records.extend(sess)
+            qa_records.extend(qa)
+
+        # --- Save checkpoint ---
+        _write_jsonl(out / _CHECKPOINT_SESSIONS,
+                     [_record_to_json_obj(r) for r in session_records])
+        _write_jsonl(out / _CHECKPOINT_QA,
+                     [_record_to_json_obj(r) for r in qa_records])
+        print(f"[Checkpoint] Saved {len(session_records)} sessions + {len(qa_records)} QA to {out}")
+
+    # --- Stage 2: Eval (parallel — each record is self-contained) ---
+    print(f"[Eval] Evaluating {len(session_records)} sessions + {len(qa_records)} QA with LLM judge (workers={max_eval_workers})...")
+
+    session_evals: list[dict[str, object] | None] = [None] * len(session_records)
+    qa_evals: list[dict[str, object] | None] = [None] * len(qa_records)
+
+    with ThreadPoolExecutor(max_workers=max_eval_workers) as pool:
+        # Submit session evals
+        session_futures = {}
+        for idx, srec in enumerate(session_records):
+            fut = pool.submit(evaluate_extraction, srec)
+            session_futures[fut] = idx
+
+        # Submit QA evals
+        qa_futures = {}
+        for idx, qrec in enumerate(qa_records):
+            fut = pool.submit(evaluate_checkpoint_qa, qrec)
+            qa_futures[fut] = idx
+
+        # Collect session results
+        done_sessions = 0
+        for fut in as_completed(session_futures):
+            idx = session_futures[fut]
+            try:
+                session_evals[idx] = fut.result()
+            except Exception as e:
+                session_evals[idx] = {"error": str(e)}
+            done_sessions += 1
+            if done_sessions % 10 == 0 or done_sessions == len(session_records):
+                print(f"  Sessions: {done_sessions}/{len(session_records)} done")
+
+        # Collect QA results
+        done_qa = 0
+        for fut in as_completed(qa_futures):
+            idx = qa_futures[fut]
+            try:
+                qa_evals[idx] = fut.result()
+            except Exception as e:
+                qa_evals[idx] = {"error": str(e)}
+            done_qa += 1
+            if done_qa % 20 == 0 or done_qa == len(qa_records):
+                print(f"  QA: {done_qa}/{len(qa_records)} done")
+
+    # --- Stage 3: Aggregate + write ---
+    agg = aggregate_metrics(
+        config.baseline,
+        session_evaluations=[e for e in session_evals if e is not None],
+        qa_evaluations=[e for e in qa_evals if e is not None],
+    )
+
+    session_rows = []
+    for srec, s_eval in zip(session_records, session_evals):
+        row = _record_to_json_obj(srec)
+        row["eval"] = s_eval
+        session_rows.append(row)
+
+    qa_rows = []
+    for qrec, q_eval in zip(qa_records, qa_evals):
+        row = _record_to_json_obj(qrec)
+        row["eval"] = q_eval
+        qa_rows.append(row)
+
+    _write_jsonl(out / "session_records.jsonl", session_rows)
+    _write_jsonl(out / "qa_records.jsonl", qa_rows)
+    _write_json(out / "aggregate_metrics.json", agg)
+
+    print(f"\n[Done] Results written to {out}")
+    print(f"  Aggregate: {json.dumps(agg, indent=2)}")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="eval_framework")
+    p.add_argument("--dataset", required=True)
+    p.add_argument("--baseline", required=True)
+    p.add_argument("--output-dir", default="eval_framework/results")
+    p.add_argument("--smoke", action="store_true")
+    p.add_argument("--dry-run", action="store_true")
+    p.add_argument("--eval-only", action="store_true",
+                    help="Skip pipeline, load from checkpoint in output-dir.")
+    p.add_argument("--max-eval-workers", type=int, default=5,
+                    help="Parallel threads for eval stage (default 5).")
+    return p
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    cfg = config_from_namespace(args)
+
+    if cfg.dry_run:
+        print(json.dumps(cfg.to_display_dict(), indent=2))
+        return
+
+    eval_only = bool(args.eval_only)
+
+    if not eval_only and not cfg.dataset_path.is_dir():
+        raise SystemExit(f"Dataset path is not a directory: {cfg.dataset_path}")
+
+    run_eval(cfg, max_eval_workers=args.max_eval_workers, eval_only=eval_only)
+
+
+if __name__ == "__main__":
+    main()

config.py

@@ -0,0 +1,30 @@
+"""Configuration types for eval runs (CLI, dry-run, and smoke execution)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class EvalConfig:
+    """Resolved paths and flags for one eval invocation."""
+
+    dataset_path: Path
+    output_dir: Path
+    baseline: str
+    smoke: bool = False
+    dry_run: bool = False
+
+    def to_display_dict(self) -> dict[str, Any]:
+        """JSON-friendly snapshot for dry-run and logging."""
+        return {
+            "dataset_path": str(self.dataset_path),
+            "output_dir": str(self.output_dir),
+            "baseline": self.baseline,
+            "smoke": self.smoke,
+            "dry_run": self.dry_run,
+            "dataset_profile": "domain_a_v2_academic",
+            "judge": "llm (OpenAI API)",
+        }

datasets/__init__.py

@@ -0,0 +1 @@
+"""Dataset loaders and schema definitions."""

datasets/convert_vistrajqa.py

@@ -0,0 +1,452 @@
+"""Convert VisTrajQA sessions-*.jsonl → eval_framework domain_a_v2 format.
+
+Reads one or more sessions-*.jsonl files and produces the three files
+expected by ``load_domain_a_v2_academic``:
+
+    <output_dir>/
+    ├── domain_a_v2.json
+    ├── stage4_memory_points.jsonl
+    └── stage4b_qa_checkpoints.jsonl
+
+Usage:
+    python -m eval_framework.datasets.convert_vistrajqa \
+        --input data/generated/sessions-vab.jsonl \
+               data/generated/sessions-eb-nav.jsonl \
+               data/generated/sessions-arena.jsonl \
+        --output eval_framework/converted/all \
+        --text-only
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import uuid as _uuid
+from pathlib import Path
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# QA type abbreviation → full name
+# ---------------------------------------------------------------------------
+
+QA_TYPE_FULL = {
+    "FR":  "factual_recall",
+    "DU":  "dynamic_update",
+    "MB":  "memory_boundary",
+    "TR":  "temporal_reasoning",
+    "KR":  "knowledge_reasoning",
+    "VFR": "visual_factual_recall",
+    "VS":  "visual_search",
+    "VU":  "visual_update",
+    "CMR": "cross_modal_reasoning",
+}
+
+
+# ---------------------------------------------------------------------------
+# Turn construction
+# ---------------------------------------------------------------------------
+
+def _build_user_text(step: dict[str, Any], *, text_only: bool) -> str:
+    """Build user turn text from a CanonicalStep.
+
+    User turn = what the agent perceives: observation + feedback + (caption if text_only).
+    """
+    parts: list[str] = []
+    obs = step.get("observation") or ""
+    if obs:
+        parts.append(f"OBSERVATION: {obs}")
+    fb = step.get("feedback") or ""
+    if fb:
+        parts.append(f"FEEDBACK: {fb}")
+    if text_only:
+        cap = step.get("image_caption") or ""
+        if cap:
+            parts.append(f"IMAGE: {cap}")
+    if not parts:
+        parts.append("(no textual observation)")
+    return "\n".join(parts)
+
+
+def _build_assistant_text(step: dict[str, Any]) -> str:
+    """Build assistant turn text: thought + action."""
+    parts: list[str] = []
+    thought = step.get("thought") or ""
+    if thought:
+        parts.append(f"THOUGHT: {thought}")
+    action = step.get("action") or ""
+    if action:
+        parts.append(f"ACTION: {action}")
+    return "\n".join(parts) or "(no action)"
+
+
+def _build_attachment(step: dict[str, Any], *, text_only: bool) -> list[dict[str, Any]]:
+    """Build attachment list for a step (caption-only for text_only mode)."""
+    cap = step.get("image_caption") or ""
+    if not cap:
+        return []
+    if text_only:
+        # Caption already inlined in user text, no separate attachment needed
+        return []
+    image_id = step.get("image_id") or step.get("image_path") or ""
+    return [{"caption": cap, "type": "image_caption", "image_id": image_id}]
+
+
+# ---------------------------------------------------------------------------
+# Session segmentation
+# ---------------------------------------------------------------------------
+
+def _segment_steps_by_probes(
+    steps: list[dict[str, Any]],
+    probes: list[dict[str, Any]],
+    total_steps: int,
+) -> list[tuple[str, list[dict[str, Any]]]]:
+    """Split steps into sessions at probe boundaries.
+
+    Returns list of (session_id, steps_in_session).
+    Session after probe i covers steps (prev_boundary+1 .. probe_i.after_step_num].
+    The remainder after the last probe is the final session.
+    """
+    probe_bounds = sorted(set(p["after_step_num"] for p in probes))
+    boundaries = [0] + probe_bounds + [total_steps]
+
+    sessions: list[tuple[str, list[dict[str, Any]]]] = []
+    for i in range(len(boundaries) - 1):
+        lo = boundaries[i]        # exclusive lower bound (step_num > lo)
+        hi = boundaries[i + 1]    # inclusive upper bound (step_num <= hi)
+        sid = f"S{i:02d}"
+        seg = [s for s in steps if lo < s["step_num"] <= hi]
+        if seg:
+            sessions.append((sid, seg))
+
+    return sessions
+
+
+def _assign_mps_to_sessions(
+    memory_points: list[dict[str, Any]],
+    sessions: list[tuple[str, list[dict[str, Any]]]],
+) -> dict[str, list[dict[str, Any]]]:
+    """Map memory points to sessions by step_num range."""
+    # Build session_id → step_num range
+    ranges: list[tuple[str, int, int]] = []
+    for sid, seg in sessions:
+        lo = min(s["step_num"] for s in seg)
+        hi = max(s["step_num"] for s in seg)
+        ranges.append((sid, lo, hi))
+
+    result: dict[str, list[dict[str, Any]]] = {sid: [] for sid, _ in sessions}
+
+    for mp in memory_points:
+        sn = mp.get("step_num") or mp.get("probe_step_num") or 0
+        assigned = False
+        for sid, lo, hi in ranges:
+            if lo <= sn <= hi:
+                result[sid].append(mp)
+                assigned = True
+                break
+        if not assigned:
+            # Fallback: assign to last session
+            result[ranges[-1][0]].append(mp)
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Memory point conversion
+# ---------------------------------------------------------------------------
+
+def _convert_mp(mp: dict[str, Any]) -> dict[str, Any]:
+    """VisTrajQA memory point → eval_framework gold memory point dict."""
+    return {
+        "memory_id": mp.get("mp_id", ""),
+        "memory_content": mp.get("content", ""),
+        "memory_type": mp.get("type", ""),
+        "memory_source": mp.get("source", "primary"),
+        "is_update": bool(mp.get("is_update", False)),
+        "original_memories": mp.get("original_memories") or [],
+        "importance": float(mp.get("importance", 0.0)),
+        "timestamp": None,
+        "update_type": mp.get("update_type") or "",
+    }
+
+
+# ---------------------------------------------------------------------------
+# Question conversion
+# ---------------------------------------------------------------------------
+
+def _convert_question(
+    q: dict[str, Any],
+    mp_content_map: dict[str, str],
+) -> dict[str, Any]:
+    """VisTrajQA question → eval_framework checkpoint question dict."""
+    qa_type = q.get("qa_type", "FR")
+    evidence_ids = q.get("evidence") or []
+    return {
+        "question": q.get("question", ""),
+        "answer": q.get("answer", ""),
+        "question_type": QA_TYPE_FULL.get(qa_type, qa_type),
+        "question_type_abbrev": qa_type,
+        "difficulty": q.get("difficulty", "medium"),
+        "evidence": [{"memory_id": mid} for mid in evidence_ids],
+    }
+
+
+# ---------------------------------------------------------------------------
+# Main conversion
+# ---------------------------------------------------------------------------
+
+def convert_one_session(
+    rec: dict[str, Any],
+    *,
+    text_only: bool = True,
+) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
+    """Convert one VisTrajQA session record → (sample_json, stage4_row, qa_row).
+
+    Returns dicts ready for serialization into the three target files.
+    """
+    sample_id = rec["session_id"]
+    sample_uuid = str(_uuid.uuid5(_uuid.NAMESPACE_DNS, sample_id))
+    steps = rec["step_plan"]
+    total_steps = rec.get("total_steps") or len(steps)
+    probes = rec.get("probes") or []
+    post_qa = rec.get("post_trajectory_qa") or []
+    memory_points = rec.get("memory_points") or []
+
+    # mp_id → content map for evidence resolution
+    mp_content_map: dict[str, str] = {
+        mp["mp_id"]: mp.get("content", "")
+        for mp in memory_points
+        if mp.get("mp_id")
+    }
+
+    # --- Session segmentation ---
+    sessions = _segment_steps_by_probes(steps, probes, total_steps)
+    mp_by_session = _assign_mps_to_sessions(memory_points, sessions)
+
+    # --- Build domain_a_v2.json sample ---
+    session_objects: list[dict[str, Any]] = []
+    for sid, seg_steps in sessions:
+        dialogue: list[dict[str, Any]] = []
+        for step in seg_steps:
+            # User turn
+            user_text = _build_user_text(step, text_only=text_only)
+            dialogue.append({
+                "role": "user",
+                "content": user_text,
+                "timestamp": f"step_{step['step_num']:04d}",
+                "attachments": _build_attachment(step, text_only=text_only),
+            })
+            # Assistant turn
+            assistant_text = _build_assistant_text(step)
+            dialogue.append({
+                "role": "assistant",
+                "content": assistant_text,
+                "timestamp": f"step_{step['step_num']:04d}",
+                "attachments": [],
+            })
+
+        sess_obj: dict[str, Any] = {
+            "_v2_session_id": sid,
+            "dialogue": dialogue,
+        }
+        # S00 carries its own memory_points in the session object
+        if sid == "S00":
+            sess_obj["memory_points"] = [_convert_mp(mp) for mp in mp_by_session.get(sid, [])]
+        session_objects.append(sess_obj)
+
+    sample_json = {
+        "uuid": sample_uuid,
+        "sample_id": sample_id,
+        "sessions": session_objects,
+        # Metadata (not consumed by loader, but useful for debugging)
+        "_source": rec.get("source", ""),
+        "_env": rec.get("env", ""),
+        "_traj_id": rec.get("traj_id", ""),
+        "_task": rec.get("task", ""),
+        "_total_steps": total_steps,
+    }
+
+    # --- Build stage4_memory_points.jsonl row ---
+    stage4_sessions: list[dict[str, Any]] = []
+    for sid, _ in sessions:
+        if sid == "S00":
+            continue  # S00 is embedded in domain_a_v2.json
+        mps = mp_by_session.get(sid, [])
+        if mps:
+            stage4_sessions.append({
+                "session_id": sid,
+                "memory_points": [_convert_mp(mp) for mp in mps],
+            })
+
+    stage4_row = {
+        "uuid": sample_uuid,
+        "sample_id": sample_id,
+        "memory_sessions": stage4_sessions,
+    }
+
+    # --- Build stage4b_qa_checkpoints.jsonl row ---
+    session_ids = [sid for sid, _ in sessions]
+    checkpoints: list[dict[str, Any]] = []
+
+    # Probe checkpoints
+    probe_by_after_step = {p["after_step_num"]: p for p in probes}
+    cumulative_sessions: list[str] = []
+    for sid, seg_steps in sessions:
+        cumulative_sessions.append(sid)
+        max_step_in_session = max(s["step_num"] for s in seg_steps)
+        probe = probe_by_after_step.get(max_step_in_session)
+        if probe is None:
+            continue
+        questions = [
+            _convert_question(q, mp_content_map)
+            for q in probe.get("questions", [])
+        ]
+        if questions:
+            checkpoints.append({
+                "checkpoint_id": f"probe_{probe['probe_id']}",
+                "covered_sessions": list(cumulative_sessions),
+                "questions": questions,
+            })
+
+    # Post-trajectory checkpoint (covers all sessions)
+    if post_qa:
+        post_questions = [
+            _convert_question(q, mp_content_map)
+            for q in post_qa
+        ]
+        if post_questions:
+            checkpoints.append({
+                "checkpoint_id": "post_trajectory",
+                "covered_sessions": session_ids,
+                "questions": post_questions,
+            })
+
+    qa_row = {
+        "uuid": sample_uuid,
+        "sample_id": sample_id,
+        "checkpoints": checkpoints,
+    }
+
+    return sample_json, stage4_row, qa_row
+
+
+def convert_files(
+    input_paths: list[Path],
+    output_dir: Path,
+    *,
+    text_only: bool = True,
+) -> None:
+    """Read VisTrajQA session files and write the three domain_a_v2 files."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    all_samples: list[dict[str, Any]] = []
+    all_stage4: list[dict[str, Any]] = []
+    all_qa: list[dict[str, Any]] = []
+
+    for path in input_paths:
+        print(f"Reading {path} ...")
+        with path.open(encoding="utf-8") as fh:
+            for line_num, line in enumerate(fh, 1):
+                line = line.strip()
+                if not line:
+                    continue
+                rec = json.loads(line)
+                sample_json, stage4_row, qa_row = convert_one_session(
+                    rec, text_only=text_only,
+                )
+                all_samples.append(sample_json)
+                all_stage4.append(stage4_row)
+                all_qa.append(qa_row)
+
+    # Write domain_a_v2.json
+    domain_path = output_dir / "domain_a_v2.json"
+    domain_path.write_text(
+        json.dumps(all_samples, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    print(f"  → {domain_path} ({len(all_samples)} samples)")
+
+    # Write stage4_memory_points.jsonl
+    stage4_path = output_dir / "stage4_memory_points.jsonl"
+    with stage4_path.open("w", encoding="utf-8") as fh:
+        for row in all_stage4:
+            fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+    print(f"  → {stage4_path} ({len(all_stage4)} rows)")
+
+    # Write stage4b_qa_checkpoints.jsonl
+    qa_path = output_dir / "stage4b_qa_checkpoints.jsonl"
+    with qa_path.open("w", encoding="utf-8") as fh:
+        for row in all_qa:
+            fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+    print(f"  → {qa_path} ({len(all_qa)} rows)")
+
+    # --- Validation ---
+    print("\nValidating ...")
+    _validate(output_dir)
+    print("Done.")
+
+
+def _validate(output_dir: Path) -> None:
+    """Quick validation: load through the eval_framework loader."""
+    try:
+        from eval_framework.datasets.domain_a_v2 import load_domain_a_v2_academic
+        bundle = load_domain_a_v2_academic(output_dir)
+        print(f"  Loaded {len(bundle.samples)} samples successfully")
+        for sample in bundle.samples:
+            n_sessions = len(sample.sessions)
+            n_turns = sum(len(s.turns) for s in sample.sessions)
+            n_checkpoints = len(sample.normalized_checkpoints)
+            n_questions = sum(len(cp.questions) for cp in sample.normalized_checkpoints)
+            n_gold = len(sample.session_gold_states)
+            n_gold_points = sum(
+                len(g.cumulative_gold_memories)
+                for g in sample.session_gold_states[-1:]
+            )
+            print(
+                f"  {sample.sample_id}: "
+                f"{n_sessions} sessions, {n_turns} turns, "
+                f"{n_checkpoints} checkpoints, {n_questions} questions, "
+                f"{n_gold_points} gold points"
+            )
+    except Exception as e:
+        print(f"  Validation failed: {e}")
+        raise
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Convert VisTrajQA sessions → eval_framework domain_a_v2 format",
+    )
+    parser.add_argument(
+        "--input", "-i",
+        nargs="+",
+        required=True,
+        help="Path(s) to sessions-*.jsonl files",
+    )
+    parser.add_argument(
+        "--output", "-o",
+        required=True,
+        help="Output directory for the three converted files",
+    )
+    parser.add_argument(
+        "--text-only",
+        action="store_true",
+        default=True,
+        help="Inline image captions into user turn text (default: true)",
+    )
+    parser.add_argument(
+        "--multimodal",
+        action="store_true",
+        help="Keep image captions as attachments instead of inlining",
+    )
+    args = parser.parse_args()
+
+    text_only = not args.multimodal
+
+    input_paths = [Path(p).expanduser().resolve() for p in args.input]
+    output_dir = Path(args.output).expanduser().resolve()
+
+    convert_files(input_paths, output_dir, text_only=text_only)
+
+
+if __name__ == "__main__":
+    main()

datasets/domain_a_v2.py

@@ -0,0 +1,286 @@
+"""Domain A v2 academic bundle: dialogue normalization + staged QA / gold state."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterator, Mapping
+
+from eval_framework.datasets.schemas import NormalizedTurn, normalize_turn
+from eval_framework.pipeline.gold_state import (
+    SessionGoldState,
+    build_session_gold_states,
+)
+
+
+@dataclass(frozen=True)
+class Stage4Record:
+    uuid: str
+    sample_id: str
+    memory_sessions: tuple[tuple[str, tuple[Mapping[str, Any], ...]], ...]
+
+
+@dataclass(frozen=True)
+class QARecord:
+    uuid: str
+    sample_id: str
+    raw_checkpoints: tuple[Mapping[str, Any], ...]
+
+
+@dataclass(frozen=True)
+class NormalizedCheckpointQuestion:
+    question: str
+    gold_answer: str
+    gold_evidence_memory_ids: tuple[str, ...]
+    gold_evidence_contents: tuple[str, ...]
+    question_type: str
+    question_type_abbrev: str
+    difficulty: str
+
+
+@dataclass(frozen=True)
+class NormalizedCheckpoint:
+    checkpoint_id: str
+    covered_sessions: tuple[str, ...]
+    questions: tuple[NormalizedCheckpointQuestion, ...]
+
+
+@dataclass(frozen=True)
+class DomainAV2Session:
+    session_id: str
+    turns: tuple[NormalizedTurn, ...]
+
+
+@dataclass(frozen=True)
+class DomainAV2AcademicSample:
+    uuid: str
+    sample_id: str
+    sessions: tuple[DomainAV2Session, ...]
+    stage4: Stage4Record
+    qa_record: QARecord
+    normalized_checkpoints: tuple[NormalizedCheckpoint, ...]
+    session_gold_states: tuple[SessionGoldState, ...]
+
+
+@dataclass(frozen=True)
+class DomainAV2AcademicBundle:
+    samples: tuple[DomainAV2AcademicSample, ...]
+
+
+def _read_jsonl(path: Path) -> Iterator[dict[str, Any]]:
+    with path.open(encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            yield json.loads(line)
+
+
+def _stage4_from_obj(obj: Mapping[str, Any]) -> Stage4Record:
+    blocks: list[tuple[str, tuple[Mapping[str, Any], ...]]] = []
+    for ms in obj.get("memory_sessions") or []:
+        sid = str(ms.get("session_id", ""))
+        pts = ms.get("memory_points") or []
+        if not isinstance(pts, list):
+            pts = []
+        blocks.append((sid, tuple(pts)))
+    return Stage4Record(
+        uuid=str(obj["uuid"]),
+        sample_id=str(obj["sample_id"]),
+        memory_sessions=tuple(blocks),
+    )
+
+
+def _qa_from_obj(obj: Mapping[str, Any]) -> QARecord:
+    cps = obj.get("checkpoints") or []
+    if not isinstance(cps, list):
+        cps = []
+    return QARecord(
+        uuid=str(obj["uuid"]),
+        sample_id=str(obj["sample_id"]),
+        raw_checkpoints=tuple(cps),
+    )
+
+
+def _normalize_checkpoint_question(
+    raw: Mapping[str, Any],
+    memory_content_map: Mapping[str, str],
+) -> NormalizedCheckpointQuestion:
+    evidence = raw.get("evidence") or []
+    mem_ids: list[str] = []
+    mem_contents: list[str] = []
+    if isinstance(evidence, list):
+        for item in evidence:
+            if isinstance(item, dict) and "memory_id" in item:
+                mid = str(item["memory_id"])
+                mem_ids.append(mid)
+                content = memory_content_map.get(mid, "")
+                if content:
+                    mem_contents.append(content)
+    return NormalizedCheckpointQuestion(
+        question=str(raw.get("question", "")),
+        gold_answer=str(raw.get("answer", "")),
+        gold_evidence_memory_ids=tuple(mem_ids),
+        gold_evidence_contents=tuple(mem_contents),
+        question_type=str(raw.get("question_type", "")),
+        question_type_abbrev=str(raw.get("question_type_abbrev", "")),
+        difficulty=str(raw.get("difficulty", "")),
+    )
+
+
+def _normalize_checkpoints(
+    raw_checkpoints: tuple[Mapping[str, Any], ...],
+    memory_content_map: Mapping[str, str],
+) -> tuple[NormalizedCheckpoint, ...]:
+    out: list[NormalizedCheckpoint] = []
+    for cp in raw_checkpoints:
+        qs = cp.get("questions") or []
+        if not isinstance(qs, list):
+            qs = []
+        covered = cp.get("covered_sessions") or []
+        if not isinstance(covered, list):
+            covered = []
+        out.append(
+            NormalizedCheckpoint(
+                checkpoint_id=str(cp.get("checkpoint_id", "")),
+                covered_sessions=tuple(str(x) for x in covered),
+                questions=tuple(
+                    _normalize_checkpoint_question(q, memory_content_map)
+                    for q in qs
+                    if isinstance(q, Mapping)
+                ),
+            )
+        )
+    return tuple(out)
+
+
+def _dialogue_turns(sample_id: str, session_id: str, dialogue: list[Any]) -> tuple[NormalizedTurn, ...]:
+    turns: list[NormalizedTurn] = []
+    for turn_index, entry in enumerate(dialogue):
+        if not isinstance(entry, dict):
+            continue
+        text = str(entry.get("content", ""))
+        attachments_raw = entry.get("attachments") or []
+        captions: list[str] = []
+        if isinstance(attachments_raw, list):
+            for att in attachments_raw:
+                if isinstance(att, dict):
+                    cap = att.get("caption", "")
+                    captions.append(cap if isinstance(cap, str) else str(cap))
+        if captions:
+            text = text + "\n\n" + "\n".join(captions)
+        ts = entry.get("timestamp")
+        timestamp = ts if isinstance(ts, str) else (str(ts) if ts is not None else None)
+        raw_turn = {
+            "sample_id": sample_id,
+            "session_id": session_id,
+            "turn_index": turn_index,
+            "role": str(entry.get("role", "user")),
+            "text": text,
+            "attachments": [],
+            "timestamp": timestamp,
+        }
+        turns.append(normalize_turn(raw_turn))
+    return tuple(turns)
+
+
+def load_domain_a_v2_academic(data_dir: Path) -> DomainAV2AcademicBundle:
+    data_dir = data_dir.resolve()
+    main_path = data_dir / "domain_a_v2.json"
+    stage4_path = data_dir / "stage4_memory_points.jsonl"
+    qa_path = data_dir / "stage4b_qa_checkpoints.jsonl"
+
+    raw_samples = json.loads(main_path.read_text(encoding="utf-8"))
+    if not isinstance(raw_samples, list):
+        raise ValueError("domain_a_v2.json must be a list")
+
+    stage4_by_id: dict[str, Stage4Record] = {}
+    for obj in _read_jsonl(stage4_path):
+        rec = _stage4_from_obj(obj)
+        stage4_by_id[rec.sample_id] = rec
+
+    qa_by_id: dict[str, QARecord] = {}
+    for obj in _read_jsonl(qa_path):
+        rec = _qa_from_obj(obj)
+        qa_by_id[rec.sample_id] = rec
+
+    built: list[DomainAV2AcademicSample] = []
+    for item in raw_samples:
+        if not isinstance(item, dict):
+            continue
+        sample_id = str(item["sample_id"])
+        uuid = str(item["uuid"])
+        stage4 = stage4_by_id.get(sample_id)
+        qa = qa_by_id.get(sample_id)
+        if stage4 is None or qa is None:
+            raise KeyError(f"missing stage4 or QA row for sample_id={sample_id}")
+
+        stage4_map = {sid: pts for sid, pts in stage4.memory_sessions}
+
+        sessions_raw = item.get("sessions") or []
+        if not isinstance(sessions_raw, list):
+            sessions_raw = []
+
+        session_blocks: list[DomainAV2Session] = []
+        ordered_ids: list[str] = []
+        s00_points: tuple[Mapping[str, Any], ...] = ()
+
+        for sess in sessions_raw:
+            if not isinstance(sess, dict):
+                continue
+            sid = str(sess.get("_v2_session_id", ""))
+            if not sid:
+                continue
+            ordered_ids.append(sid)
+            dialogue = sess.get("dialogue") or []
+            if not isinstance(dialogue, list):
+                dialogue = []
+            session_blocks.append(
+                DomainAV2Session(
+                    session_id=sid,
+                    turns=_dialogue_turns(sample_id, sid, dialogue),
+                )
+            )
+            if sid == "S00":
+                mps = sess.get("memory_points") or []
+                if isinstance(mps, list):
+                    s00_points = tuple(mps)
+
+        gold_states = build_session_gold_states(
+            ordered_ids,
+            s00_memory_points=s00_points,
+            stage4_by_session_id=stage4_map,
+        )
+
+        # Build memory_id -> memory_content map from all sources
+        memory_content_map: dict[str, str] = {}
+        for mp_raw in s00_points:
+            if isinstance(mp_raw, Mapping):
+                mid = mp_raw.get("memory_id")
+                mc = mp_raw.get("memory_content")
+                if mid is not None and mc is not None:
+                    memory_content_map[str(mid)] = str(mc)
+        for _sid, pts in stage4.memory_sessions:
+            for mp_raw in pts:
+                if isinstance(mp_raw, Mapping):
+                    mid = mp_raw.get("memory_id")
+                    mc = mp_raw.get("memory_content")
+                    if mid is not None and mc is not None:
+                        memory_content_map[str(mid)] = str(mc)
+
+        built.append(
+            DomainAV2AcademicSample(
+                uuid=uuid,
+                sample_id=sample_id,
+                sessions=tuple(session_blocks),
+                stage4=stage4,
+                qa_record=qa,
+                normalized_checkpoints=_normalize_checkpoints(
+                    qa.raw_checkpoints, memory_content_map
+                ),
+                session_gold_states=gold_states,
+            )
+        )
+
+    return DomainAV2AcademicBundle(samples=tuple(built))

datasets/schemas.py

@@ -0,0 +1,106 @@
+"""Normalized runtime schemas shared across adapters, pipeline, and evaluators."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Mapping
+
+MemoryDeltaOp = str
+
+_VALID_DELTA_OPS: frozenset[str] = frozenset(
+    {"add", "update", "keep", "suppress", "archive"}
+)
+
+
+@dataclass(frozen=True)
+class Attachment:
+    """Caption-first attachment; image_id is optional for caption-only items."""
+
+    caption: str
+    type: str = "image_caption"
+    image_id: str | None = None
+
+
+@dataclass(frozen=True)
+class NormalizedTurn:
+    sample_id: str
+    session_id: str
+    turn_index: int
+    role: str
+    text: str
+    attachments: tuple[Attachment, ...] = ()
+    timestamp: str | None = None
+
+
+def normalize_turn(raw: Mapping[str, Any]) -> NormalizedTurn:
+    """Build a turn record, keeping attachments that only carry captions."""
+    attachments: list[Attachment] = []
+    for item in raw.get("attachments") or []:
+        if not isinstance(item, dict):
+            continue
+        cap = item.get("caption", "")
+        caption = cap if isinstance(cap, str) else str(cap)
+        iid = item.get("image_id")
+        if iid is None or iid == "":
+            image_id: str | None = None
+        else:
+            image_id = str(iid)
+        typ = item.get("type", "image_caption")
+        type_str = typ if isinstance(typ, str) else str(typ)
+        attachments.append(
+            Attachment(caption=caption, type=type_str, image_id=image_id)
+        )
+    ts = raw.get("timestamp")
+    timestamp = ts if isinstance(ts, str) else (str(ts) if ts is not None else None)
+    return NormalizedTurn(
+        sample_id=str(raw["sample_id"]),
+        session_id=str(raw["session_id"]),
+        turn_index=int(raw["turn_index"]),
+        role=str(raw["role"]),
+        text=str(raw["text"]),
+        attachments=tuple(attachments),
+        timestamp=timestamp,
+    )
+
+
+@dataclass(frozen=True)
+class MemorySnapshotRecord:
+    memory_id: str
+    text: str
+    session_id: str
+    status: str
+    source: str | None = None
+    raw_backend_id: str | None = None
+    raw_backend_type: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class MemoryDeltaRecord:
+    session_id: str
+    op: MemoryDeltaOp
+    text: str
+    linked_previous: tuple[str, ...] = ()
+    raw_backend_id: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.op not in _VALID_DELTA_OPS:
+            raise ValueError(f"invalid memory delta op: {self.op!r}")
+
+
+@dataclass(frozen=True)
+class RetrievalItem:
+    rank: int
+    memory_id: str
+    text: str
+    score: float
+    raw_backend_id: str | None = None
+
+
+@dataclass(frozen=True)
+class RetrievalRecord:
+    query: str
+    top_k: int
+    items: list[RetrievalItem]
+    raw_trace: dict[str, Any] = field(default_factory=dict)

docs/DATA_CONVERSION.md

@@ -0,0 +1,142 @@
+# VisTrajQA → Eval Framework 数据适配指南
+
+## 概述
+
+`convert_vistrajqa.py` 将 VisTrajQA 的 `sessions-*.jsonl` 转换为 eval_framework 所需的 domain_a_v2 三文件格式，从而可以用 Mem-Gallery / A-Mem / MemoryOS 等 baseline 进行统一评测。
+
+## 快速使用
+
+```bash
+# 转换所有数据源（text-only 模式，默认）
+python -m eval_framework.datasets.convert_vistrajqa \
+    --input data/generated/sessions-vab.jsonl \
+           data/generated/sessions-eb-nav.jsonl \
+           data/generated/sessions-arena.jsonl \
+           data/generated/sessions-eb-alfred.jsonl \
+           data/generated/sessions-infini-thor.jsonl \
+    --output eval_framework/converted/all
+
+# 只转换某个数据源
+python -m eval_framework.datasets.convert_vistrajqa \
+    --input data/generated/sessions-vab.jsonl \
+    --output eval_framework/converted/vab
+
+# multimodal 模式（image caption 作为 attachment 而非内联文本）
+python -m eval_framework.datasets.convert_vistrajqa \
+    --input data/generated/sessions-vab.jsonl \
+    --output eval_framework/converted/vab-mm \
+    --multimodal
+
+# 转换后直接跑 eval
+python -m eval_framework.cli \
+    --dataset eval_framework/converted/all \
+    --baseline FUMemory \
+    --output-dir eval_framework/results/FUMemory
+```
+
+## 转换映射
+
+### 数据结构映射
+
+```
+VisTrajQA session                →  eval_framework sample
+├── session_id                   →  sample_id
+├── step_plan[]                  →  sessions[].dialogue[] (user + assistant turns)
+├── probes[]                     →  checkpoints[] (probe checkpoints)
+├── post_trajectory_qa[]         →  checkpoints[-1] (post-trajectory checkpoint)
+└── memory_points[]              →  gold memory points (S00 embedded + stage4)
+```
+
+### Session 切分
+
+一条 VisTrajQA 轨迹（如 30 步，4 个 probe 在 step 6/12/18/24）按 probe 边界切分为 5 个 session：
+
+```
+步骤 1-6   → S00   (probe 1 在此 session 结束后触发)
+步骤 7-12  → S01   (probe 2)
+步骤 13-18 → S02   (probe 3)
+步骤 19-24 → S03   (probe 4)
+步骤 25-30 → S04   (post-trajectory QA 在全部 session 结束后触发)
+```
+
+这样保证 eval_framework 的 runner 在每个 session 完成后恰好触发对应的 checkpoint。
+
+### Turn 构建
+
+每个 step 生成 2 个 dialogue turn：
+
+| Turn | Role | 内容 |
+|------|------|------|
+| User turn | `user` | OBSERVATION + FEEDBACK + IMAGE caption（text-only 模式） |
+| Assistant turn | `assistant` | THOUGHT + ACTION |
+
+**text-only 模式**（默认）：image caption 直接写入 user turn 文本，格式为 `IMAGE: <caption>`。适用于所有 text-only baseline。
+
+**multimodal 模式**（`--multimodal`）：image caption 作为 `attachment` 附加，不写入正文。适用于 MMMemory 等多模态 baseline。
+
+### Memory Point 映射
+
+| VisTrajQA 字段 | eval_framework 字段 | 说明 |
+|----------------|---------------------|------|
+| `mp_id` | `memory_id` | 如 `mp_S04_1` |
+| `content` | `memory_content` | 一句话事实描述 |
+| `type` | `memory_type` | `event_memory` / `state_memory` / `spatial_memory` |
+| `source` | `memory_source` | `primary` (文本) / `secondary` (推断) |
+| `is_update` | `is_update` | 是否为更新型记忆 |
+| `original_memories` | `original_memories` | 被替换的旧内容列表 |
+| `importance` | `importance` | 0.4 / 0.6 / 0.8 / 1.0 |
+| `update_type` | `update_type` | `status_update` / `location_change` / ... |
+
+Memory point 按 `step_num` 分配到对应 session：
+- S00 的 memory points 嵌入在 `domain_a_v2.json` 的 session 对象中
+- 其他 session 的 memory points 写入 `stage4_memory_points.jsonl`
+
+### QA / Checkpoint 映射
+
+**Probe checkpoint**：每个 probe 生成一个 checkpoint，`covered_sessions` 为该 probe 及之前所有 session。
+
+**Post-trajectory checkpoint**：覆盖全部 session，包含 9 类 QA。
+
+| VisTrajQA QA type | eval_framework question_type | 缩写 |
+|----|----|-----|
+| FR | factual_recall | FR |
+| DU | dynamic_update | DU |
+| MB | memory_boundary | MB |
+| TR | temporal_reasoning | TR |
+| KR | knowledge_reasoning | KR |
+| VFR | visual_factual_recall | VFR |
+| VS | visual_search | VS |
+| VU | visual_update | VU |
+| CMR | cross_modal_reasoning | CMR |
+
+Evidence 字段从 `["mp_S04_1"]`（字符串列表）转换为 `[{"memory_id": "mp_S04_1"}]`（字典列表）以匹配 eval_framework 格式。
+
+## 输出文件
+
+```
+eval_framework/converted/all/
+├── domain_a_v2.json               # 主对话数据 (JSON array)
+├── stage4_memory_points.jsonl     # 每 session 的 gold memory points
+└── stage4b_qa_checkpoints.jsonl   # checkpoint QA 题目
+```
+
+## 评测维度与 VisTrajQA 的对应
+
+| eval_framework 维度 | 测量内容 | 对应 VisTrajQA 特性 |
+|-----|-----|-----|
+| Memory Recall | 记忆系统存储了多少 gold points | 直接对应，所有 MP 类型 |
+| Memory Correctness | 存储的记忆是否正确 | 检测 hallucination |
+| Update Handling | 更新型记忆是否正确替换 | 对应 `is_update=true` 的 MP |
+| Interference Rejection | 干扰信息是否被过滤 | VisTrajQA 无 interference 标注，此维度为空 |
+| QA Accuracy | 问答正确率 | 对应 9 类 QA (FR/DU/MB/TR/KR/VFR/VS/VU/CMR) |
+| Evidence Coverage | 回答引用了多少 gold evidence | 对应 evidence memory_point_ids |
+
+> **注意**：VisTrajQA 没有 interference（干扰信息）标注，因此 eval_framework 的 Interference Rejection 维度在评测结果中会为空值。MB（Memory Boundary）类型的题目在 QA 层面测试了类似能力。
+
+## 注意事项
+
+1. **text-only baseline（FU/ST/LT/GA/MG/RF）**：使用默认 `--text-only`，image caption 内联到用户消息文本中
+2. **multimodal baseline（MM/MMFU/NG/AUGUSTUS）**：使用 `--multimodal`，caption 作为 attachment
+3. **caption 质量**：text-only baseline 对图像的理解完全依赖 caption 质量。如果 `image_caption` 为空，用户 turn 中不会有任何视觉信息
+4. **Arena 数据**：observation 恒为空字符串，视觉信息完全来自 image_caption
+5. **转换器会自动验证**：运行后会调用 `load_domain_a_v2_academic` 检验输出是否合法

docs/EXPERIMENTS.md

@@ -0,0 +1,87 @@
+# 实验记录
+
+## 实验环境
+
+- **数据**：VisTrajQA VAB smoke（1 sample, vab_minecraft, 30 步, 5 sessions, 45 QA）
+- **转换模式**：text-only（image caption 内联到 user turn 文本）
+- **Judge / Answer 模型**：从 `.env` 读取 `OPENAI_MODEL`
+- **日期**：2026-04-15
+
+---
+
+## 总表
+
+### Memory 维度
+
+| Baseline | Recall ↑ | Update Recall ↑ | Correctness ↑ | Hallucination ↓ | Irrelevant ↓ | Update Score ↑ |
+|----------|----------|-----------------|---------------|-----------------|--------------|----------------|
+| Dummy | 95.0% | 90.0% | 50.0% | 5.0% | 45.0% | 71.4% |
+| FUMemory | 95.0% | 90.0% | 80.0% | 10.0% | 10.0% | 69.0% |
+| STMemory | 95.0% | 90.0% | 83.3% | 6.7% | 10.0% | 71.4% |
+| LTMemory | 95.0% | 90.0% | 83.3% | 6.7% | 10.0% | 69.0% |
+| GAMemory | 92.1% | 90.0% | 62.5% | 22.5% | 15.0% | 71.4% |
+| MGMemory | 95.0% | 90.0% | 83.3% | 6.7% | 10.0% | 71.4% |
+| RFMemory | 95.0% | 90.0% | 90.0% | 6.7% | 3.3% | 71.4% |
+| MMMemory | 95.0% | 90.0% | 83.3% | 6.7% | 10.0% | 71.4% |
+| MMFUMemory | 95.0% | 90.0% | 83.3% | 6.7% | 10.0% | 71.4% |
+| NGMemory | 95.0% | 90.0% | 83.3% | 6.7% | 10.0% | 71.4% |
+| AUGUSTUSMemory | 95.0% | 90.0% | 83.3% | 6.7% | 10.0% | 71.4% |
+| UniversalRAGMemory | 95.0% | 90.0% | 80.0% | 10.0% | 10.0% | 71.4% |
+| Mem0 | 47.9% | 56.7% | 57.3% | 18.0% | 24.7% | 81.0% |
+| Mem0-Graph | 48.2% | 56.7% | 37.9% | 14.6% | 38.1% | 71.4% |
+| SimpleMem | 95.0% | 90.0% | 48.3% | 5.0% | 46.7% | 71.4% |
+| Omni-SimpleMem | — | — | — | — | — | — |
+| MemVerse | 95.0% | 90.0% | 65.0% | 23.3% | 11.7% | 71.4% |
+| Zep | — | — | — | — | — | — |
+| A-Mem | 95.0% | 90.0% | 56.7% | 5.0% | 38.3% | 71.4% |
+| MemoryOS | 47.1% | 41.0% | 33.8% | 3.5% | 62.8% | 50.0% |
+
+### QA 维度
+
+| Baseline | QA Correct ↑ | QA Hallucination ↓ | QA Omission ↓ | Evidence Coverage ↑ |
+|----------|--------------|---------------------|---------------|---------------------|
+| Dummy | 44.4% | 24.4% | 31.1% | 47.6% |
+| FUMemory | 57.8% | 40.0% | 2.2% | 65.0% |
+| STMemory | 31.1% | 33.3% | 35.6% | 32.0% |
+| LTMemory | 40.0% | 26.7% | 33.3% | 51.5% |
+| GAMemory | 37.8% | 20.0% | 42.2% | 51.5% |
+| MGMemory | 60.0% | 35.6% | 4.4% | 68.9% |
+| RFMemory | 60.0% | 37.8% | 2.2% | 63.1% |
+| MMMemory | 57.8% | 40.0% | 2.2% | 66.0% |
+| MMFUMemory | 60.0% | 37.8% | 2.2% | 67.0% |
+| NGMemory | 62.2% | 24.4% | 13.3% | 71.8% |
+| AUGUSTUSMemory | 64.4% | 22.2% | 13.3% | 68.9% |
+| UniversalRAGMemory | 40.0% | 28.9% | 31.1% | 49.5% |
+| Mem0 | 26.7% | 28.9% | 44.4% | 31.1% |
+| Mem0-Graph | 31.1% | 20.0% | 48.9% | 32.0% |
+| SimpleMem | 57.8% | 22.2% | 20.0% | 35.9% |
+| Omni-SimpleMem | — | — | — | — |
+| MemVerse | 40.0% | 28.9% | 31.1% | 40.8% |
+| Zep | — | — | — | — |
+| A-Mem | 46.7% | 20.0% | 33.3% | 50.5% |
+| MemoryOS | 28.9% | 24.4% | 46.7% | 37.9% |
+
+### QA 分类型正确率
+
+| Baseline | FR | DU | MB | TR | KR | VFR | VS | VU | CMR |
+|----------|----|----|----|----|----|----|----|----|-----|
+| Dummy | 5/5 | 0/5 | 4/5 | 2/5 | 2/5 | 4/5 | 0/5 | 1/5 | 2/5 |
+| FUMemory | 4/5 | 3/5 | 5/5 | 4/5 | 4/5 | 4/5 | 0/5 | 2/5 | 0/5 |
+| STMemory | 0/5 | 0/5 | 5/5 | 1/5 | 3/5 | 2/5 | 0/5 | 0/5 | 3/5 |
+| LTMemory | 2/5 | 1/5 | 5/5 | 1/5 | 3/5 | 4/5 | 0/5 | 0/5 | 2/5 |
+| GAMemory | 4/5 | 0/5 | 5/5 | 0/5 | 2/5 | 4/5 | 0/5 | 0/5 | 2/5 |
+| MGMemory | 4/5 | 3/5 | 5/5 | 4/5 | 4/5 | 4/5 | 0/5 | 1/5 | 2/5 |
+| RFMemory | 4/5 | 3/5 | 5/5 | 4/5 | 4/5 | 4/5 | 0/5 | 1/5 | 2/5 |
+| MMMemory | 4/5 | 3/5 | 5/5 | 4/5 | 4/5 | 4/5 | 0/5 | 2/5 | 0/5 |
+| MMFUMemory | 4/5 | 3/5 | 5/5 | 4/5 | 4/5 | 4/5 | 0/5 | 2/5 | 1/5 |
+| NGMemory | 5/5 | 3/5 | 5/5 | 3/5 | 3/5 | 4/5 | 0/5 | 2/5 | 3/5 |
+| AUGUSTUSMemory | 5/5 | 2/5 | 5/5 | 4/5 | 3/5 | 4/5 | 1/5 | 2/5 | 3/5 |
+| UniversalRAGMemory | 2/5 | 1/5 | 5/5 | 1/5 | 3/5 | 4/5 | 0/5 | 0/5 | 2/5 |
+| Mem0 | 2/5 | 1/5 | 5/5 | 0/5 | 0/5 | 2/5 | 0/5 | 0/5 | 2/5 |
+| Mem0-Graph | 2/5 | 0/5 | 5/5 | 0/5 | 1/5 | 3/5 | 0/5 | 0/5 | 3/5 |
+| SimpleMem | 5/5 | 3/5 | 4/5 | 3/5 | 2/5 | 2/5 | 0/5 | 3/5 | 4/5 |
+| Omni-SimpleMem | — | — | — | — | — | — | — | — | — |
+| MemVerse | 5/5 | 0/5 | 5/5 | 0/5 | 2/5 | 3/5 | 0/5 | 0/5 | 3/5 |
+| Zep | — | — | — | — | — | — | — | — | — |
+| A-Mem | 3/5 | 3/5 | 5/5 | 0/5 | 2/5 | 4/5 | 0/5 | 0/5 | 4/5 |
+| MemoryOS | 4/5 | 1/5 | 5/5 | 0/5 | 0/5 | 1/5 | 0/5 | 0/5 | 2/5 |

docs/GUIDE.md

@@ -0,0 +1,423 @@
+# Eval Framework 使用指南
+
+## 1. 整体架构
+
+```
+eval_framework/
+├── cli.py                    # 入口：CLI 解析 + 三阶段编排 (Pipeline → Eval → Aggregate)
+├── config.py                 # EvalConfig 数据类
+├── openai_compat.py          # GPT-5 系列 max_tokens→max_completion_tokens 兼容补丁
+├── datasets/
+│   ├── schemas.py            # 运行时共享数据结构 (NormalizedTurn, MemorySnapshotRecord, RetrievalRecord 等)
+│   └── domain_a_v2.py        # domain_a_v2 数据集加载器
+├── memory_adapters/
+│   ├── base.py               # MemoryAdapter 抽象基类 (7 个接口方法)
+│   ├── registry.py           # Baseline 注册表 + Mem-Gallery 默认配置覆盖
+│   ├── memgallery_native.py  # Mem-Gallery 11 种内置 baseline 的统一适配器
+│   ├── amem.py               # A-Mem 外部 baseline 适配器
+│   ├── memoryos.py           # MemoryOS 外部 baseline 适配器
+│   └── export_utils.py       # 快照/检索结果归一化工具
+├── pipeline/
+│   ├── runner.py             # 按 session 顺序喂入对话 → 生成 snapshot/delta → 触发 QA
+│   ├── qa_runner.py          # 对每个 checkpoint question 做 retrieve + answer
+│   ├── gold_state.py         # Gold memory points 累积构建
+│   └── records.py            # PipelineSessionRecord / PipelineCheckpointQARecord
+├── evaluators/
+│   ├── extraction.py         # Session 级评估：Recall + Correctness + Update + Interference
+│   ├── qa.py                 # Checkpoint QA 评估：Answer 正确性 + Evidence 覆盖率
+│   └── aggregate.py          # 聚合所有 session/QA 评估到 baseline 级汇总指标
+└── judges/
+    ├── llm_client.py         # OpenAI 兼容 LLM 调用 + JSON 解析 + 重试 + 并发控制
+    └── prompts.py            # 6 套 LLM judge prompt 模板
+```
+
+## 2. 运行流程
+
+整个 eval 分三个阶段（`cli.py: run_eval()`）：
+
+### Stage 1 — Pipeline（串行，适配器有状态）
+
+```
+for each sample:
+    adapter = create_adapter(baseline_name)
+    adapter.reset()
+    for each session in sample.sessions:
+        for each turn in session.turns:
+            adapter.ingest_turn(turn)          # 喂入一条对话
+        adapter.end_session(session_id)        # 触发 session 后处理（如 GA 反思、RF 优化）
+        snapshot = adapter.snapshot_memories()  # 拍快照
+        delta = adapter.export_memory_delta()  # 导出本 session 增量
+        → PipelineSessionRecord
+
+        # 当某个 checkpoint 的 covered_sessions 全部完成时触发 QA
+        for each question in checkpoint:
+            retrieval = adapter.retrieve(question, top_k=5)
+            answer = answer_fn(question, retrieval)   # 可注入外部 LLM 回答
+            → PipelineCheckpointQARecord
+```
+
+Pipeline 结束后写入 checkpoint 文件 `pipeline_sessions.jsonl` + `pipeline_qa.jsonl`，支持 `--eval-only` 跳过此阶段直接从 checkpoint 恢复。
+
+### Stage 2 — Eval（并行，ThreadPoolExecutor）
+
+- **Session 评估**（`evaluators/extraction.py`）— 每个 session 4+ 次 LLM 调用：
+  1. **Recall**：本 session 的 gold points 中有多少被 delta 覆盖？
+  2. **Correctness**：每条 delta 记忆是 correct / hallucination / irrelevant？
+  3. **Update handling**：每个 update gold point → updated / both / outdated
+  4. **Interference rejection**：每个 interference gold point → rejected / memorized
+
+- **QA 评估**（`evaluators/qa.py`）— 每个 question 2 次 LLM 调用：
+  1. **Answer 正确性**：Correct / Hallucination / Omission
+  2. **Evidence 覆盖率**：cited memories 覆盖了多少 gold evidence points
+
+### Stage 3 — Aggregate
+
+将所有 session 和 QA 级别的评估结果聚合为 6 个维度的 baseline 级指标：
+
+| 维度 | 聚合方式 | 关键指标 |
+|------|---------|---------|
+| Memory Recall | 按 session 平均 | `avg_recall`, `avg_update_recall` |
+| Memory Correctness | 按 session 平均 | `avg_correctness`, `avg_hallucination` |
+| Update Handling | 跨 session 池化 | `score` (updated=1.0, both=0.5, outdated=0.0) |
+| Interference Rejection | 跨 session 池化 | `score` (rejected/total) |
+| Question Answering | 跨 question 池化 | `correct_ratio`, `hallucination_ratio`, `omission_ratio` |
+| Evidence Coverage | 跨 question 池化 | `hit_rate` |
+
+输出文件：
+- `session_records.jsonl` — 每条含 pipeline 数据 + eval 结果
+- `qa_records.jsonl` — 同上
+- `aggregate_metrics.json` — baseline 级汇总
+
+## 3. 支持的 Baselines
+
+### 3.1 Mem-Gallery 内置（11 种）
+
+通过 `MemGalleryNativeAdapter` 统一包装，需要在 `eval_framework/` 同级目录放置 `memengine/` 和 `default_config/`（从 Mem-Gallery 的 `benchmark/` 目录复制）。
+
+| Baseline | 类型 | 特性 | 额外依赖 |
+|----------|------|------|---------|
+| `FUMemory` | text-only | 全量存储（FIFO 截断） | — |
+| `STMemory` | text-only | 短期记忆 | — |
+| `LTMemory` | text-only | 长期记忆，embedding 检索 | sentence-transformers |
+| `GAMemory` | text-only | 带 importance judge + 自反思 | LLM API |
+| `MGMemory` | text-only | 多层存储（working/FIFO/recall/archival） | LLM API, sentence-transformers |
+| `RFMemory` | text-only | 带 reflection optimizer | LLM API |
+| `MMMemory` | multimodal | 多模态记忆 | torch |
+| `MMFUMemory` | multimodal | 多模态全量存储 | torch |
+| `NGMemory` | multimodal | 知识图谱节点存储 | torch |
+| `AUGUSTUSMemory` | multimodal | 概念抽取 + 图谱 | LLM API, torch |
+| `UniversalRAGMemory` | multimodal | RAG routing + 存储 | LLM API |
+
+### 3.2 外部适配器
+
+| Baseline | 来源 | 安装方式 | 需要外部服务 |
+|----------|------|---------|-------------|
+| `Mem0` | [mem0ai/mem0](https://github.com/mem0ai/mem0) | `pip install mem0ai` | 否（内置 Qdrant + SQLite） |
+| `Mem0-Graph` | 同上（graph 模式） | `pip install "mem0ai[graph]"` | 需要 Neo4j |
+| `SimpleMem` | [aiming-lab/SimpleMem](https://github.com/aiming-lab/SimpleMem) | clone + requirements | 否 |
+| `Omni-SimpleMem` | 同上（omni 模式） | 同上 | 否 |
+| `Zep` | [getzep/zep](https://github.com/getzep/zep) | `pip install zep-python` | 需要 Zep server |
+| `A-Mem` | [A-Mem](https://arxiv.org/abs/2504.19413) | clone 源码 | 否 |
+| `MemoryOS` | [MemoryOS](https://github.com/memodb-io/memobase) | clone 源码 | 否 |
+
+**论文来源：**
+
+| Baseline | 论文 | GitHub |
+|----------|------|--------|
+| Mem0 / Mem0-Graph | [arXiv:2504.19413](https://arxiv.org/abs/2504.19413) | https://github.com/mem0ai/mem0 |
+| SimpleMem | [arXiv:2601.02553](https://arxiv.org/abs/2601.02553) | https://github.com/aiming-lab/SimpleMem |
+| Omni-SimpleMem | [arXiv:2604.01007](https://arxiv.org/abs/2604.01007) | https://github.com/aiming-lab/SimpleMem |
+| MemVerse | [arXiv:2512.03627](https://arxiv.org/abs/2512.03627) | https://github.com/KnowledgeXLab/MemVerse |
+| Memobase | — | https://github.com/memodb-io/memobase |
+| Supermemory | — | https://github.com/supermemoryai/supermemory |
+| Zep | [arXiv:2501.13956](https://arxiv.org/abs/2501.13956) | https://github.com/getzep/zep |
+
+### 3.3 添加新 Baseline
+
+实现 `MemoryAdapter` 的 7 个抽象方法：
+
+```python
+class MyAdapter(MemoryAdapter):
+    def reset(self) -> None: ...
+    def ingest_turn(self, turn: NormalizedTurn) -> None: ...
+    def end_session(self, session_id: str) -> None: ...
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]: ...
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]: ...
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord: ...
+    def get_capabilities(self) -> dict[str, Any]: ...
+```
+
+然后在 `registry.py` 的 `EXTERNAL_ADAPTER_REGISTRY` 中注册。
+
+## 4. 数据适配
+
+### 4.1 数据集格式（domain_a_v2）
+
+加载器 `load_domain_a_v2_academic(data_dir)` 要求 `data_dir` 下有三个文件：
+
+```
+data_dir/
+├── domain_a_v2.json                # 主对话数据（JSON array）
+├── stage4_memory_points.jsonl      # 每 session 的 gold memory points
+└── stage4b_qa_checkpoints.jsonl    # checkpoint QA 题目
+```
+
+**`domain_a_v2.json`** 中每个 sample 结构：
+
+```json
+{
+  "uuid": "unique-id",
+  "sample_id": "sample_001",
+  "sessions": [
+    {
+      "_v2_session_id": "S00",
+      "dialogue": [
+        {
+          "role": "user",
+          "content": "Hello...",
+          "timestamp": "2025-01-01T10:00:00",
+          "attachments": [{"caption": "photo of...", "type": "image_caption"}]
+        },
+        {"role": "assistant", "content": "Hi..."}
+      ],
+      "memory_points": [...]   // 仅 S00 需要
+    },
+    {"_v2_session_id": "S01", "dialogue": [...]}
+  ]
+}
+```
+
+**`stage4_memory_points.jsonl`** 每行一个 sample：
+
+```json
+{
+  "uuid": "...", "sample_id": "sample_001",
+  "memory_sessions": [
+    {
+      "session_id": "S01",
+      "memory_points": [
+        {
+          "memory_id": "m001",
+          "memory_content": "User prefers dark mode",
+          "memory_type": "preference",
+          "memory_source": "normal",
+          "is_update": false,
+          "original_memories": [],
+          "importance": 0.8
+        }
+      ]
+    }
+  ]
+}
+```
+
+**`stage4b_qa_checkpoints.jsonl`** 每行一个 sample：
+
+```json
+{
+  "uuid": "...", "sample_id": "sample_001",
+  "checkpoints": [
+    {
+      "checkpoint_id": "cp01",
+      "covered_sessions": ["S00", "S01"],
+      "questions": [
+        {
+          "question": "What theme does the user prefer?",
+          "answer": "Dark mode",
+          "question_type": "preference_recall",
+          "question_type_abbrev": "pref",
+          "difficulty": "easy",
+          "evidence": [{"memory_id": "m001"}]
+        }
+      ]
+    }
+  ]
+}
+```
+
+### 4.2 适配自有数据
+
+若要接入新数据源，有两条路径：
+
+**路径 A：��换为 domain_a_v2 格式**（推荐）
+- 将原始对话整理为上述三文件格式
+- 直接使用现有 CLI 运行
+
+**路径 B：编写新的 dataset loader**
+- 在 `datasets/` 下新建加载器，返回 `DomainAV2AcademicBundle`（或等价结构）
+- 在 `cli.py` 的 `run_eval()` 中通过 `load_domain_bundle` 参数注入
+
+### 4.3 关键数据结构
+
+每条对话 turn 会被归一化为 `NormalizedTurn`：
+
+```python
+NormalizedTurn(
+    sample_id="sample_001",
+    session_id="S01",
+    turn_index=0,
+    role="user",           # "user" | "assistant"
+    text="Hello...",
+    attachments=(Attachment(caption="...", type="image_caption"),),
+    timestamp="2025-01-01T10:00:00",
+)
+```
+
+Memory 的 gold 标注支持三种来源标记：
+- `normal` — 正常记忆点
+- `interference` — 干扰信息（不应被记忆）
+- `is_update=True` — 更新型记忆（应替换旧记忆）
+
+## 5. 环境配置（uv）
+
+### 5.1 安装 uv
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+### 5.2 初始化项目环境
+
+```bash
+cd /data1/toby/nips26
+
+# 创建虚拟环境
+uv venv .venv --python 3.11
+source .venv/bin/activate
+```
+
+### 5.3 安装核心依赖
+
+```bash
+# 最小依赖（可跑 FUMemory/STMemory 等纯文本 baseline）
+uv pip install openai tenacity
+
+# embedding 检索类 baseline（LTMemory, GAMemory, MGMemory 等）
+uv pip install sentence-transformers
+
+# 多模态 baseline（MMMemory, NGMemory, AUGUSTUSMemory 等）
+uv pip install torch torchvision transformers
+
+# 外部 baseline（A-Mem, MemoryOS）— 按各自文档安装额外依赖
+# A-Mem 需要其源码目录下的 requirements
+# MemoryOS 需要 memoryos 包
+```
+
+### 5.4 环境变量（.env 文件）
+
+在项目根目录 (`nips26/`) 创建 `.env` 文件，框架会自动加载：
+
+```bash
+# .env
+# 必需 — LLM API（pipeline 答题 + judge 评估统一使用）
+OPENAI_API_KEY=sk-...
+OPENAI_BASE_URL=https://api.openai.com/v1   # 或兼容端点
+OPENAI_MODEL=gpt-4o
+
+# 可选
+OPENAI_TEMPERATURE=0.0
+OPENAI_MAX_TOKENS=1024
+OPENAI_TIMEOUT=120
+JUDGE_TEMPERATURE=0.0         # judge 专用温度
+LLM_MAX_CONCURRENT=5          # LLM 并发上限
+```
+
+### 5.5 Mem-Gallery 本地依赖
+
+Mem-Gallery 内置 baseline 需要将其源码放到 `eval_framework/` 的同级目录：
+
+```bash
+# 假设 Mem-Gallery repo 在 /path/to/Mem-Gallery
+cp -r /path/to/Mem-Gallery/benchmark/memengine  /data1/toby/nips26/
+cp -r /path/to/Mem-Gallery/benchmark/default_config  /data1/toby/nips26/
+```
+
+最终目录结构应为：
+
+```
+nips26/
+├── eval_framework/
+├── memengine/          # Mem-Gallery 记忆引擎
+└── default_config/     # Mem-Gallery 默认配置
+```
+
+## 6. 运行示例
+
+### 基本运行
+
+```bash
+# 运行单个 baseline
+python -m eval_framework.cli \
+    --dataset /path/to/domain_a_v2_data/ \
+    --baseline FUMemory \
+    --output-dir eval_framework/results/FUMemory
+
+# smoke 模式（只跑第 1 个 sample，快速验证）
+python -m eval_framework.cli \
+    --dataset /path/to/domain_a_v2_data/ \
+    --baseline FUMemory \
+    --output-dir eval_framework/results/FUMemory_smoke \
+    --smoke
+
+# dry-run（不实际运行，打印配置）
+python -m eval_framework.cli \
+    --dataset /path/to/domain_a_v2_data/ \
+    --baseline FUMemory \
+    --dry-run
+
+# 仅重跑 eval 阶段（从 checkpoint 恢复，pipeline 不重跑）
+python -m eval_framework.cli \
+    --dataset /path/to/domain_a_v2_data/ \
+    --baseline FUMemory \
+    --output-dir eval_framework/results/FUMemory \
+    --eval-only
+
+# 调整 eval 并发数
+python -m eval_framework.cli \
+    --dataset /path/to/domain_a_v2_data/ \
+    --baseline MGMemory \
+    --output-dir eval_framework/results/MGMemory \
+    --max-eval-workers 10
+```
+
+### 批量跑所有 baseline
+
+```bash
+DATASET="/path/to/domain_a_v2_data"
+for baseline in FUMemory STMemory LTMemory GAMemory MGMemory RFMemory A-Mem MemoryOS; do
+    echo "=== Running $baseline ==="
+    python -m eval_framework.cli \
+        --dataset "$DATASET" \
+        --baseline "$baseline" \
+        --output-dir "eval_framework/results/$baseline"
+done
+```
+
+### 输出文件说明
+
+运行完成后 `output-dir` 下包含：
+
+```
+results/FUMemory/
+├── pipeline_sessions.jsonl     # Stage 1 checkpoint — session 级 pipeline 结果
+├── pipeline_qa.jsonl           # Stage 1 checkpoint — QA 级 pipeline 结果
+├── session_records.jsonl       # 最终 session 结果（含 eval）
+├── qa_records.jsonl            # 最终 QA 结果（含 eval）
+└── aggregate_metrics.json      # baseline 级汇总指标
+```
+
+## 7. LLM API 开销估算
+
+每个 sample 的 LLM 调用量：
+
+| 来源 | 调用次数 |
+|------|---------|
+| Pipeline answer（每个 QA question） | N_questions |
+| Session Recall judge | N_sessions |
+| Session Correctness judge | N_sessions |
+| Update judge | N_update_points（逐条） |
+| Interference judge | N_interference_points（逐条） |
+| QA Answer judge | N_questions |
+| QA Evidence judge | N_questions |
+
+典型场景下一个 sample 约 20-50 次 LLM 调用。通过 `LLM_MAX_CONCURRENT` 控制并发避免 rate limit。

docs/OUTPUT_FORMAT.md

@@ -0,0 +1,261 @@
+# Eval Framework 输出格式
+
+## 输出目录结构
+
+运行完成后 `--output-dir` 下包含 5 个文件：
+
+```
+output-dir/
+├── pipeline_sessions.jsonl     # Stage 1 checkpoint — pipeline 中间结果（session 级）
+├── pipeline_qa.jsonl           # Stage 1 checkpoint — pipeline 中间结果（QA 级）
+├── session_records.jsonl       # 最终结果：session pipeline 数据 + eval 评判
+├── qa_records.jsonl            # 最终结果：QA pipeline 数据 + eval 评判
+└── aggregate_metrics.json      # 最终结果：baseline 级别汇总指标
+```
+
+## 文件详解
+
+### 1. `session_records.jsonl`
+
+每行一个 session，包含 pipeline 原始数据和 `eval` 评判结果：
+
+```json
+{
+  "sample_id": "vab_minecraft_...",
+  "sample_uuid": "uuid-...",
+  "session_id": "S01",
+  "memory_snapshot": [
+    {
+      "memory_id": "3",
+      "text": "user: OBSERVATION: ...\nassistant: THOUGHT: ...",
+      "session_id": "S01",
+      "status": "active",
+      "source": "FUMemory",
+      "raw_backend_id": "3",
+      "raw_backend_type": "linear",
+      "metadata": {}
+    }
+  ],
+  "memory_delta": [
+    {
+      "session_id": "S01",
+      "op": "add",
+      "text": "user: OBSERVATION: ...",
+      "linked_previous": [],
+      "raw_backend_id": "3",
+      "metadata": {"baseline": "FUMemory"}
+    }
+  ],
+  "gold_state": {
+    "session_id": "S01",
+    "cumulative_gold_memories": [...],
+    "session_new_memories": [...],
+    "session_update_memories": [...],
+    "session_interference_memories": []
+  },
+  "eval": {
+    "session_id": "S01",
+    "recall": 0.8,
+    "covered_count": 4,
+    "num_gold": 5,
+    "update_recall": 1.0,
+    "update_covered_count": 2,
+    "update_total": 2,
+    "recall_reasoning": "4 of 5 gold points are covered...",
+    "correctness_rate": 0.75,
+    "num_memories": 8,
+    "num_correct": 6,
+    "num_hallucination": 1,
+    "num_irrelevant": 1,
+    "correctness_reasoning": "...",
+    "correctness_records": [
+      {"id": 1, "label": "correct"},
+      {"id": 2, "label": "hallucination"}
+    ],
+    "update_score": 1.0,
+    "update_num_updated": 2,
+    "update_num_both": 0,
+    "update_num_outdated": 0,
+    "update_total_items": 2,
+    "update_records": [
+      {"memory_id": "mp_S08_3", "label": "updated", "reasoning": "..."}
+    ],
+    "interference_score": null,
+    "interference_num_rejected": 0,
+    "interference_num_memorized": 0,
+    "interference_total_items": 0,
+    "interference_records": []
+  }
+}
+```
+
+**eval 字段说明：**
+
+| 字段 | 含义 |
+|------|------|
+| `recall` | 本 session gold points 被 delta 覆盖的比例 (0-1) |
+| `update_recall` | update 类型 gold points 的覆盖比例 |
+| `correctness_rate` | delta 中正确记忆的比例 |
+| `num_hallucination` | delta 中幻觉记忆数量 |
+| `num_irrelevant` | delta 中无关记忆数量 |
+| `update_score` | 更新处理得分 (updated=1.0, both=0.5, outdated=0.0) |
+| `interference_score` | 干扰拒绝得分 (rejected=1.0, memorized=0.0) |
+
+### 2. `qa_records.jsonl`
+
+每行一个 QA question，包含检索结果、模型回答和评判：
+
+```json
+{
+  "sample_id": "vab_minecraft_...",
+  "sample_uuid": "uuid-...",
+  "checkpoint_id": "probe_e980c238",
+  "question": "What was in the agent's inventory at step 1?",
+  "gold_answer": "At step 1, the agent's inventory was empty.",
+  "gold_evidence_memory_ids": ["mp_S04_1"],
+  "gold_evidence_contents": ["The agent started with empty inventory"],
+  "question_type": "factual_recall",
+  "question_type_abbrev": "FR",
+  "difficulty": "easy",
+  "retrieval": {
+    "query": "What was in the agent's inventory at step 1?",
+    "top_k": 5,
+    "items": [
+      {
+        "rank": 0,
+        "memory_id": "memgallery:string_bundle",
+        "text": "user: OBSERVATION: Your Inventory: ...",
+        "score": 1.0,
+        "raw_backend_id": null
+      }
+    ],
+    "raw_trace": {"baseline": "FUMemory"}
+  },
+  "generated_answer": "The agent's inventory was empty at step 1.",
+  "cited_memories": ["user: OBSERVATION: Inventory: nothing"],
+  "eval": {
+    "answer_label": "Correct",
+    "answer_reasoning": "The response matches the reference answer...",
+    "answer_is_valid": true,
+    "evidence_hit_rate": 1.0,
+    "evidence_covered_count": 1,
+    "num_evidence": 1,
+    "evidence_reasoning": "The cited memory covers the gold evidence...",
+    "num_cited_memories": 1
+  }
+}
+```
+
+**eval 字段说明：**
+
+| 字段 | 含义 |
+|------|------|
+| `answer_label` | `Correct` / `Hallucination` / `Omission` |
+| `answer_is_valid` | 评判是否成功（非 LLM 错误） |
+| `evidence_hit_rate` | cited memories 覆盖了多少 gold evidence (0-1) |
+| `evidence_covered_count` | 被覆盖的 gold evidence 数量 |
+| `num_cited_memories` | 模型回答时引用的记忆条数 |
+
+### 3. `aggregate_metrics.json`
+
+baseline 级别的 6 维汇总指标：
+
+```json
+{
+  "baseline_id": "FUMemory",
+  "memory_recall": {
+    "avg_recall": 0.72,
+    "avg_update_recall": 0.65,
+    "num_sessions_with_recall": 110,
+    "num_sessions_with_update": 85,
+    "total_covered": 320,
+    "total_gold": 445
+  },
+  "memory_correctness": {
+    "avg_correctness": 0.81,
+    "avg_hallucination": 0.08,
+    "avg_irrelevant": 0.11,
+    "num_sessions": 110,
+    "total_memories": 1200,
+    "total_correct": 972,
+    "total_hallucination": 96,
+    "total_irrelevant": 132
+  },
+  "update_handling": {
+    "score": 0.65,
+    "num_updated": 52,
+    "num_both": 18,
+    "num_outdated": 15,
+    "num_total": 85
+  },
+  "interference_rejection": {
+    "score": 0.0,
+    "num_rejected": 0,
+    "num_memorized": 0,
+    "num_total": 0
+  },
+  "question_answering": {
+    "correct_ratio": 0.58,
+    "hallucination_ratio": 0.22,
+    "omission_ratio": 0.20,
+    "num_total": 990,
+    "num_valid": 990
+  },
+  "evidence_coverage": {
+    "hit_rate": 0.43,
+    "num_covered": 425,
+    "num_total": 990
+  }
+}
+```
+
+**6 个维度：**
+
+| 维度 | 聚合方式 | 核心指标 | 方向 |
+|------|---------|---------|------|
+| Memory Recall | 按 session 平均 | `avg_recall` | ↑ |
+| Memory Correctness | 按 session 平均 | `avg_correctness`, `avg_hallucination` | ↑, ↓ |
+| Update Handling | 跨 session 池化 | `score` | ↑ |
+| Interference Rejection | 跨 session 池化 | `score` | ↑ |
+| Question Answering | 跨 question 池化 | `correct_ratio`, `hallucination_ratio` | ↑, ↓ |
+| Evidence Coverage | 跨 question 池化 | `hit_rate` | ↑ |
+
+### 4. `pipeline_sessions.jsonl` / `pipeline_qa.jsonl`
+
+Stage 1 的 checkpoint 文件，结构与 `session_records.jsonl` / `qa_records.jsonl` 相同但**不含 `eval` 字段**。
+
+用途：`--eval-only` 模式跳过 pipeline 直接从 checkpoint 恢复，只重跑 eval 阶段。典型场景：
+
+```bash
+# 首次完整运行
+python -m eval_framework.cli --dataset ... --baseline FUMemory --output-dir results/FU
+
+# 换 judge 模型重评（不重跑 pipeline）
+OPENAI_MODEL=gpt-4o-mini python -m eval_framework.cli \
+    --dataset ... --baseline FUMemory --output-dir results/FU --eval-only
+```
+
+## 结果分析示例
+
+```python
+import json
+
+# 读取汇总
+with open("results/FUMemory/aggregate_metrics.json") as f:
+    agg = json.load(f)
+print(f"Recall: {agg['memory_recall']['avg_recall']:.2%}")
+print(f"QA Correct: {agg['question_answering']['correct_ratio']:.2%}")
+
+# 按 QA type 分析正确率
+qa_by_type = {}
+with open("results/FUMemory/qa_records.jsonl") as f:
+    for line in f:
+        rec = json.loads(line)
+        qt = rec["question_type_abbrev"]
+        label = rec["eval"]["answer_label"]
+        qa_by_type.setdefault(qt, []).append(label)
+
+for qt, labels in sorted(qa_by_type.items()):
+    correct = sum(1 for l in labels if l == "Correct")
+    print(f"  {qt}: {correct}/{len(labels)} = {correct/len(labels):.0%}")
+```

evaluators/__init__.py

@@ -0,0 +1,11 @@
+"""Session- and checkpoint-level evaluators using batch LLM judge."""
+
+from eval_framework.evaluators.aggregate import aggregate_metrics
+from eval_framework.evaluators.extraction import evaluate_extraction
+from eval_framework.evaluators.qa import evaluate_checkpoint_qa
+
+__all__ = [
+    "aggregate_metrics",
+    "evaluate_checkpoint_qa",
+    "evaluate_extraction",
+]

evaluators/aggregate.py

@@ -0,0 +1,175 @@
+"""Roll up per-session and per-QA evaluations into baseline-level summaries.
+
+Recall & correctness: per-session average (not pooled cumulative).
+Interference: pooled across sessions.
+QA & evidence: pooled across questions.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+
+
+def _safe_div(a: float, b: float) -> float:
+    return a / b if b else 0.0
+
+
+def aggregate_metrics(
+    baseline_id: str,
+    *,
+    session_evaluations: Sequence[Mapping[str, object]] = (),
+    qa_evaluations: Sequence[Mapping[str, object]] = (),
+) -> dict[str, object]:
+    """Aggregate all per-session and per-QA evaluations."""
+
+    # --- Per-session recall (average) ---
+    recall_scores: list[float] = []
+    update_recall_scores: list[float] = []
+
+    # --- Per-session correctness (average) ---
+    correctness_scores: list[float] = []
+    hallucination_scores: list[float] = []
+    irrelevant_scores: list[float] = []
+
+    # --- Update handling (pooled) ---
+    upd_num_updated = 0
+    upd_num_both = 0
+    upd_num_outdated = 0
+    upd_total_items = 0
+
+    # --- Interference rejection (pooled) ---
+    interf_num_rejected = 0
+    interf_num_memorized = 0
+    interf_total_items = 0
+
+    # --- Per-session detail counters (for reference) ---
+    total_gold_points = 0
+    total_covered = 0
+    total_memories = 0
+    total_correct = 0
+    total_hallucination = 0
+    total_irrelevant = 0
+
+    for s in session_evaluations:
+        # Recall: per-session score
+        r = s.get("recall")
+        if r is not None:
+            recall_scores.append(float(r))
+
+        ur = s.get("update_recall")
+        if ur is not None:
+            update_recall_scores.append(float(ur))
+
+        # Correctness: per-session score
+        cr = s.get("correctness_rate")
+        if cr is not None:
+            correctness_scores.append(float(cr))
+
+        nm = int(s.get("num_memories", 0))
+        if nm > 0:
+            hallucination_scores.append(
+                float(s.get("num_hallucination", 0)) / nm
+            )
+            irrelevant_scores.append(
+                float(s.get("num_irrelevant", 0)) / nm
+            )
+
+        # Detail counters
+        c = s.get("covered_count")
+        if c is not None:
+            total_covered += int(c)
+        total_gold_points += int(s.get("num_gold", 0))
+        total_memories += nm
+        total_correct += int(s.get("num_correct", 0))
+        total_hallucination += int(s.get("num_hallucination", 0))
+        total_irrelevant += int(s.get("num_irrelevant", 0))
+
+        # Update handling (pooled)
+        upd_num_updated += int(s.get("update_num_updated", 0))
+        upd_num_both += int(s.get("update_num_both", 0))
+        upd_num_outdated += int(s.get("update_num_outdated", 0))
+        upd_total_items += int(s.get("update_total_items", 0))
+
+        # Interference rejection (pooled)
+        interf_num_rejected += int(s.get("interference_num_rejected", 0))
+        interf_num_memorized += int(s.get("interference_num_memorized", 0))
+        interf_total_items += int(s.get("interference_total_items", 0))
+
+    # --- QA (pooled) ---
+    qa_total = 0
+    qa_valid = 0
+    qa_correct = 0
+    qa_hallucination = 0
+    qa_omission = 0
+    evidence_covered = 0
+    evidence_total = 0
+
+    for q in qa_evaluations:
+        qa_total += 1
+        label = q.get("answer_label")
+        if label in ("Correct", "Hallucination", "Omission"):
+            qa_valid += 1
+            if label == "Correct":
+                qa_correct += 1
+            elif label == "Hallucination":
+                qa_hallucination += 1
+            elif label == "Omission":
+                qa_omission += 1
+
+        ec = q.get("evidence_covered_count")
+        if ec is not None:
+            evidence_covered += int(ec)
+        evidence_total += int(q.get("num_evidence", 0))
+
+    n_recall = len(recall_scores)
+    n_update = len(update_recall_scores)
+    n_correct = len(correctness_scores)
+    n_hallu = len(hallucination_scores)
+    n_irrel = len(irrelevant_scores)
+
+    return {
+        "baseline_id": baseline_id,
+        "memory_recall": {
+            "avg_recall": _safe_div(sum(recall_scores), n_recall),
+            "avg_update_recall": _safe_div(sum(update_recall_scores), n_update),
+            "num_sessions_with_recall": n_recall,
+            "num_sessions_with_update": n_update,
+            "total_covered": total_covered,
+            "total_gold": total_gold_points,
+        },
+        "memory_correctness": {
+            "avg_correctness": _safe_div(sum(correctness_scores), n_correct),
+            "avg_hallucination": _safe_div(sum(hallucination_scores), n_hallu),
+            "avg_irrelevant": _safe_div(sum(irrelevant_scores), n_irrel),
+            "num_sessions": n_correct,
+            "total_memories": total_memories,
+            "total_correct": total_correct,
+            "total_hallucination": total_hallucination,
+            "total_irrelevant": total_irrelevant,
+        },
+        "update_handling": {
+            "score": _safe_div(upd_num_updated * 1.0 + upd_num_both * 0.5, upd_total_items),
+            "num_updated": upd_num_updated,
+            "num_both": upd_num_both,
+            "num_outdated": upd_num_outdated,
+            "num_total": upd_total_items,
+        },
+        "interference_rejection": {
+            "score": _safe_div(interf_num_rejected, interf_total_items),
+            "num_rejected": interf_num_rejected,
+            "num_memorized": interf_num_memorized,
+            "num_total": interf_total_items,
+        },
+        "question_answering": {
+            "correct_ratio": _safe_div(qa_correct, qa_valid),
+            "hallucination_ratio": _safe_div(qa_hallucination, qa_valid),
+            "omission_ratio": _safe_div(qa_omission, qa_valid),
+            "num_total": qa_total,
+            "num_valid": qa_valid,
+        },
+        "evidence_coverage": {
+            "hit_rate": _safe_div(evidence_covered, evidence_total),
+            "num_covered": evidence_covered,
+            "num_total": evidence_total,
+        },
+    }

evaluators/extraction.py

@@ -0,0 +1,193 @@
+"""Unified session evaluation: recall + correctness (includes update & interference).
+
+Per session, 2 LLM calls — both scoped to THIS SESSION's memory delta only:
+  Call 1 — Recall: how many of this session's gold points are covered by the
+           session's memory delta (add/update ops)?
+  Call 2 — Correctness: is each delta memory correct, hallucinated, or irrelevant?
+           (reference = this session's gold points + interference)
+
+Aggregate: per-session recall/correctness averaged across sessions.
+"""
+
+from __future__ import annotations
+
+from eval_framework.judges import (
+    evaluate_correctness_batch,
+    evaluate_interference_single,
+    evaluate_recall_batch,
+    evaluate_update_single,
+)
+from eval_framework.pipeline.records import PipelineSessionRecord
+
+
+def _delta_to_text(session: PipelineSessionRecord) -> str:
+    """Only the memories added or updated in THIS session (not the full snapshot)."""
+    lines: list[str] = []
+    idx = 0
+    for d in session.memory_delta:
+        if d.op in ("add", "update"):
+            idx += 1
+            lines.append(f"[{idx}] {d.text}")
+    return "\n".join(lines)
+
+
+def _delta_texts(session: PipelineSessionRecord) -> list[str]:
+    """Text list of memories added or updated in THIS session."""
+    return [d.text for d in session.memory_delta if d.op in ("add", "update")]
+
+
+def _build_recall_gold_points(session: PipelineSessionRecord) -> list[str]:
+    """Current session's new + update gold points only (NOT cumulative)."""
+    out: list[str] = []
+    for g in session.gold_state.session_new_memories:
+        out.append(f"[normal] {g.memory_content}")
+    for g in session.gold_state.session_update_memories:
+        out.append(f"[update] {g.memory_content}")
+    return out
+
+
+def _build_correctness_gold_points(session: PipelineSessionRecord) -> list[str]:
+    """Current session's new + update + interference gold points as reference."""
+    out: list[str] = []
+    for g in session.gold_state.session_new_memories:
+        out.append(f"[normal] {g.memory_content}")
+    for g in session.gold_state.session_update_memories:
+        out.append(f"[update] {g.memory_content}")
+    for g in session.gold_state.session_interference_memories:
+        out.append(f"[interference] {g.memory_content}")
+    return out
+
+
+def evaluate_extraction(
+    session: PipelineSessionRecord,
+    **_kwargs: object,
+) -> dict[str, object]:
+    """Unified session evaluation: recall + correctness in 2 LLM calls.
+
+    Uses only THIS session's new gold points for recall and correctness,
+    not the cumulative history. Aggregate averages per-session scores.
+    """
+
+    delta_str = _delta_to_text(session)
+    delta_texts = _delta_texts(session)
+    interference_total = len(session.gold_state.session_interference_memories)
+
+    # --- Call 1: Recall (this session's gold points vs this session's delta) ---
+    recall_gold = _build_recall_gold_points(session)
+
+    if not recall_gold:
+        recall = None
+        update_recall = None
+        recall_result: dict[str, object] = {
+            "covered_count": 0, "update_covered_count": 0,
+            "total": 0, "update_total": 0,
+            "reasoning": "No new gold points in this session.",
+        }
+    elif not delta_str.strip():
+        recall = 0.0
+        update_recall = 0.0
+        update_total = sum(1 for p in recall_gold if p.startswith("[update]"))
+        recall_result = {
+            "covered_count": 0, "update_covered_count": 0,
+            "total": len(recall_gold), "update_total": update_total,
+            "reasoning": "No add/update memories in this session's delta.",
+        }
+    else:
+        recall_result = evaluate_recall_batch(delta_str, recall_gold)
+
+    covered = recall_result.get("covered_count")
+    upd_covered = recall_result.get("update_covered_count")
+    total_gold = recall_result.get("total", len(recall_gold))
+    upd_total = recall_result.get("update_total", 0)
+
+    if recall_gold:
+        recall = float(covered) / float(total_gold) if covered is not None and total_gold else None
+        update_recall = float(upd_covered) / float(upd_total) if upd_covered is not None and upd_total else None
+
+    # --- Call 2: Correctness (this session's delta memories, reference = this session's golds) ---
+    correctness_gold = _build_correctness_gold_points(session)
+    correctness_result = evaluate_correctness_batch(delta_texts, correctness_gold, interference_total)
+    correctness_records = correctness_result.get("results", [])
+
+    num_correct = sum(1 for r in correctness_records if r.get("label") == "correct")
+    num_hallucination = sum(1 for r in correctness_records if r.get("label") == "hallucination")
+    num_irrelevant = sum(1 for r in correctness_records if r.get("label") == "irrelevant")
+    num_memories = len(delta_texts)
+    correctness_rate = float(num_correct) / float(num_memories) if num_memories else 0.0
+
+    # --- Call 3+: Update handling (one LLM call per update gold point) ---
+    update_records: list[dict[str, object]] = []
+    for g in session.gold_state.session_update_memories:
+        res = evaluate_update_single(
+            delta_str,
+            new_content=g.memory_content,
+            old_contents=list(g.original_memories),
+        )
+        update_records.append({
+            "memory_id": g.memory_id,
+            "label": res["label"],
+            "reasoning": res["reasoning"],
+        })
+
+    num_updated = sum(1 for r in update_records if r["label"] == "updated")
+    num_both = sum(1 for r in update_records if r["label"] == "both")
+    num_outdated = sum(1 for r in update_records if r["label"] == "outdated")
+    update_total_items = len(update_records)
+    # Score: updated=1.0, both=0.5, outdated=0.0
+    update_score = (
+        (num_updated * 1.0 + num_both * 0.5) / update_total_items
+        if update_total_items else None
+    )
+
+    # --- Call 4+: Interference rejection (one LLM call per interference gold point) ---
+    interference_records: list[dict[str, object]] = []
+    for g in session.gold_state.session_interference_memories:
+        res = evaluate_interference_single(
+            delta_str,
+            interference_content=g.memory_content,
+        )
+        interference_records.append({
+            "memory_id": g.memory_id,
+            "label": res["label"],
+            "reasoning": res["reasoning"],
+        })
+
+    num_rejected = sum(1 for r in interference_records if r["label"] == "rejected")
+    num_memorized = sum(1 for r in interference_records if r["label"] == "memorized")
+    interference_total_items = len(interference_records)
+    # Score: rejected=1.0, memorized=0.0
+    interference_score = (
+        float(num_rejected) / interference_total_items
+        if interference_total_items else None
+    )
+
+    return {
+        "session_id": session.session_id,
+        "recall": recall,
+        "covered_count": covered,
+        "num_gold": total_gold,
+        "update_recall": update_recall,
+        "update_covered_count": upd_covered,
+        "update_total": upd_total,
+        "recall_reasoning": recall_result.get("reasoning", ""),
+        "correctness_rate": correctness_rate,
+        "num_memories": num_memories,
+        "num_correct": num_correct,
+        "num_hallucination": num_hallucination,
+        "num_irrelevant": num_irrelevant,
+        "correctness_reasoning": correctness_result.get("reasoning", ""),
+        "correctness_records": correctness_records,
+        # Update handling
+        "update_score": update_score,
+        "update_num_updated": num_updated,
+        "update_num_both": num_both,
+        "update_num_outdated": num_outdated,
+        "update_total_items": update_total_items,
+        "update_records": update_records,
+        # Interference rejection
+        "interference_score": interference_score,
+        "interference_num_rejected": num_rejected,
+        "interference_num_memorized": num_memorized,
+        "interference_total_items": interference_total_items,
+        "interference_records": interference_records,
+    }

evaluators/qa.py

@@ -0,0 +1,70 @@
+"""Checkpoint QA evaluation: answer quality + batch evidence coverage.
+
+Two dimensions:
+1. Answer evaluation: Correct / Hallucination / Omission (1 LLM call)
+2. Evidence coverage: how many gold evidence points are covered by the
+   memories the model actually *cited* when answering? (1 LLM call)
+"""
+
+from __future__ import annotations
+
+from eval_framework.judges import evaluate_evidence_batch, evaluate_qa_llm
+from eval_framework.pipeline.records import PipelineCheckpointQARecord
+
+
+def evaluate_checkpoint_qa(
+    record: PipelineCheckpointQARecord,
+    **_kwargs: object,
+) -> dict[str, object]:
+    """LLM-judged QA evaluation: answer correctness + evidence coverage."""
+
+    # --- Build cited-memories text (what the model actually used) ---
+    if record.cited_memories:
+        cited_lines = [f"[{i + 1}] {m}" for i, m in enumerate(record.cited_memories)]
+        cited_str = "\n".join(cited_lines)
+    else:
+        # Fallback: use full retrieval (legacy records without cited_memories)
+        cited_lines = [f"[{item.rank}] {item.text}" for item in record.retrieval.items]
+        cited_str = "\n".join(cited_lines) if cited_lines else ""
+
+    # --- Answer evaluation (1 LLM call, unchanged) ---
+    gold_evidence_str = (
+        "\n".join(record.gold_evidence_contents)
+        if record.gold_evidence_contents
+        else "No evidence available."
+    )
+    answer_result = evaluate_qa_llm(
+        question=record.question,
+        reference_answer=record.gold_answer,
+        key_memory_points=gold_evidence_str,
+        system_response=record.generated_answer,
+    )
+    answer_label = answer_result.get("evaluation_result")
+
+    # --- Evidence coverage (1 LLM call, batch) ---
+    # Only check against cited memories, not the full retrieval
+    gold_contents = list(record.gold_evidence_contents)
+    evidence_result: dict[str, object] = {
+        "covered_count": 0, "total": len(gold_contents), "reasoning": ""
+    }
+
+    if gold_contents and cited_str.strip():
+        evidence_result = evaluate_evidence_batch(cited_str, gold_contents)
+
+    covered = evidence_result.get("covered_count")
+    total_ev = evidence_result.get("total", len(gold_contents))
+    if covered is not None and total_ev:
+        evidence_hit_rate = float(covered) / float(total_ev)
+    else:
+        evidence_hit_rate = 0.0
+
+    return {
+        "answer_label": answer_label,
+        "answer_reasoning": answer_result.get("reasoning", ""),
+        "answer_is_valid": answer_label in ("Correct", "Hallucination", "Omission"),
+        "evidence_hit_rate": evidence_hit_rate,
+        "evidence_covered_count": covered,
+        "num_evidence": total_ev,
+        "evidence_reasoning": evidence_result.get("reasoning", ""),
+        "num_cited_memories": len(record.cited_memories),
+    }

judges/__init__.py

@@ -0,0 +1,215 @@
+"""Judge stack: batch LLM evaluation.
+
+Session: 2 calls (recall + correctness) + per-item calls for update/interference.
+QA: 2 calls (answer + evidence).
+"""
+
+from __future__ import annotations
+
+from eval_framework.judges.llm_client import llm_request_for_json
+from eval_framework.judges.prompts import (
+    CORRECTNESS_BATCH_PROMPT,
+    EVIDENCE_BATCH_PROMPT,
+    INTERFERENCE_EVAL_PROMPT,
+    QA_EVALUATION_PROMPT,
+    RECALL_BATCH_PROMPT,
+    UPDATE_EVAL_PROMPT,
+)
+
+__all__ = [
+    "evaluate_recall_batch",
+    "evaluate_correctness_batch",
+    "evaluate_update_single",
+    "evaluate_interference_single",
+    "evaluate_evidence_batch",
+    "evaluate_qa_llm",
+    "llm_request_for_json",
+]
+
+
+def evaluate_recall_batch(
+    extracted_memories_str: str,
+    gold_points_tagged: list[str],
+) -> dict[str, object]:
+    """One LLM call: how many gold points are covered? Distinguishes update sub-score.
+
+    gold_points_tagged: list of "[normal] content" or "[update] content" strings.
+    Returns {covered_count, update_covered_count, total, update_total, reasoning}.
+    """
+    if not extracted_memories_str.strip():
+        update_total = sum(1 for p in gold_points_tagged if p.startswith("[update]"))
+        return {
+            "covered_count": 0, "update_covered_count": 0,
+            "total": len(gold_points_tagged), "update_total": update_total,
+            "reasoning": "No extracted memories.",
+        }
+    if not gold_points_tagged:
+        return {
+            "covered_count": 0, "update_covered_count": 0,
+            "total": 0, "update_total": 0, "reasoning": "No gold points.",
+        }
+
+    numbered = "\n".join(f"[{i+1}] {p}" for i, p in enumerate(gold_points_tagged))
+    update_total = sum(1 for p in gold_points_tagged if p.startswith("[update]"))
+
+    prompt = RECALL_BATCH_PROMPT.format(memories=extracted_memories_str, gold_points=numbered)
+    try:
+        result = llm_request_for_json(prompt)
+        covered = int(result.get("covered_count", 0))
+        upd_covered = int(result.get("update_covered_count", 0))
+        return {
+            "covered_count": min(covered, len(gold_points_tagged)),
+            "update_covered_count": min(upd_covered, update_total),
+            "total": len(gold_points_tagged),
+            "update_total": update_total,
+            "reasoning": result.get("reasoning", ""),
+        }
+    except Exception as e:
+        return {
+            "covered_count": None, "update_covered_count": None,
+            "total": len(gold_points_tagged), "update_total": update_total,
+            "reasoning": f"LLM error: {e}",
+        }
+
+
+def evaluate_correctness_batch(
+    snapshot_memories: list[str],
+    gold_points_tagged: list[str],
+    interference_total: int,
+) -> dict[str, object]:
+    """One LLM call: is each snapshot memory correct? Includes interference detection.
+
+    gold_points_tagged: list of "[normal] content", "[update] content", "[interference] content".
+    Returns {results: [{id, label}], interference_memorized_count, interference_total, reasoning}.
+    """
+    if not snapshot_memories:
+        return {
+            "results": [],
+            "interference_memorized_count": 0,
+            "interference_total": interference_total,
+            "reasoning": "No snapshot memories.",
+        }
+
+    numbered_memories = "\n".join(f"[{i+1}] {m}" for i, m in enumerate(snapshot_memories))
+    numbered_golds = "\n".join(f"- {p}" for p in gold_points_tagged) if gold_points_tagged else "(no ground-truth)"
+
+    prompt = CORRECTNESS_BATCH_PROMPT.format(memories=numbered_memories, gold_points=numbered_golds)
+    try:
+        result = llm_request_for_json(prompt)
+        raw_results = result.get("results", [])
+        valid_labels = {"correct", "hallucination", "irrelevant"}
+        cleaned = []
+        for r in raw_results:
+            label = str(r.get("label", "irrelevant")).lower().strip()
+            if label not in valid_labels:
+                label = "irrelevant"
+            cleaned.append({"id": r.get("id"), "label": label})
+        interf_mem = int(result.get("interference_memorized_count", 0))
+        return {
+            "results": cleaned,
+            "interference_memorized_count": min(interf_mem, interference_total),
+            "interference_total": interference_total,
+            "reasoning": result.get("reasoning", ""),
+        }
+    except Exception as e:
+        return {
+            "results": [],
+            "interference_memorized_count": None,
+            "interference_total": interference_total,
+            "reasoning": f"LLM error: {e}",
+        }
+
+
+def evaluate_update_single(
+    delta_memories_str: str,
+    new_content: str,
+    old_contents: list[str],
+) -> dict[str, object]:
+    """One LLM call: how did the system handle a single memory update?
+
+    Returns {label: "updated"|"both"|"outdated", reasoning}.
+    """
+    old_str = "\n".join(f"- {o}" for o in old_contents) if old_contents else "(none)"
+    prompt = UPDATE_EVAL_PROMPT.format(
+        memories=delta_memories_str,
+        new_content=new_content,
+        old_contents=old_str,
+    )
+    try:
+        result = llm_request_for_json(prompt)
+        label = str(result.get("label", "outdated")).lower().strip()
+        if label not in ("updated", "both", "outdated"):
+            label = "outdated"
+        return {"label": label, "reasoning": result.get("reasoning", "")}
+    except Exception as e:
+        return {"label": None, "reasoning": f"LLM error: {e}"}
+
+
+def evaluate_interference_single(
+    delta_memories_str: str,
+    interference_content: str,
+) -> dict[str, object]:
+    """One LLM call: did the system incorrectly memorize an interference point?
+
+    Returns {label: "rejected"|"memorized", reasoning}.
+    """
+    prompt = INTERFERENCE_EVAL_PROMPT.format(
+        memories=delta_memories_str,
+        interference_content=interference_content,
+    )
+    try:
+        result = llm_request_for_json(prompt)
+        label = str(result.get("label", "memorized")).lower().strip()
+        if label not in ("rejected", "memorized"):
+            label = "memorized"
+        return {"label": label, "reasoning": result.get("reasoning", "")}
+    except Exception as e:
+        return {"label": None, "reasoning": f"LLM error: {e}"}
+
+
+def evaluate_evidence_batch(
+    retrieved_memories_str: str,
+    evidence_points: list[str],
+) -> dict[str, object]:
+    """One LLM call: how many gold evidence points are covered by retrieval?"""
+    if not retrieved_memories_str.strip():
+        return {"covered_count": 0, "total": len(evidence_points), "reasoning": "No retrieved memories."}
+    if not evidence_points:
+        return {"covered_count": 0, "total": 0, "reasoning": "No evidence points."}
+
+    numbered = "\n".join(f"[{i+1}] {p}" for i, p in enumerate(evidence_points))
+    prompt = EVIDENCE_BATCH_PROMPT.format(retrieved_memories=retrieved_memories_str, gold_evidence_points=numbered)
+    try:
+        result = llm_request_for_json(prompt)
+        covered = int(result.get("covered_count", 0))
+        return {
+            "covered_count": min(covered, len(evidence_points)),
+            "total": len(evidence_points),
+            "reasoning": result.get("reasoning", ""),
+        }
+    except Exception as e:
+        return {"covered_count": None, "total": len(evidence_points), "reasoning": f"LLM error: {e}"}
+
+
+def evaluate_qa_llm(
+    question: str,
+    reference_answer: str,
+    key_memory_points: str,
+    system_response: str,
+) -> dict[str, object]:
+    """LLM judge: classify the QA response as Correct/Hallucination/Omission."""
+    if not system_response.strip():
+        return {"evaluation_result": "Omission", "reasoning": "Empty system response."}
+
+    prompt = QA_EVALUATION_PROMPT.format(
+        question=question, reference_answer=reference_answer,
+        key_memory_points=key_memory_points, response=system_response,
+    )
+    try:
+        result = llm_request_for_json(prompt)
+        label = result.get("evaluation_result", "Omission")
+        if label not in ("Correct", "Hallucination", "Omission"):
+            label = "Omission"
+        return {"evaluation_result": label, "reasoning": result.get("reasoning", "")}
+    except Exception as e:
+        return {"evaluation_result": None, "reasoning": f"LLM judge error: {e}"}

judges/llm_client.py

@@ -0,0 +1,156 @@
+"""OpenAI LLM client for judge calls with retry logic and concurrency control."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import logging
+import threading
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+from tenacity import retry, stop_after_attempt, wait_random_exponential, before_sleep_log
+
+# Load .env from project root (walk up from this file to find it)
+_PROJECT_ROOT = Path(__file__).resolve().parents[2]
+load_dotenv(_PROJECT_ROOT / ".env")
+
+logger = logging.getLogger(__name__)
+
+_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*\n?(.*?)\n?\s*```", re.DOTALL)
+_JSON_FENCE_OPEN_RE = re.compile(r"```(?:json)?\s*\n?(.*)", re.DOTALL)
+
+_client: Any = None
+_client_lock = threading.Lock()
+_semaphore: threading.Semaphore | None = None
+
+
+def _build_client() -> Any:
+    from openai import OpenAI
+    return OpenAI(
+        api_key=os.getenv("OPENAI_API_KEY", ""),
+        base_url=os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1"),
+    )
+
+
+def _get_client() -> Any:
+    global _client
+    if _client is None:
+        with _client_lock:
+            if _client is None:
+                _client = _build_client()
+    return _client
+
+
+def _get_semaphore() -> threading.Semaphore:
+    global _semaphore
+    if _semaphore is None:
+        max_concurrent = int(os.getenv("LLM_MAX_CONCURRENT", "5"))
+        _semaphore = threading.Semaphore(max_concurrent)
+    return _semaphore
+
+
+def _common_params() -> dict[str, Any]:
+    params: dict[str, Any] = {}
+    model = os.getenv("OPENAI_MODEL") or ""
+    max_tok = os.getenv("OPENAI_MAX_TOKENS")
+    if max_tok:
+        if model.startswith("gpt-5") or model.startswith("o"):
+            params["max_completion_tokens"] = int(max_tok)
+        else:
+            params["max_tokens"] = int(max_tok)
+    params["temperature"] = float(os.getenv("JUDGE_TEMPERATURE", "0.0"))
+    if os.getenv("OPENAI_TIMEOUT"):
+        params["timeout"] = int(os.getenv("OPENAI_TIMEOUT"))
+    return params
+
+
+@retry(
+    wait=wait_random_exponential(min=2, max=60),
+    stop=stop_after_attempt(8),
+    reraise=True,
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+)
+def llm_request_for_json(prompt: str) -> dict[str, Any]:
+    """Send a prompt to the LLM and parse the JSON block from the response.
+
+    Respects global concurrency limit (LLM_MAX_CONCURRENT env var, default 5).
+    """
+    sem = _get_semaphore()
+    sem.acquire()
+    try:
+        client = _get_client()
+        model = os.getenv("OPENAI_MODEL") or "gpt-4o"
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            **_common_params(),
+        )
+        content = response.choices[0].message.content or ""
+    finally:
+        sem.release()
+
+    parsed = _extract_json(content)
+    if parsed is not None:
+        return parsed
+
+    raise ValueError(f"No JSON block found in model output: {content[:500]}")
+
+
+def _extract_json(content: str) -> dict[str, Any] | None:
+    """Try to extract a JSON object from model output, with truncation repair."""
+    # 1. Closed fence: ```json ... ```
+    for match in _JSON_FENCE_RE.finditer(content):
+        candidate = match.group(1).strip()
+        if candidate.startswith("{"):
+            try:
+                return json.loads(candidate)
+            except json.JSONDecodeError:
+                pass
+
+    # 2. Open fence (output truncated before closing ```): ```json ...EOF
+    match = _JSON_FENCE_OPEN_RE.search(content)
+    if match:
+        candidate = match.group(1).strip()
+        if candidate.startswith("{"):
+            repaired = _repair_truncated_json(candidate)
+            if repaired is not None:
+                return repaired
+
+    # 3. Raw JSON without fences
+    stripped = content.strip()
+    if stripped.startswith("{"):
+        try:
+            return json.loads(stripped)
+        except json.JSONDecodeError:
+            repaired = _repair_truncated_json(stripped)
+            if repaired is not None:
+                return repaired
+
+    return None
+
+
+def _repair_truncated_json(text: str) -> dict[str, Any] | None:
+    """Best-effort repair of truncated JSON by closing open brackets/braces."""
+    # Remove trailing partial tokens (incomplete key/value after last comma)
+    text = re.sub(r',\s*"[^"]*$', "", text)         # trailing partial key
+    text = re.sub(r',\s*\{[^}]*$', "", text)         # trailing partial object
+    text = re.sub(r',\s*$', "", text)                 # trailing comma
+
+    # Count unclosed brackets and braces, then append closers
+    open_braces = text.count("{") - text.count("}")
+    open_brackets = text.count("[") - text.count("]")
+    suffix = "]" * max(open_brackets, 0) + "}" * max(open_braces, 0)
+    candidate = text + suffix
+
+    try:
+        result = json.loads(candidate)
+        if isinstance(result, dict):
+            logger.warning("Repaired truncated JSON (appended %r)", suffix)
+            return result
+    except json.JSONDecodeError:
+        pass
+    return None

judges/prompts.py

@@ -0,0 +1,223 @@
+"""LLM judge prompt templates — batch mode.
+
+Each session: 2 LLM calls (recall + correctness).
+Each QA question: 2 LLM calls (answer + evidence).
+"""
+
+# ---------------------------------------------------------------------------
+# Session Call 1: Recall (Gold -> Snapshot)
+# ---------------------------------------------------------------------------
+RECALL_BATCH_PROMPT = """You are a **Memory Recall Evaluator**.
+Determine how many of the **Expected Memory Points** are covered by the system's **Extracted Memories**.
+
+# Inputs
+
+1. **Extracted Memories** (what the system actually stored):
+{memories}
+
+2. **Expected Memory Points** (numbered, each tagged [normal] or [update]):
+{gold_points}
+
+# Instructions
+
+- Go through each Expected Memory Point and check whether the Extracted Memories contain information that covers it.
+- Semantic matching is acceptable; exact wording is NOT required.
+- Count **total** covered points AND separately count how many **[update]** tagged points are covered.
+
+# Output
+
+```json
+{{
+  "covered_count": <int>,
+  "update_covered_count": <int>,
+  "total": <int>,
+  "update_total": <int>,
+  "reasoning": "Brief summary"
+}}
+```
+"""
+
+# ---------------------------------------------------------------------------
+# Session Call 2: Correctness (Snapshot -> Gold)
+# ---------------------------------------------------------------------------
+CORRECTNESS_BATCH_PROMPT = """You are a **Memory Correctness Evaluator**.
+Evaluate whether each memory stored by the system is factually correct.
+
+# Inputs
+
+1. **System Memories** (numbered, what the system actually stored):
+{memories}
+
+2. **Ground-Truth Reference Points** (tagged [normal], [update], or [interference]):
+{gold_points}
+
+# Instructions
+
+For **each** System Memory, classify it as one of:
+- **correct**: The memory is factually accurate and consistent with the [normal] or [update] ground-truth points.
+- **hallucination**: The memory contains fabricated or incorrect information, OR it contains content from [interference] points (information that should NOT have been memorized).
+- **irrelevant**: The memory is not wrong per se, but is trivial filler or not related to any ground-truth point.
+
+**IMPORTANT**: If a System Memory matches or contains information from an [interference] tagged point, it MUST be classified as **hallucination**, because the system should have ignored that information.
+
+Also count how many [interference] ground-truth points appear in the System Memories.
+
+# Output
+
+```json
+{{
+  "results": [
+    {{"id": 1, "label": "correct|hallucination|irrelevant"}},
+    {{"id": 2, "label": "correct|hallucination|irrelevant"}}
+  ],
+  "interference_memorized_count": <int>,
+  "interference_total": <int>,
+  "reasoning": "Brief justification"
+}}
+```
+"""
+
+# ---------------------------------------------------------------------------
+# Session: Update handling (per update gold point)
+# ---------------------------------------------------------------------------
+UPDATE_EVAL_PROMPT = """You are a **Memory Update Evaluator**.
+Determine how a memory system handled an information update.
+
+# Inputs
+
+1. **System Memories** (what the system currently stores after this session):
+{memories}
+
+2. **Updated Fact** (the NEW correct information):
+{new_content}
+
+3. **Outdated Fact(s)** (the OLD information that should have been replaced):
+{old_contents}
+
+# Instructions
+
+Check the System Memories and classify the update handling as one of:
+
+- **updated**: The system stores ONLY the new/updated information. The outdated fact is no longer present. This is the ideal outcome.
+- **both**: The system stores BOTH the new and the old information. The update was partially handled — the new fact was added but the old was not removed.
+- **outdated**: The system stores ONLY the old/outdated information. The update was missed entirely — the new fact is absent.
+
+Use semantic matching, not exact wording.
+
+# Output
+
+```json
+{{
+  "label": "updated|both|outdated",
+  "reasoning": "Brief justification"
+}}
+```
+"""
+
+# ---------------------------------------------------------------------------
+# Session: Interference rejection (per interference gold point)
+# ---------------------------------------------------------------------------
+INTERFERENCE_EVAL_PROMPT = """You are a **Memory Interference Evaluator**.
+Determine whether a memory system incorrectly stored information that should have been ignored.
+
+# Inputs
+
+1. **System Memories** (what the system currently stores after this session):
+{memories}
+
+2. **Interference Content** (information that should NOT have been memorized):
+{interference_content}
+
+# Instructions
+
+Check whether the System Memories contain the interference content (or its semantic equivalent). Classify as:
+
+- **rejected**: The interference content is NOT present in the system memories. The system correctly ignored it.
+- **memorized**: The interference content IS present (or semantically equivalent) in the system memories. The system incorrectly stored it.
+
+Use semantic matching, not exact wording.
+
+# Output
+
+```json
+{{
+  "label": "rejected|memorized",
+  "reasoning": "Brief justification"
+}}
+```
+"""
+
+# ---------------------------------------------------------------------------
+# QA Evidence Coverage (batch per question)
+# ---------------------------------------------------------------------------
+EVIDENCE_BATCH_PROMPT = """You are an **Evidence Retrieval Evaluator**.
+Determine how many of the **Gold Evidence Points** are covered by the system's **Retrieved Memories** when answering a question.
+
+# Inputs
+
+1. **Retrieved Memories** (what the system retrieved to answer the question):
+{retrieved_memories}
+
+2. **Gold Evidence Points** (key facts needed for the correct answer, numbered):
+{gold_evidence_points}
+
+# Instructions
+
+Go through each Gold Evidence Point and check whether the Retrieved Memories contain information that covers it. Semantic matching is acceptable; exact wording is NOT required.
+
+Count how many Gold Evidence Points are **fully covered or logically implied** by the Retrieved Memories.
+
+# Output
+
+```json
+{{
+  "covered_count": <int>,
+  "total": <int>,
+  "reasoning": "Brief summary"
+}}
+```
+"""
+
+# ---------------------------------------------------------------------------
+# QA Answer evaluation
+# ---------------------------------------------------------------------------
+QA_EVALUATION_PROMPT = """You are an **evaluation expert for AI memory system question answering**.
+Based **only** on the provided **"Question"**, **"Reference Answer"**, and **"Key Memory Points"**, strictly evaluate the **accuracy** of the **"Memory System Response."** Classify it as one of **"Correct"**, **"Hallucination"**, or **"Omission."** Do **not** use any external knowledge or subjective inference.
+
+# Evaluation Criteria
+
+### 1. Correct
+* The response accurately answers the question and is **semantically equivalent** to the Reference Answer.
+* No contradictions with Key Memory Points or Reference Answer.
+* Synonyms, paraphrasing, and reasonable summarization are acceptable.
+
+### 2. Hallucination
+* The response includes information that **contradicts** the Reference Answer or Key Memory Points.
+* When the Reference Answer is *unknown/uncertain*, yet the response provides a specific fact.
+
+### 3. Omission
+* The response is **incomplete** compared to the Reference Answer.
+* It states "don't know" or "no related memory" even though relevant information exists.
+* For multi-element questions, missing **any** element counts as Omission.
+
+## Priority Rules
+* Both missing info AND fabricated info -> **Hallucination**.
+* No fabrication but missing info -> **Omission**.
+* Fully equivalent -> **Correct**.
+
+# Information
+
+* **Question:** {question}
+* **Reference Answer:** {reference_answer}
+* **Key Memory Points:** {key_memory_points}
+* **Memory System Response:** {response}
+
+# Output
+
+```json
+{{
+  "reasoning": "Concise evaluation rationale",
+  "evaluation_result": "Correct | Hallucination | Omission"
+}}
+```
+"""

memory_adapters/__init__.py

@@ -0,0 +1,27 @@
+"""Memory system adapters for the eval framework."""
+
+from eval_framework.memory_adapters.base import MemoryAdapter
+from eval_framework.memory_adapters.memgallery_native import (
+    MemGalleryNativeAdapter,
+    instantiate_memgallery_memory,
+)
+from eval_framework.memory_adapters.registry import (
+    EXTERNAL_ADAPTER_KEYS,
+    EXTERNAL_ADAPTER_REGISTRY,
+    MEMGALLERY_NATIVE_BASELINES,
+    MEMGALLERY_NATIVE_REGISTRY,
+    create_external_adapter,
+    create_memgallery_adapter,
+)
+
+__all__ = [
+    "EXTERNAL_ADAPTER_KEYS",
+    "EXTERNAL_ADAPTER_REGISTRY",
+    "MEMGALLERY_NATIVE_BASELINES",
+    "MEMGALLERY_NATIVE_REGISTRY",
+    "MemoryAdapter",
+    "MemGalleryNativeAdapter",
+    "create_external_adapter",
+    "create_memgallery_adapter",
+    "instantiate_memgallery_memory",
+]

memory_adapters/amem.py

@@ -0,0 +1,258 @@
+"""Adapter for the external A-Mem baseline."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import sys
+from pathlib import Path
+from typing import Any, Callable
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+
+_BACKEND_ID = "A-Mem"
+
+INTEGRATION_ERROR = (
+    f"{_BACKEND_ID} backend unavailable."
+)
+
+
+class AMemAdapter(MemoryAdapter):
+    """Thin wrapper around A-Mem's robust memory system."""
+
+    def __init__(
+        self,
+        *,
+        backend: Any | None = None,
+        backend_factory: Callable[[], Any] | None = None,
+        source_root: str | os.PathLike[str] | None = None,
+        model_name: str = "all-MiniLM-L6-v2",
+        llm_backend: str = "openai",
+        llm_model: str | None = None,
+        api_key: str | None = None,
+        api_base: str | None = None,
+        sglang_host: str = "http://localhost",
+        sglang_port: int = 30000,
+    ) -> None:
+        self._source_root = Path(source_root).resolve() if source_root else self._default_source_root()
+        resolved_llm_model = llm_model or os.getenv("OPENAI_MODEL") or "gpt-5.1"
+        self._backend: Any | None = None
+        self._backend_factory = backend_factory
+        self._integration_error: str | None = None
+        self._session_id = ""
+        self._prev_snapshot_ids: set[str] = set()
+        self._note_session_map: dict[str, str] = {}
+
+        if backend is not None:
+            self._backend = backend
+        else:
+            try:
+                if self._backend_factory is None:
+                    self._backend_factory = self._build_backend_factory(
+                        model_name=model_name,
+                        llm_backend=llm_backend,
+                    llm_model=resolved_llm_model,
+                        api_key=api_key,
+                        api_base=api_base,
+                        sglang_host=sglang_host,
+                        sglang_port=sglang_port,
+                    )
+                self._backend = self._backend_factory()
+            except Exception as exc:
+                self._integration_error = str(exc)
+
+    @staticmethod
+    def _default_source_root() -> Path:
+        here = Path(__file__).resolve()
+        # memory_adapters/ -> eval_framework/ -> our/ -> Benchmark/
+        return (here.parents[2].parent / "data_pipline" / "A-mem").resolve()
+
+    def _build_backend_factory(
+        self,
+        *,
+        model_name: str,
+        llm_backend: str,
+        llm_model: str,
+        api_key: str | None,
+        api_base: str | None,
+        sglang_host: str,
+        sglang_port: int,
+    ) -> Callable[[], Any]:
+        if not self._source_root.is_dir():
+            raise RuntimeError(
+                f"{_BACKEND_ID}: source root not found at {self._source_root}"
+            )
+        src = str(self._source_root)
+        if src not in sys.path:
+            sys.path.insert(0, src)
+        mod = importlib.import_module("memory_layer_robust")
+        backend_cls = getattr(mod, "RobustAgenticMemorySystem")
+        return lambda: backend_cls(
+            model_name=model_name,
+            llm_backend=llm_backend,
+            llm_model=llm_model,
+            api_key=api_key or os.getenv("OPENAI_API_KEY"),
+            api_base=api_base or os.getenv("OPENAI_BASE_URL"),
+            sglang_host=sglang_host,
+            sglang_port=sglang_port,
+        )
+
+    def _runtime_error(self) -> RuntimeError:
+        detail = self._integration_error or INTEGRATION_ERROR
+        return RuntimeError(
+            f"{_BACKEND_ID}: backend unavailable — {detail}"
+        )
+
+    def reset(self) -> None:
+        if self._backend_factory is None and self._backend is None:
+            raise self._runtime_error()
+        if self._backend_factory is not None:
+            self._backend = self._backend_factory()
+        self._prev_snapshot_ids = set()
+        self._note_session_map = {}
+        self._session_id = ""
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        backend = self._require_backend()
+        self._session_id = turn.session_id
+        text = self._turn_text(turn)
+        note_id = backend.add_note(text, time=turn.timestamp)
+        self._note_session_map[str(note_id)] = turn.session_id
+
+    def end_session(self, session_id: str) -> None:
+        self._require_backend()
+        self._session_id = session_id
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        backend = self._require_backend()
+        rows: list[MemorySnapshotRecord] = []
+        for note_id, note in getattr(backend, "memories", {}).items():
+            sid = self._note_session_map.get(str(note_id), self._session_id)
+            content = str(getattr(note, "content", ""))
+            context = getattr(note, "context", "")
+            keywords = list(getattr(note, "keywords", []) or [])
+            tags = list(getattr(note, "tags", []) or [])
+            # Include A-Mem enrichments in the snapshot text so that the
+            # eval captures what the system actually processed, not just
+            # the raw input.
+            enriched_parts = [content]
+            if context:
+                enriched_parts.append(f"[context] {context}")
+            if keywords:
+                enriched_parts.append(f"[keywords] {', '.join(keywords)}")
+            if tags:
+                enriched_parts.append(f"[tags] {', '.join(tags)}")
+            rows.append(
+                MemorySnapshotRecord(
+                    memory_id=str(getattr(note, "id", note_id)),
+                    text="\n".join(enriched_parts),
+                    session_id=sid,
+                    status="active",
+                    source=_BACKEND_ID,
+                    raw_backend_id=str(getattr(note, "id", note_id)),
+                    raw_backend_type="a_mem_note",
+                    metadata={
+                        "timestamp": getattr(note, "timestamp", None),
+                        "context": context,
+                        "keywords": keywords,
+                        "tags": tags,
+                        "links": list(getattr(note, "links", []) or []),
+                    },
+                )
+            )
+        return rows
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        """Export delta by diffing current snapshot against previous snapshot."""
+        self._require_backend()
+        current_snapshot = self.snapshot_memories()
+        deltas: list[MemoryDeltaRecord] = []
+        current_ids: set[str] = set()
+
+        for snap in current_snapshot:
+            current_ids.add(snap.memory_id)
+            if snap.memory_id not in self._prev_snapshot_ids:
+                deltas.append(
+                    MemoryDeltaRecord(
+                        session_id=session_id,
+                        op="add",
+                        text=snap.text,
+                        linked_previous=(),
+                        raw_backend_id=snap.raw_backend_id,
+                        metadata={
+                            "baseline": _BACKEND_ID,
+                            "backend_type": snap.raw_backend_type,
+                        },
+                    )
+                )
+
+        self._prev_snapshot_ids = current_ids
+        return deltas
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        backend = self._require_backend()
+        items: list[RetrievalItem] = []
+        memories = list(getattr(backend, "memories", {}).values())
+        retriever = getattr(backend, "retriever", None)
+        if retriever is not None and hasattr(retriever, "search"):
+            for rank, idx in enumerate(retriever.search(query, top_k)):
+                if 0 <= int(idx) < len(memories):
+                    note = memories[int(idx)]
+                    items.append(
+                        RetrievalItem(
+                            rank=rank,
+                            memory_id=str(getattr(note, "id", idx)),
+                            text=str(getattr(note, "content", "")),
+                            score=1.0 / float(rank + 1),
+                            raw_backend_id=str(getattr(note, "id", idx)),
+                        )
+                    )
+        if not items and hasattr(backend, "find_related_memories_raw"):
+            raw = backend.find_related_memories_raw(query, k=top_k)
+            if raw:
+                items.append(
+                    RetrievalItem(
+                        rank=0,
+                        memory_id="a_mem:bundle",
+                        text=str(raw),
+                        score=1.0,
+                        raw_backend_id=None,
+                    )
+                )
+        return RetrievalRecord(
+            query=query,
+            top_k=top_k,
+            items=items[:top_k],
+            raw_trace={"baseline": _BACKEND_ID},
+        )
+
+    def get_capabilities(self) -> dict[str, Any]:
+        available = self._backend is not None or self._backend_factory is not None
+        return {
+            "backend": _BACKEND_ID,
+            "baseline": _BACKEND_ID,
+            "available": available and self._integration_error is None,
+            "integration_status": "integrated" if available and self._integration_error is None else "unavailable",
+            "integration_error": self._integration_error or INTEGRATION_ERROR,
+            "delta_granularity": "ingest_turn_only",
+            "snapshot_mode": "full_store",
+        }
+
+    def _require_backend(self) -> Any:
+        if self._backend is None:
+            raise self._runtime_error()
+        return self._backend
+
+    @staticmethod
+    def _turn_text(turn: NormalizedTurn) -> str:
+        parts = [f"{turn.role}: {turn.text}"]
+        for att in turn.attachments:
+            parts.append(f"[{att.type}] {att.caption}")
+        return "\n".join(parts)

memory_adapters/amem_v2.py

@@ -0,0 +1,142 @@
+"""Adapter for A-Mem (new API: agentic_memory.AgenticMemorySystem)."""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+load_dotenv(Path(__file__).resolve().parents[2] / ".env")
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+
+_DEFAULT_SOURCE = Path("/data1/toby/nips26/baselines/A-Mem")
+
+
+class AMemV2Adapter(MemoryAdapter):
+    """Adapter for A-Mem (new agentic_memory API)."""
+
+    def __init__(
+        self,
+        *,
+        source_root: str | os.PathLike[str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        root = Path(source_root or _DEFAULT_SOURCE).resolve()
+        if str(root) not in sys.path:
+            sys.path.insert(0, str(root))
+
+        from agentic_memory.memory_system import AgenticMemorySystem
+
+        self._cls = AgenticMemorySystem
+        self._backend: Any = None
+        self._session_id = ""
+        self._prev_snapshot_ids: set[str] = set()
+        self._init_backend()
+
+    def _init_backend(self) -> None:
+        self._backend = self._cls(
+            model_name="all-MiniLM-L6-v2",
+            llm_backend="openai",
+            llm_model=os.getenv("OPENAI_MODEL") or "gpt-4o",
+            api_key=os.getenv("OPENAI_API_KEY"),
+        )
+
+    def reset(self) -> None:
+        self._init_backend()
+        self._prev_snapshot_ids = set()
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        self._session_id = turn.session_id
+        text = f"{turn.role}: {turn.text}"
+        for att in turn.attachments:
+            text += f"\n[{att.type}] {att.caption}"
+        self._backend.add_note(text, time=turn.timestamp)
+
+    def end_session(self, session_id: str) -> None:
+        self._session_id = session_id
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        rows: list[MemorySnapshotRecord] = []
+        for mid, note in self._backend.memories.items():
+            content = str(getattr(note, "content", ""))
+            context = getattr(note, "context", "")
+            keywords = list(getattr(note, "keywords", []) or [])
+            parts = [content]
+            if context:
+                parts.append(f"[context] {context}")
+            if keywords:
+                parts.append(f"[keywords] {', '.join(keywords)}")
+            rows.append(MemorySnapshotRecord(
+                memory_id=str(mid),
+                text="\n".join(parts),
+                session_id=self._session_id,
+                status="active",
+                source="A-Mem",
+                raw_backend_id=str(mid),
+                raw_backend_type="a_mem_note",
+                metadata={},
+            ))
+        return rows
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        current = self.snapshot_memories()
+        current_ids = {s.memory_id for s in current}
+        deltas = [
+            MemoryDeltaRecord(
+                session_id=session_id, op="add", text=s.text,
+                linked_previous=(), raw_backend_id=s.raw_backend_id,
+                metadata={"baseline": "A-Mem"},
+            )
+            for s in current if s.memory_id not in self._prev_snapshot_ids
+        ]
+        self._prev_snapshot_ids = current_ids
+        return deltas
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        items: list[RetrievalItem] = []
+        try:
+            results = self._backend.search(query, k=top_k)
+            for i, r in enumerate(results[:top_k]):
+                text = r.get("content", str(r)) if isinstance(r, dict) else str(r)
+                mid = r.get("id", str(i)) if isinstance(r, dict) else str(i)
+                score = float(r.get("score", 1.0 / (i + 1))) if isinstance(r, dict) else 1.0 / (i + 1)
+                items.append(RetrievalItem(
+                    rank=i, memory_id=str(mid), text=text,
+                    score=score, raw_backend_id=str(mid),
+                ))
+        except Exception:
+            # Fallback to raw search
+            try:
+                raw = self._backend.find_related_memories_raw(query, k=top_k)
+                if raw:
+                    items.append(RetrievalItem(
+                        rank=0, memory_id="bundle", text=str(raw),
+                        score=1.0, raw_backend_id=None,
+                    ))
+            except Exception:
+                pass
+
+        return RetrievalRecord(
+            query=query, top_k=top_k, items=items[:top_k],
+            raw_trace={"baseline": "A-Mem"},
+        )
+
+    def get_capabilities(self) -> dict[str, Any]:
+        return {
+            "backend": "A-Mem",
+            "baseline": "A-Mem",
+            "available": self._backend is not None,
+            "delta_granularity": "snapshot_diff",
+            "snapshot_mode": "full",
+        }

memory_adapters/base.py

@@ -0,0 +1,45 @@
+"""Abstract memory adapter API for eval baselines."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalRecord,
+)
+
+
+class MemoryAdapter(ABC):
+    """Baseline-agnostic adapter surface used by the eval pipeline."""
+
+    @abstractmethod
+    def reset(self) -> None:
+        """Clear backend state and any adapter-side bookkeeping."""
+
+    @abstractmethod
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        """Feed one conversation turn into the memory system."""
+
+    @abstractmethod
+    def end_session(self, session_id: str) -> None:
+        """Notify the adapter that a session boundary was reached (optional for many backends)."""
+
+    @abstractmethod
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        """Return a normalized view of memories observable in the backend."""
+
+    @abstractmethod
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        """Export memory changes for the given session since the last call."""
+
+    @abstractmethod
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        """Run retrieval and normalize results."""
+
+    @abstractmethod
+    def get_capabilities(self) -> dict[str, Any]:
+        """Describe adapter behavior limits (deltas, snapshots, backend id)."""

memory_adapters/dummy.py

@@ -0,0 +1,118 @@
+"""Dummy memory adapter for end-to-end pipeline testing.
+
+Stores all ingested turns as raw text and retrieves by simple substring match.
+No external dependencies required.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+
+
+class DummyAdapter(MemoryAdapter):
+    """Minimal adapter that stores turns verbatim — for pipeline testing."""
+
+    def __init__(self) -> None:
+        self._memories: list[dict[str, str]] = []
+        self._session_id = ""
+        self._prev_ids: set[str] = set()
+
+    def reset(self) -> None:
+        self._memories = []
+        self._session_id = ""
+        self._prev_ids = set()
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        self._session_id = turn.session_id
+        text = f"{turn.role}: {turn.text}"
+        for att in turn.attachments:
+            text += f"\n[{att.type}] {att.caption}"
+        mid = str(len(self._memories))
+        self._memories.append({
+            "id": mid,
+            "text": text,
+            "session_id": turn.session_id,
+        })
+
+    def end_session(self, session_id: str) -> None:
+        self._session_id = session_id
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        return [
+            MemorySnapshotRecord(
+                memory_id=m["id"],
+                text=m["text"],
+                session_id=m["session_id"],
+                status="active",
+                source="Dummy",
+                raw_backend_id=m["id"],
+                raw_backend_type="dummy",
+                metadata={},
+            )
+            for m in self._memories
+        ]
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        current_ids = {m["id"] for m in self._memories}
+        new_ids = current_ids - self._prev_ids
+        deltas = [
+            MemoryDeltaRecord(
+                session_id=session_id,
+                op="add",
+                text=m["text"],
+                linked_previous=(),
+                raw_backend_id=m["id"],
+                metadata={"baseline": "Dummy"},
+            )
+            for m in self._memories
+            if m["id"] in new_ids
+        ]
+        self._prev_ids = current_ids
+        return deltas
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        query_lower = query.lower()
+        scored = []
+        for m in self._memories:
+            text_lower = m["text"].lower()
+            # Simple word overlap score
+            query_words = set(query_lower.split())
+            text_words = set(text_lower.split())
+            overlap = len(query_words & text_words)
+            scored.append((overlap, m))
+        scored.sort(key=lambda x: x[0], reverse=True)
+
+        items = [
+            RetrievalItem(
+                rank=i,
+                memory_id=m["id"],
+                text=m["text"],
+                score=float(overlap) / max(len(query.split()), 1),
+                raw_backend_id=m["id"],
+            )
+            for i, (overlap, m) in enumerate(scored[:top_k])
+        ]
+        return RetrievalRecord(
+            query=query,
+            top_k=top_k,
+            items=items,
+            raw_trace={"baseline": "Dummy"},
+        )
+
+    def get_capabilities(self) -> dict[str, Any]:
+        return {
+            "backend": "Dummy",
+            "baseline": "Dummy",
+            "available": True,
+            "delta_granularity": "per_turn",
+            "snapshot_mode": "full",
+        }

memory_adapters/export_utils.py

@@ -0,0 +1,123 @@
+"""Helpers to map turns, backend memory dicts, and recall outputs into shared schemas."""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+from eval_framework.datasets.schemas import (
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+
+
+def turn_to_observation_dict(turn: NormalizedTurn) -> dict[str, Any]:
+    """Build a Mem-Gallery store observation from a normalized turn."""
+    parts: list[str] = [turn.text]
+    for att in turn.attachments:
+        parts.append(f"[{att.type}] {att.caption}")
+    text = "\n".join(parts)
+    obs: dict[str, Any] = {"text": text}
+    if turn.timestamp:
+        obs["timestamp"] = turn.timestamp
+    obs["dialogue_id"] = f"{turn.session_id}:{turn.turn_index}"
+    return obs
+
+
+def memory_element_text(element: Mapping[str, Any]) -> str:
+    """Best-effort text extraction from a Mem-Gallery memory dict."""
+    raw = element.get("text", "")
+    if isinstance(raw, list):
+        return " ".join(str(x) for x in raw)
+    if raw is None:
+        base = ""
+    else:
+        base = str(raw)
+    image = element.get("image")
+    if isinstance(image, dict):
+        cap = image.get("caption")
+        if cap:
+            base = f"{base}\n[image] {cap}".strip()
+    return base
+
+
+def linear_element_to_snapshot(
+    element: Mapping[str, Any],
+    *,
+    memory_id: str,
+    session_id: str,
+    source: str,
+    status: str = "active",
+) -> MemorySnapshotRecord:
+    """Map a linear-storage memory dict into MemorySnapshotRecord."""
+    cid = element.get("counter_id")
+    raw_id = str(cid) if cid is not None else memory_id
+    return MemorySnapshotRecord(
+        memory_id=memory_id,
+        text=memory_element_text(element),
+        session_id=session_id,
+        status=status,
+        source=source,
+        raw_backend_id=raw_id,
+        raw_backend_type="linear",
+        metadata={},
+    )
+
+
+def normalize_recall_to_retrieval(
+    query: str,
+    top_k: int,
+    raw: Any,
+    *,
+    raw_trace: dict[str, Any] | None = None,
+) -> RetrievalRecord:
+    """Normalize Mem-Gallery recall outputs into RetrievalRecord."""
+    trace = dict(raw_trace or {})
+    items: list[RetrievalItem] = []
+
+    if isinstance(raw, str):
+        items.append(
+            RetrievalItem(
+                rank=0,
+                memory_id="memgallery:string_bundle",
+                text=raw,
+                score=1.0,
+                raw_backend_id=None,
+            )
+        )
+    elif isinstance(raw, list):
+        for i, row in enumerate(raw[: max(0, top_k)]):
+            if isinstance(row, dict):
+                mid = row.get("counter_id")
+                items.append(
+                    RetrievalItem(
+                        rank=i,
+                        memory_id=str(mid if mid is not None else i),
+                        text=memory_element_text(row),
+                        score=float(row.get("score", 1.0)),
+                        raw_backend_id=str(mid) if mid is not None else None,
+                    )
+                )
+            else:
+                items.append(
+                    RetrievalItem(
+                        rank=i,
+                        memory_id=str(i),
+                        text=str(row),
+                        score=1.0,
+                        raw_backend_id=None,
+                    )
+                )
+    else:
+        items.append(
+            RetrievalItem(
+                rank=0,
+                memory_id="memgallery:object_bundle",
+                text=str(raw),
+                score=1.0,
+                raw_backend_id=None,
+            )
+        )
+
+    return RetrievalRecord(query=query, top_k=top_k, items=items[:top_k], raw_trace=trace)

memory_adapters/mem0_adapter.py

@@ -0,0 +1,185 @@
+"""Adapters for Mem0 and Mem0-Graph baselines."""
+
+from __future__ import annotations
+
+import os
+import uuid as _uuid
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+load_dotenv(Path(__file__).resolve().parents[2] / ".env")
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+
+
+class Mem0Adapter(MemoryAdapter):
+    """Adapter for Mem0 (vector mode)."""
+
+    def __init__(self, *, use_graph: bool = False, **kwargs: Any) -> None:
+        from mem0 import Memory
+
+        self._user_id = f"eval_{_uuid.uuid4().hex[:8]}"
+        self._session_id = ""
+        self._prev_snapshot_ids: set[str] = set()
+
+        config: dict[str, Any] = {
+            "llm": {
+                "provider": "openai",
+                "config": {
+                    "model": os.getenv("OPENAI_MODEL") or "gpt-4o",
+                    "api_key": os.getenv("OPENAI_API_KEY") or "",
+                },
+            },
+            "embedder": {
+                "provider": "openai",
+                "config": {
+                    "model": "text-embedding-3-small",
+                    "api_key": os.getenv("OPENAI_API_KEY") or "",
+                    "embedding_dims": 1536,
+                },
+            },
+        }
+
+        base_url = os.getenv("OPENAI_BASE_URL")
+        if base_url:
+            config["llm"]["config"]["openai_base_url"] = base_url
+            config["embedder"]["config"]["openai_base_url"] = base_url
+
+        if use_graph:
+            config["graph_store"] = {
+                "provider": "kuzu",
+                "config": {
+                    "url": "/tmp/mem0_kuzu_eval",
+                },
+            }
+
+        self._memory = Memory.from_config(config)
+        self._use_graph = use_graph
+
+    def reset(self) -> None:
+        self._memory.delete_all(user_id=self._user_id)
+        self._user_id = f"eval_{_uuid.uuid4().hex[:8]}"
+        self._prev_snapshot_ids = set()
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        self._session_id = turn.session_id
+        text = f"{turn.role}: {turn.text}"
+        for att in turn.attachments:
+            text += f"\n[{att.type}] {att.caption}"
+        # Truncate to avoid excessively long inputs that break graph entity extraction
+        text = text[:2000]
+        try:
+            self._memory.add(
+                messages=[{"role": turn.role, "content": text}],
+                user_id=self._user_id,
+            )
+        except Exception:
+            # Graph mode can fail on entity embedding; fall back silently
+            pass
+
+    def end_session(self, session_id: str) -> None:
+        self._session_id = session_id
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        all_mems = self._memory.get_all(user_id=self._user_id)
+        rows: list[MemorySnapshotRecord] = []
+
+        # Vector results (standard mode)
+        results = all_mems.get("results", []) if isinstance(all_mems, dict) else all_mems
+        for mem in results:
+            mid = str(mem.get("id", ""))
+            text = str(mem.get("memory", ""))
+            rows.append(MemorySnapshotRecord(
+                memory_id=mid, text=text,
+                session_id=self._session_id, status="active",
+                source="Mem0", raw_backend_id=mid,
+                raw_backend_type="mem0_vector", metadata={},
+            ))
+
+        # Graph relations (graph mode)
+        relations = all_mems.get("relations", []) if isinstance(all_mems, dict) else []
+        for i, rel in enumerate(relations):
+            if isinstance(rel, dict):
+                src = rel.get("source", "")
+                rtype = rel.get("relationship", "")
+                tgt = rel.get("target") or rel.get("destination", "")
+                text = f"{src} → {rtype} → {tgt}"
+                mid = f"rel_{i}"
+                rows.append(MemorySnapshotRecord(
+                    memory_id=mid, text=text,
+                    session_id=self._session_id, status="active",
+                    source="Mem0-Graph", raw_backend_id=mid,
+                    raw_backend_type="mem0_graph_relation", metadata=rel,
+                ))
+
+        return rows
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        current = self.snapshot_memories()
+        current_ids = {s.memory_id for s in current}
+        deltas = [
+            MemoryDeltaRecord(
+                session_id=session_id,
+                op="add",
+                text=s.text,
+                linked_previous=(),
+                raw_backend_id=s.raw_backend_id,
+                metadata={"baseline": "Mem0"},
+            )
+            for s in current if s.memory_id not in self._prev_snapshot_ids
+        ]
+        self._prev_snapshot_ids = current_ids
+        return deltas
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        results = self._memory.search(query=query, user_id=self._user_id, limit=top_k)
+        items: list[RetrievalItem] = []
+
+        # Vector results
+        search_results = results.get("results", []) if isinstance(results, dict) else results
+        for i, r in enumerate(search_results[:top_k]):
+            items.append(RetrievalItem(
+                rank=len(items),
+                memory_id=str(r.get("id", i)),
+                text=str(r.get("memory", "")),
+                score=float(r.get("score", 1.0 / (i + 1))),
+                raw_backend_id=str(r.get("id", "")),
+            ))
+
+        # Graph relations
+        relations = results.get("relations", []) if isinstance(results, dict) else []
+        for rel in relations:
+            if isinstance(rel, dict) and len(items) < top_k:
+                src = rel.get("source", "")
+                rtype = rel.get("relationship", "")
+                tgt = rel.get("target") or rel.get("destination", "")
+                items.append(RetrievalItem(
+                    rank=len(items),
+                    memory_id=f"rel_{len(items)}",
+                    text=f"{src} → {rtype} → {tgt}",
+                    score=0.9,
+                    raw_backend_id=None,
+                ))
+
+        return RetrievalRecord(
+            query=query, top_k=top_k, items=items[:top_k],
+            raw_trace={"baseline": "Mem0-Graph" if self._use_graph else "Mem0"},
+        )
+
+    def get_capabilities(self) -> dict[str, Any]:
+        return {
+            "backend": "Mem0-Graph" if self._use_graph else "Mem0",
+            "baseline": "Mem0-Graph" if self._use_graph else "Mem0",
+            "available": True,
+            "delta_granularity": "snapshot_diff",
+            "snapshot_mode": "full",
+        }

memory_adapters/memgallery_native.py

@@ -0,0 +1,395 @@
+"""Mem-Gallery native baseline wrappers with conservative schema normalization."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, Callable
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+from eval_framework.memory_adapters.export_utils import (
+    linear_element_to_snapshot,
+    memory_element_text,
+    normalize_recall_to_retrieval,
+    turn_to_observation_dict,
+)
+
+
+def _deep_merge_dict(base: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
+    out = copy.deepcopy(base)
+    for key, val in overrides.items():
+        if (
+            key in out
+            and isinstance(out[key], dict)
+            and isinstance(val, dict)
+        ):
+            out[key] = _deep_merge_dict(out[key], val)
+        else:
+            out[key] = copy.deepcopy(val)
+    return out
+
+
+def _default_config_for_baseline(name: str) -> dict[str, Any]:
+    import default_config.DefaultMemoryConfig as dmc  # type: ignore[import-not-found]
+
+    key = {
+        "FUMemory": "DEFAULT_FUMEMORY",
+        "STMemory": "DEFAULT_STMEMORY",
+        "LTMemory": "DEFAULT_LTMEMORY",
+        "GAMemory": "DEFAULT_GAMEMORY",
+        "MGMemory": "DEFAULT_MGMEMORY",
+        "RFMemory": "DEFAULT_RFMEMORY",
+        "MMMemory": "DEFAULT_MMMEMORY",
+        "MMFUMemory": "DEFAULT_MMFUMEMORY",
+        "NGMemory": "DEFAULT_NGMEMORY",
+        "AUGUSTUSMemory": "DEFAULT_AUGUSTUSMEMORY",
+        "UniversalRAGMemory": "DEFAULT_UNIVERSALRAGMEMORY",
+    }[name]
+    cfg = getattr(dmc, key)
+    return copy.deepcopy(cfg)
+
+
+def _import_memory_class(name: str) -> Callable[..., Any]:
+    modmap = {
+        "FUMemory": ("memengine.memory.FUMemory", "FUMemory"),
+        "STMemory": ("memengine.memory.STMemory", "STMemory"),
+        "LTMemory": ("memengine.memory.LTMemory", "LTMemory"),
+        "GAMemory": ("memengine.memory.GAMemory", "GAMemory"),
+        "MGMemory": ("memengine.memory.MGMemory", "MGMemory"),
+        "RFMemory": ("memengine.memory.RFMemory", "RFMemory"),
+        "MMMemory": ("memengine.memory.MMMemory", "MMMemory"),
+        "MMFUMemory": ("memengine.memory.MMFUMemory", "MMFUMemory"),
+        "NGMemory": ("memengine.memory.NGMemory", "NGMemory"),
+        "AUGUSTUSMemory": ("memengine.memory.AUGUSTUSMemory", "AUGUSTUSMemory"),
+        "UniversalRAGMemory": ("memengine.memory.UniversalRAGMemory", "UniversalRAGMemory"),
+    }
+    module_path, cls_name = modmap[name]
+    import importlib
+
+    mod = importlib.import_module(module_path)
+    return getattr(mod, cls_name)
+
+
+def instantiate_memgallery_memory(
+    baseline_name: str,
+    config: dict[str, Any] | None = None,
+) -> Any:
+    """Construct a Mem-Gallery memory object with optional config overrides."""
+    base_cfg = _default_config_for_baseline(baseline_name)
+    merged = _deep_merge_dict(base_cfg, config or {})
+    from memengine.config.Config import MemoryConfig  # type: ignore[import-not-found]
+
+    cls = _import_memory_class(baseline_name)
+    return cls(MemoryConfig(merged))
+
+
+def _graph_nodes_to_snapshots(
+    storage: Any,
+    *,
+    session_id: str,
+    source: str,
+    include_concepts: bool = False,
+) -> list[MemorySnapshotRecord]:
+    out: list[MemorySnapshotRecord] = []
+    order = getattr(storage, "memory_order_map", []) or []
+    node_concepts = getattr(storage, "node_concepts", {})
+    for mid_idx, node_id in enumerate(order):
+        node = storage.node[node_id]
+        cid = node.get("counter_id", mid_idx)
+        memory_id = f"n{node_id}"
+        text = memory_element_text(node)
+        # For AUGUSTUS: append concept tags extracted by the system
+        if include_concepts:
+            concepts = node_concepts.get(node_id, set())
+            if concepts:
+                text = f"{text}\n[concepts] {', '.join(sorted(concepts))}"
+        out.append(
+            MemorySnapshotRecord(
+                memory_id=memory_id,
+                text=text,
+                session_id=session_id,
+                status="active",
+                source=source,
+                raw_backend_id=str(cid),
+                raw_backend_type="graph_node",
+                metadata={"node_id": node_id},
+            )
+        )
+    return out
+
+
+def _linear_storage_snapshots(
+    storage: Any,
+    *,
+    session_id: str,
+    source: str,
+) -> list[MemorySnapshotRecord]:
+    rows: list[MemorySnapshotRecord] = []
+    for i, m in enumerate(storage.memory_list):
+        cid = m.get("counter_id", i)
+        rows.append(
+            linear_element_to_snapshot(
+                m,
+                memory_id=str(cid),
+                session_id=session_id,
+                source=source,
+            )
+        )
+    return rows
+
+
+def collect_memgallery_snapshots(
+    memory: Any,
+    baseline_name: str,
+    session_id: str,
+) -> list[MemorySnapshotRecord]:
+    """Best-effort snapshot of backend-visible memories."""
+    source = baseline_name
+    if baseline_name == "MGMemory":
+        out: list[MemorySnapshotRecord] = []
+        # store_op/recall_op have their own main_context references;
+        # prefer store_op's view as it holds the actual stored data.
+        mc = getattr(memory.store_op, "main_context", None) or memory.main_context
+        recall_storage = getattr(memory.recall_op, "recall_storage",
+                                 getattr(memory, "recall_storage", None))
+        archival_storage = getattr(memory.recall_op, "archival_storage",
+                                   getattr(memory, "archival_storage", None))
+        storages = [("wm", mc["working_context"]), ("fifo", mc["FIFO_queue"])]
+        if recall_storage is not None:
+            storages.append(("recall", recall_storage))
+        if archival_storage is not None:
+            storages.append(("archival", archival_storage))
+        for prefix, st in storages:
+            for i, m in enumerate(st.memory_list):
+                cid = m.get("counter_id", i)
+                mid = f"{prefix}-{cid}"
+                rows = linear_element_to_snapshot(
+                    m,
+                    memory_id=mid,
+                    session_id=session_id,
+                    source=source,
+                )
+                out.append(rows)
+        gsum = mc.get("recursive_summary", {}).get("global")
+        if gsum and str(gsum) != "None":
+            out.append(
+                MemorySnapshotRecord(
+                    memory_id="recursive_summary",
+                    text=str(gsum),
+                    session_id=session_id,
+                    status="active",
+                    source=source,
+                    raw_backend_id=None,
+                    raw_backend_type="mg_summary",
+                    metadata={},
+                )
+            )
+        return out
+
+    if baseline_name == "RFMemory":
+        rows = _linear_storage_snapshots(
+            memory.storage, session_id=session_id, source=source
+        )
+        insight = getattr(memory, "insight", {}).get("global_insight", "")
+        if insight:
+            rows.append(
+                MemorySnapshotRecord(
+                    memory_id="rf_insight",
+                    text=str(insight),
+                    session_id=session_id,
+                    status="active",
+                    source=source,
+                    raw_backend_id=None,
+                    raw_backend_type="rf_insight",
+                    metadata={},
+                )
+            )
+        return rows
+
+    if baseline_name == "NGMemory":
+        return _graph_nodes_to_snapshots(
+            memory.storage, session_id=session_id, source=source
+        )
+
+    if baseline_name == "AUGUSTUSMemory":
+        return _graph_nodes_to_snapshots(
+            memory.contextual_memory, session_id=session_id, source=source,
+            include_concepts=True,
+        )
+
+    if baseline_name == "UniversalRAGMemory":
+        return _linear_storage_snapshots(
+            memory.storage, session_id=session_id, source=source
+        )
+
+    if hasattr(memory, "storage") and hasattr(memory.storage, "memory_list"):
+        return _linear_storage_snapshots(
+            memory.storage, session_id=session_id, source=source
+        )
+
+    return []
+
+
+class MemGalleryNativeAdapter(MemoryAdapter):
+    """Thin wrapper that forwards to Mem-Gallery memories and normalizes I/O."""
+
+    def __init__(self, memory: Any, *, baseline_name: str) -> None:
+        self._memory = memory
+        self._baseline_name = baseline_name
+        self._session_id = ""
+        self._prev_snapshot_ids: set[str] = set()
+        self._pending_user_turn: NormalizedTurn | None = None
+        self._session_turns: list[str] = []  # collect turn texts for RF optimize
+
+    @classmethod
+    def from_baseline(
+        cls,
+        baseline_name: str,
+        *,
+        config: dict[str, Any] | None = None,
+    ) -> MemGalleryNativeAdapter:
+        mem = instantiate_memgallery_memory(baseline_name, config)
+        return cls(mem, baseline_name=baseline_name)
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        """Buffer user turns; store merged user+assistant pair on assistant turn.
+
+        This matches the original Mem-Gallery benchmark behavior where each
+        dialogue round (user + assistant) is merged into a single observation
+        before calling store().
+        """
+        self._session_id = turn.session_id
+        if turn.role == "user":
+            # Flush any prior unpaired user turn, then buffer this one
+            if self._pending_user_turn is not None:
+                self._store_observation(self._pending_user_turn, assistant_turn=None)
+            self._pending_user_turn = turn
+        else:
+            # Assistant turn: merge with buffered user turn and store
+            self._store_observation(self._pending_user_turn, assistant_turn=turn)
+            self._pending_user_turn = None
+
+    def _store_observation(
+        self,
+        user_turn: NormalizedTurn | None,
+        assistant_turn: NormalizedTurn | None,
+    ) -> None:
+        """Build a merged observation dict (matching original benchmark format) and store."""
+        parts: list[str] = []
+        timestamp = None
+        dialogue_id = ""
+        if user_turn is not None:
+            parts.append(f"user: {user_turn.text}")
+            for att in user_turn.attachments:
+                parts.append(f"[{att.type}] {att.caption}")
+            timestamp = user_turn.timestamp
+            dialogue_id = f"{user_turn.session_id}:{user_turn.turn_index}"
+        if assistant_turn is not None:
+            parts.append(f"assistant: {assistant_turn.text}")
+            for att in assistant_turn.attachments:
+                parts.append(f"[{att.type}] {att.caption}")
+            if timestamp is None:
+                timestamp = assistant_turn.timestamp
+            if not dialogue_id:
+                dialogue_id = f"{assistant_turn.session_id}:{assistant_turn.turn_index}"
+
+        obs: dict[str, Any] = {"text": "\n".join(parts)}
+        if timestamp:
+            obs["timestamp"] = timestamp
+        obs["dialogue_id"] = dialogue_id
+        self._memory.store(obs)
+        self._session_turns.append(obs["text"])
+
+    def end_session(self, session_id: str) -> None:
+        # Flush any remaining unpaired user turn
+        if self._pending_user_turn is not None:
+            self._store_observation(self._pending_user_turn, assistant_turn=None)
+            self._pending_user_turn = None
+
+        # --- Trigger backend-specific post-session processing ---
+        # GAMemory: self-reflection generates insights and stores them
+        if self._baseline_name == "GAMemory":
+            try:
+                self._memory.manage("reflect")
+            except Exception:
+                pass  # reflection may fail if accumulated importance < threshold
+
+        # RFMemory: optimize generates a global insight from the session trial
+        if self._baseline_name == "RFMemory" and self._session_turns:
+            try:
+                trial = "\n".join(self._session_turns)
+                self._memory.optimize(new_trial=trial)
+            except Exception:
+                pass
+
+        self._session_turns = []
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        sid = self._session_id or ""
+        return collect_memgallery_snapshots(
+            self._memory, self._baseline_name, sid
+        )
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        """Export delta by diffing current backend snapshot against previous snapshot.
+
+        This reflects what the backend ACTUALLY stores, not what was fed in.
+        For FU/ST/LT/GA/RF (LinearStorage), this is the raw observations added.
+        For MGMemory, this includes FIFO items, summaries, and archival entries.
+        """
+        current_snapshot = self.snapshot_memories()
+        prev_ids = self._prev_snapshot_ids
+        deltas: list[MemoryDeltaRecord] = []
+        current_ids: set[str] = set()
+
+        for snap in current_snapshot:
+            current_ids.add(snap.memory_id)
+            if snap.memory_id not in prev_ids:
+                deltas.append(
+                    MemoryDeltaRecord(
+                        session_id=session_id,
+                        op="add",
+                        text=snap.text,
+                        linked_previous=(),
+                        raw_backend_id=snap.raw_backend_id,
+                        metadata={
+                            "baseline": self._baseline_name,
+                            "source": snap.source,
+                            "backend_type": snap.raw_backend_type,
+                        },
+                    )
+                )
+
+        self._prev_snapshot_ids = current_ids
+        return deltas
+
+    def reset(self) -> None:
+        self._memory.reset()
+        self._prev_snapshot_ids = set()
+        self._pending_user_turn = None
+        self._session_turns = []
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        raw = self._memory.recall(query)
+        trace: dict[str, Any] = {"baseline": self._baseline_name}
+        ro = getattr(self._memory, "recall_op", None)
+        if ro is not None and hasattr(ro, "last_retrieved_ids"):
+            trace["last_retrieved_ids"] = list(ro.last_retrieved_ids)
+        return normalize_recall_to_retrieval(query, top_k, raw, raw_trace=trace)
+
+    def get_capabilities(self) -> dict[str, Any]:
+        return {
+            "backend": "MemGallery",
+            "baseline": self._baseline_name,
+            "delta_granularity": "ingest_turn_only",
+            "snapshot_mode": "conservative",
+            "notes": (
+                "Deltas record adapter ingest only; backend-internal rewrite, reflection, "
+                "or graph reshaping is not diffed. Snapshots read observable storage where supported."
+            ),
+        }

memory_adapters/memoryos.py

@@ -0,0 +1,357 @@
+"""Adapter for the external MemoryOS baseline."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any, Callable
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+
+_BACKEND_ID = "MemoryOS"
+
+INTEGRATION_ERROR = (
+    f"{_BACKEND_ID} backend unavailable."
+)
+
+
+class MemoryOSAdapter(MemoryAdapter):
+    """Thin wrapper around MemoryOS's local Python API."""
+
+    def __init__(
+        self,
+        *,
+        backend: Any | None = None,
+        backend_factory: Callable[[], Any] | None = None,
+        source_root: str | os.PathLike[str] | None = None,
+        storage_root: str | os.PathLike[str] | None = None,
+        user_id: str = "eval_user",
+        assistant_id: str = "eval_assistant",
+        llm_model: str | None = None,
+        embedding_model_name: str = "all-MiniLM-L6-v2",
+        openai_api_key: str | None = None,
+        openai_base_url: str | None = None,
+    ) -> None:
+        self._source_root = Path(source_root).resolve() if source_root else self._default_source_root()
+        self._storage_root = Path(storage_root).resolve() if storage_root else Path(
+            tempfile.mkdtemp(prefix="memoryos_eval_")
+        )
+        self._user_id = user_id
+        self._assistant_id = assistant_id
+        self._llm_model = llm_model or os.getenv("OPENAI_MODEL") or "gpt-5.1"
+        self._embedding_model_name = embedding_model_name
+        self._openai_api_key = openai_api_key or os.getenv("OPENAI_API_KEY")
+        self._openai_base_url = openai_base_url or os.getenv("OPENAI_BASE_URL")
+        self._backend_factory = backend_factory
+        self._backend: Any | None = None
+        self._integration_error: str | None = None
+        self._session_id = ""
+        self._prev_snapshot_ids: set[str] = set()
+        self._pending_user_turns: list[NormalizedTurn] = []
+
+        if backend is not None:
+            self._backend = backend
+        else:
+            try:
+                if self._backend_factory is None:
+                    self._backend_factory = self._build_backend_factory()
+                self._backend = self._backend_factory()
+            except Exception as exc:
+                self._integration_error = str(exc)
+
+    @staticmethod
+    def _default_source_root() -> Path:
+        here = Path(__file__).resolve()
+        # memory_adapters/ -> eval_framework/ -> nips26/ -> baselines/MemoryOS/memoryos-pypi
+        return (here.parents[2] / "baselines" / "MemoryOS" / "memoryos-pypi").resolve()
+
+    def _build_backend_factory(self) -> Callable[[], Any]:
+        if not self._source_root.is_dir():
+            raise RuntimeError(
+                f"{_BACKEND_ID}: source root not found at {self._source_root}"
+            )
+        src = str(self._source_root)
+        if src not in sys.path:
+            sys.path.insert(0, src)
+        mod = importlib.import_module("memoryos")
+        backend_cls = getattr(mod, "Memoryos")
+
+        def _factory() -> Any:
+            run_root = self._storage_root / "runtime"
+            shutil.rmtree(run_root, ignore_errors=True)
+            run_root.mkdir(parents=True, exist_ok=True)
+            return backend_cls(
+                user_id=self._user_id,
+                openai_api_key=self._openai_api_key or "",
+                openai_base_url=self._openai_base_url,
+                data_storage_path=str(run_root),
+                llm_model=self._llm_model,
+                assistant_id=self._assistant_id,
+                embedding_model_name=self._embedding_model_name,
+            )
+
+        return _factory
+
+    def _runtime_error(self) -> RuntimeError:
+        detail = self._integration_error or INTEGRATION_ERROR
+        return RuntimeError(
+            f"{_BACKEND_ID}: backend unavailable — {detail}"
+        )
+
+    def reset(self) -> None:
+        if self._backend_factory is None and self._backend is None:
+            raise self._runtime_error()
+        if self._backend_factory is not None:
+            self._backend = self._backend_factory()
+        self._prev_snapshot_ids = set()
+        self._pending_user_turns = []
+        self._session_id = ""
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        self._require_backend()
+        self._session_id = turn.session_id
+        if turn.role == "assistant":
+            self._store_pair(turn)
+        else:
+            self._pending_user_turns.append(turn)
+
+    def end_session(self, session_id: str) -> None:
+        self._require_backend()
+        self._session_id = session_id
+        if self._pending_user_turns:
+            synthetic = self._pending_user_turns[-1]
+            self._store_memory(
+                session_id=session_id,
+                user_input=self._joined_user_text(),
+                agent_response="",
+                timestamp=synthetic.timestamp,
+            )
+            self._pending_user_turns = []
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        backend = self._require_backend()
+        rows: list[MemorySnapshotRecord] = []
+        sid = self._session_id
+
+        for idx, qa in enumerate(backend.short_term_memory.get_all()):
+            rows.append(
+                MemorySnapshotRecord(
+                    memory_id=f"st:{idx}",
+                    text=self._format_qa_text(qa),
+                    session_id=sid,
+                    status="active",
+                    source=_BACKEND_ID,
+                    raw_backend_id=f"st:{idx}",
+                    raw_backend_type="short_term",
+                    metadata={"timestamp": qa.get("timestamp")},
+                )
+            )
+
+        for internal_session_id, session in getattr(backend.mid_term_memory, "sessions", {}).items():
+            for page_idx, page in enumerate(session.get("details", [])):
+                rows.append(
+                    MemorySnapshotRecord(
+                        memory_id=f"mt:{internal_session_id}:{page_idx}",
+                        text=self._format_qa_text(page),
+                        session_id=sid,
+                        status="active",
+                        source=_BACKEND_ID,
+                        raw_backend_id=str(page.get("page_id", f"{internal_session_id}:{page_idx}")),
+                        raw_backend_type="mid_term_page",
+                        metadata={"memoryos_session_id": internal_session_id},
+                    )
+                )
+
+        user_profile = backend.user_long_term_memory.get_raw_user_profile(backend.user_id)
+        if user_profile and str(user_profile).lower() != "none":
+            rows.append(
+                MemorySnapshotRecord(
+                    memory_id="lt:user_profile",
+                    text=str(user_profile),
+                    session_id=sid,
+                    status="active",
+                    source=_BACKEND_ID,
+                    raw_backend_id="user_profile",
+                    raw_backend_type="user_profile",
+                    metadata={},
+                )
+            )
+
+        for idx, item in enumerate(backend.user_long_term_memory.get_user_knowledge()):
+            rows.append(
+                MemorySnapshotRecord(
+                    memory_id=f"lt:user:{idx}",
+                    text=str(item.get("knowledge", "")),
+                    session_id=sid,
+                    status="active",
+                    source=_BACKEND_ID,
+                    raw_backend_id=f"user:{idx}",
+                    raw_backend_type="user_knowledge",
+                    metadata={"timestamp": item.get("timestamp")},
+                )
+            )
+
+        assistant_ltm = getattr(backend, "assistant_long_term_memory", None)
+        if assistant_ltm is not None and hasattr(assistant_ltm, "get_assistant_knowledge"):
+            for idx, item in enumerate(assistant_ltm.get_assistant_knowledge()):
+                rows.append(
+                    MemorySnapshotRecord(
+                        memory_id=f"lt:assistant:{idx}",
+                        text=str(item.get("knowledge", "")),
+                        session_id=sid,
+                        status="active",
+                        source=_BACKEND_ID,
+                        raw_backend_id=f"assistant:{idx}",
+                        raw_backend_type="assistant_knowledge",
+                        metadata={"timestamp": item.get("timestamp")},
+                    )
+                )
+        return rows
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        """Export delta by diffing current snapshot against previous snapshot."""
+        self._require_backend()
+        current_snapshot = self.snapshot_memories()
+        deltas: list[MemoryDeltaRecord] = []
+        current_ids: set[str] = set()
+
+        for snap in current_snapshot:
+            current_ids.add(snap.memory_id)
+            if snap.memory_id not in self._prev_snapshot_ids:
+                deltas.append(
+                    MemoryDeltaRecord(
+                        session_id=session_id,
+                        op="add",
+                        text=snap.text,
+                        linked_previous=(),
+                        raw_backend_id=snap.raw_backend_id,
+                        metadata={
+                            "baseline": _BACKEND_ID,
+                            "backend_type": snap.raw_backend_type,
+                        },
+                    )
+                )
+
+        self._prev_snapshot_ids = current_ids
+        return deltas
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        backend = self._require_backend()
+        raw = backend.retriever.retrieve_context(query, user_id=backend.user_id)
+        items: list[RetrievalItem] = []
+
+        for page in raw.get("retrieved_pages", []):
+            items.append(
+                RetrievalItem(
+                    rank=len(items),
+                    memory_id=f"page:{len(items)}",
+                    text=self._format_qa_text(page),
+                    score=1.0 / float(len(items) + 1),
+                    raw_backend_id=page.get("page_id"),
+                )
+            )
+        for item in raw.get("retrieved_user_knowledge", []):
+            items.append(
+                RetrievalItem(
+                    rank=len(items),
+                    memory_id=f"user:{len(items)}",
+                    text=str(item.get("knowledge", "")),
+                    score=1.0 / float(len(items) + 1),
+                    raw_backend_id=None,
+                )
+            )
+        for item in raw.get("retrieved_assistant_knowledge", []):
+            items.append(
+                RetrievalItem(
+                    rank=len(items),
+                    memory_id=f"assistant:{len(items)}",
+                    text=str(item.get("knowledge", "")),
+                    score=1.0 / float(len(items) + 1),
+                    raw_backend_id=None,
+                )
+            )
+        return RetrievalRecord(
+            query=query,
+            top_k=top_k,
+            items=items[:top_k],
+            raw_trace={"baseline": _BACKEND_ID, "retrieved_at": raw.get("retrieved_at")},
+        )
+
+    def get_capabilities(self) -> dict[str, Any]:
+        available = self._backend is not None or self._backend_factory is not None
+        return {
+            "backend": _BACKEND_ID,
+            "baseline": _BACKEND_ID,
+            "available": available and self._integration_error is None,
+            "integration_status": "integrated" if available and self._integration_error is None else "unavailable",
+            "integration_error": self._integration_error or INTEGRATION_ERROR,
+            "delta_granularity": "ingest_pair_only",
+            "snapshot_mode": "short_mid_long_term",
+        }
+
+    def _require_backend(self) -> Any:
+        if self._backend is None:
+            raise self._runtime_error()
+        return self._backend
+
+    def _store_pair(self, assistant_turn: NormalizedTurn) -> None:
+        user_input = self._joined_user_text()
+        self._store_memory(
+            session_id=assistant_turn.session_id,
+            user_input=user_input,
+            agent_response=self._turn_text(assistant_turn),
+            timestamp=assistant_turn.timestamp,
+        )
+        self._pending_user_turns = []
+
+    def _store_memory(
+        self,
+        *,
+        session_id: str,
+        user_input: str,
+        agent_response: str,
+        timestamp: str | None,
+    ) -> None:
+        backend = self._require_backend()
+        backend.add_memory(
+            user_input=user_input,
+            agent_response=agent_response,
+            timestamp=timestamp,
+            meta_data={"session_id": session_id},
+        )
+
+    def _joined_user_text(self) -> str:
+        if not self._pending_user_turns:
+            return ""
+        return "\n".join(self._turn_text(turn) for turn in self._pending_user_turns)
+
+    @staticmethod
+    def _turn_text(turn: NormalizedTurn) -> str:
+        parts = [turn.text]
+        for att in turn.attachments:
+            parts.append(f"[{att.type}] {att.caption}")
+        return "\n".join(parts)
+
+    @staticmethod
+    def _format_qa_text(item: dict[str, Any]) -> str:
+        parts = []
+        user_text = item.get("user_input", "")
+        if user_text:
+            parts.append(f"user: {user_text}")
+        assistant_text = item.get("agent_response", "")
+        if assistant_text:
+            parts.append(f"assistant: {assistant_text}")
+        if not parts:
+            parts.append(str(item))
+        return "\n".join(parts)

memory_adapters/memverse_adapter.py

@@ -0,0 +1,203 @@
+"""Adapter for MemVerse — uses build_memory for storage + cosine retrieval."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from dotenv import load_dotenv
+
+load_dotenv(Path(__file__).resolve().parents[2] / ".env")
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+
+_DEFAULT_SOURCE = Path("/data1/toby/nips26/baselines/MemVerse")
+
+
+class MemVerseAdapter(MemoryAdapter):
+    """Adapter for MemVerse using build_memory + cosine retrieval.
+
+    Bypasses the async orchestrator/LightRAG and uses the core
+    memory building + embedding-based retrieval directly.
+    """
+
+    def __init__(
+        self,
+        *,
+        source_root: str | os.PathLike[str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        root = Path(source_root or _DEFAULT_SOURCE).resolve()
+        if str(root) not in sys.path:
+            sys.path.insert(0, str(root))
+
+        from openai import OpenAI
+
+        self._client = OpenAI(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            base_url=os.getenv("OPENAI_BASE_URL"),
+        )
+        self._model = os.getenv("OPENAI_MODEL") or "gpt-4o"
+
+        # Working directory for memory files
+        self._work_dir = Path(tempfile.mkdtemp(prefix="memverse_eval_"))
+        self._root = root
+        self._session_id = ""
+        self._prev_snapshot_ids: set[str] = set()
+        self._memories: list[dict[str, Any]] = []  # {id, text, embedding, output}
+        self._conversation: list[dict[str, Any]] = []
+        self._turn_counter = 0
+
+        # Load system prompts for memory agents
+        self._prompts: dict[str, str] = {}
+        for name in ["core_memory_agent", "episodic_memory_agent", "semantic_memory_agent"]:
+            prompt_path = root / "MemoryKB" / "Long_Term_Memory" / "system" / f"{name}.txt"
+            if prompt_path.exists():
+                self._prompts[name] = prompt_path.read_text(encoding="utf-8").strip()
+
+    def _get_embedding(self, text: str) -> np.ndarray:
+        resp = self._client.embeddings.create(
+            model="text-embedding-3-small",
+            input=text,
+        )
+        return np.array(resp.data[0].embedding)
+
+    def _cosine_sim(self, a: np.ndarray, b: np.ndarray) -> float:
+        norm = np.linalg.norm(a) * np.linalg.norm(b)
+        if norm == 0:
+            return 0.0
+        return float(np.dot(a, b) / norm)
+
+    def reset(self) -> None:
+        self._memories = []
+        self._conversation = []
+        self._prev_snapshot_ids = set()
+        self._turn_counter = 0
+        if self._work_dir.exists():
+            shutil.rmtree(self._work_dir, ignore_errors=True)
+        self._work_dir = Path(tempfile.mkdtemp(prefix="memverse_eval_"))
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        self._session_id = turn.session_id
+        text = f"{turn.role}: {turn.text}"
+        for att in turn.attachments:
+            text += f"\n[{att.type}] {att.caption}"
+
+        entry_id = f"turn_{self._turn_counter}"
+        self._turn_counter += 1
+
+        entry = {
+            "id": entry_id,
+            "query": text,
+            "videocaption": None,
+            "audiocaption": None,
+            "imagecaption": None,
+        }
+        self._conversation.append(entry)
+
+        # Build memory: get embedding + LLM extraction for each memory type
+        embedding = self._get_embedding(text)
+
+        # Use the first available prompt (core memory agent) for extraction
+        prompt = next(iter(self._prompts.values()), "Extract key facts from this text.")
+        try:
+            resp = self._client.chat.completions.create(
+                model=self._model,
+                messages=[
+                    {"role": "system", "content": prompt},
+                    {"role": "user", "content": text},
+                ],
+                temperature=0,
+                max_tokens=512,
+            )
+            output = resp.choices[0].message.content or ""
+        except Exception:
+            output = text
+
+        self._memories.append({
+            "id": entry_id,
+            "text": text,
+            "output": output,
+            "embedding": embedding,
+            "session_id": turn.session_id,
+        })
+
+    def end_session(self, session_id: str) -> None:
+        self._session_id = session_id
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        return [
+            MemorySnapshotRecord(
+                memory_id=m["id"],
+                text=m["output"],
+                session_id=m.get("session_id", self._session_id),
+                status="active",
+                source="MemVerse",
+                raw_backend_id=m["id"],
+                raw_backend_type="memverse",
+                metadata={},
+            )
+            for m in self._memories
+        ]
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        current = self.snapshot_memories()
+        current_ids = {s.memory_id for s in current}
+        deltas = [
+            MemoryDeltaRecord(
+                session_id=session_id, op="add", text=s.text,
+                linked_previous=(), raw_backend_id=s.raw_backend_id,
+                metadata={"baseline": "MemVerse"},
+            )
+            for s in current if s.memory_id not in self._prev_snapshot_ids
+        ]
+        self._prev_snapshot_ids = current_ids
+        return deltas
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        if not self._memories:
+            return RetrievalRecord(query=query, top_k=top_k, items=[], raw_trace={})
+
+        query_emb = self._get_embedding(query)
+        scored = []
+        for m in self._memories:
+            sim = self._cosine_sim(query_emb, m["embedding"])
+            scored.append((sim, m))
+        scored.sort(key=lambda x: x[0], reverse=True)
+
+        items = [
+            RetrievalItem(
+                rank=i,
+                memory_id=m["id"],
+                text=m["output"],
+                score=float(sim),
+                raw_backend_id=m["id"],
+            )
+            for i, (sim, m) in enumerate(scored[:top_k])
+        ]
+        return RetrievalRecord(
+            query=query, top_k=top_k, items=items,
+            raw_trace={"baseline": "MemVerse"},
+        )
+
+    def get_capabilities(self) -> dict[str, Any]:
+        return {
+            "backend": "MemVerse",
+            "baseline": "MemVerse",
+            "available": True,
+            "delta_granularity": "per_turn",
+            "snapshot_mode": "full",
+        }

memory_adapters/registry.py

@@ -0,0 +1,410 @@
+"""Registry and factory for native Mem-Gallery and external placeholder adapters."""
+
+from __future__ import annotations
+
+import os
+import sys
+import types
+from contextlib import nullcontext
+from functools import partial
+import importlib
+from pathlib import Path
+from typing import Any, Callable
+
+from eval_framework.memory_adapters.amem import AMemAdapter
+from eval_framework.memory_adapters.base import MemoryAdapter
+from eval_framework.memory_adapters.dummy import DummyAdapter
+from eval_framework.memory_adapters.memgallery_native import MemGalleryNativeAdapter
+from eval_framework.memory_adapters.memoryos import MemoryOSAdapter
+
+MEMGALLERY_NATIVE_BASELINES: frozenset[str] = frozenset(
+    {
+        "FUMemory",
+        "STMemory",
+        "LTMemory",
+        "GAMemory",
+        "MGMemory",
+        "RFMemory",
+        "MMMemory",
+        "MMFUMemory",
+        "NGMemory",
+        "AUGUSTUSMemory",
+        "UniversalRAGMemory",
+    }
+)
+
+
+def _word_mode_truncation(number: int = 50_000) -> dict[str, Any]:
+    return {
+        "method": "LMTruncation",
+        "mode": "word",
+        "number": number,
+        "path": "",
+    }
+
+
+def _text_encoder_override() -> dict[str, Any]:
+    return {
+        "method": "STEncoder",
+        "path": "all-MiniLM-L6-v2",
+    }
+
+
+def _openai_llm_override() -> dict[str, Any]:
+    return {
+        "method": "APILLM",
+        "name": os.getenv("OPENAI_MODEL") or "gpt-5.1",
+        "api_key": os.getenv("OPENAI_API_KEY") or "",
+        "base_url": os.getenv("OPENAI_BASE_URL") or "https://api.openai.com/v1",
+        "temperature": float(os.getenv("OPENAI_TEMPERATURE", "0.0")),
+    }
+
+
+def _default_memgallery_runtime_overrides(baseline_name: str) -> dict[str, Any]:
+    overrides: dict[str, Any] = {}
+
+    # --- text-only baselines ---
+    if baseline_name in {"FUMemory", "STMemory", "LTMemory", "RFMemory"}:
+        overrides["recall"] = {"truncation": _word_mode_truncation()}
+    if baseline_name == "LTMemory":
+        overrides.setdefault("recall", {})
+        overrides["recall"]["text_retrieval"] = {"encoder": _text_encoder_override()}
+    if baseline_name == "GAMemory":
+        overrides = {
+            "recall": {
+                "truncation": _word_mode_truncation(),
+                "text_retrieval": {"encoder": _text_encoder_override()},
+                "importance_judge": {"LLM_config": _openai_llm_override()},
+            },
+            "reflect": {
+                "reflector": {"LLM_config": _openai_llm_override()},
+            },
+        }
+    if baseline_name == "MGMemory":
+        overrides = {
+            "recall": {
+                "truncation": _word_mode_truncation(),
+                "recall_retrieval": {"encoder": _text_encoder_override()},
+                "archival_retrieval": {"encoder": _text_encoder_override()},
+                "trigger": {"LLM_config": _openai_llm_override()},
+            },
+            "store": {
+                "flush_checker": _word_mode_truncation(),
+                "summarizer": {"LLM_config": _openai_llm_override()},
+            },
+        }
+    if baseline_name == "RFMemory":
+        overrides.setdefault("optimize", {})
+        overrides["optimize"] = {
+            "reflector": {"LLM_config": _openai_llm_override()},
+        }
+
+    # --- multimodal baselines ---
+    if baseline_name == "MMMemory":
+        overrides = {
+            "recall": {
+                "truncation": _word_mode_truncation(),
+            },
+        }
+    if baseline_name == "MMFUMemory":
+        overrides = {
+            "recall": {
+                "truncation": _word_mode_truncation(),
+            },
+        }
+    if baseline_name == "NGMemory":
+        overrides = {
+            "recall": {
+                "truncation": _word_mode_truncation(),
+            },
+        }
+    if baseline_name == "AUGUSTUSMemory":
+        overrides = {
+            "recall": {
+                "truncation": _word_mode_truncation(),
+            },
+            "concept_extractor": {
+                "llm": _openai_llm_override(),
+            },
+        }
+    if baseline_name == "UniversalRAGMemory":
+        overrides = {
+            "recall": {
+                "truncation": _word_mode_truncation(),
+                "text_retrieval": {"encoder": _text_encoder_override()},
+            },
+            "routing": {
+                "llm": _openai_llm_override(),
+            },
+        }
+    return overrides
+
+
+def _resolve_baselines_root() -> Path:
+    """Return the ``baselines/`` directory (sibling of eval_framework/).
+
+    Layout::
+
+        nips26/
+        ├── eval_framework/
+        └── baselines/
+            ├── memengine/
+            └── default_config/
+    """
+    # registry.py -> memory_adapters/ -> eval_framework/ -> nips26/
+    return Path(__file__).resolve().parents[2] / "baselines"
+
+
+def _ensure_memgallery_benchmark_on_path() -> Path:
+    """Add ``baselines/`` to sys.path so that ``memengine`` and
+    ``default_config`` packages are importable."""
+    baselines_root = _resolve_baselines_root()
+    if not (baselines_root / "memengine").is_dir():
+        raise FileNotFoundError(
+            f"memengine/ not found under {baselines_root}. "
+            f"Clone MemEngine into baselines/memengine."
+        )
+    s = str(baselines_root)
+    if s not in sys.path:
+        sys.path.insert(0, s)
+    _bootstrap_memengine_namespace(baselines_root)
+    return baselines_root
+
+
+def _bootstrap_memengine_namespace(root: Path) -> None:
+    """
+    Pre-seed lightweight namespace packages for the co-located memengine package.
+
+    memengine's package-level ``__init__.py`` eagerly imports all memories and function
+    modules, which pulls in heavyweight optional dependencies like ``torch`` even for
+    simple baselines such as ``FUMemory``. By registering package shells in ``sys.modules``
+    first, we can import only the specific submodules we need.
+
+    *root* is the ``our/`` directory that contains ``memengine/``.
+    """
+    package_paths = {
+        "memengine": root / "memengine",
+        "memengine.config": root / "memengine" / "config",
+        "memengine.memory": root / "memengine" / "memory",
+        "memengine.function": root / "memengine" / "function",
+        "memengine.operation": root / "memengine" / "operation",
+        "memengine.utils": root / "memengine" / "utils",
+    }
+    for package_name, package_path in package_paths.items():
+        existing = sys.modules.get(package_name)
+        if existing is not None:
+            continue
+        module = types.ModuleType(package_name)
+        module.__path__ = [str(package_path)]  # type: ignore[attr-defined]
+        module.__package__ = package_name
+        sys.modules[package_name] = module
+
+    for package_name in package_paths:
+        if "." not in package_name:
+            continue
+        parent_name, child_name = package_name.rsplit(".", 1)
+        parent = sys.modules.get(parent_name)
+        child = sys.modules.get(package_name)
+        if parent is not None and child is not None and not hasattr(parent, child_name):
+            setattr(parent, child_name, child)
+
+    _bootstrap_optional_dependency_stubs()
+    _populate_safe_memengine_function_exports()
+
+
+def _bootstrap_optional_dependency_stubs() -> None:
+    """Provide narrow stubs for optional imports needed only on unused code paths."""
+    if "torch" not in sys.modules:
+        try:
+            sys.modules["torch"] = importlib.import_module("torch")
+        except Exception:
+            pass
+    if "torch" not in sys.modules:
+        torch_module = types.ModuleType("torch")
+
+        def _torch_unavailable(*args: Any, **kwargs: Any) -> Any:
+            del args, kwargs
+            raise RuntimeError(
+                "PyTorch is required for encoder-backed or tensor-based Mem-Gallery "
+                "baselines, but `torch` is not installed in this environment."
+            )
+
+        torch_module.cuda = types.SimpleNamespace(is_available=lambda: False)  # type: ignore[attr-defined]
+        torch_module.device = lambda spec: spec  # type: ignore[attr-defined]
+        torch_module.no_grad = lambda: nullcontext()  # type: ignore[attr-defined]
+        torch_module.from_numpy = _torch_unavailable  # type: ignore[attr-defined]
+        torch_module.stack = _torch_unavailable  # type: ignore[attr-defined]
+        torch_module.sort = _torch_unavailable  # type: ignore[attr-defined]
+        torch_module.matmul = _torch_unavailable  # type: ignore[attr-defined]
+        torch_module.ones = _torch_unavailable  # type: ignore[attr-defined]
+        torch_module.nn = types.SimpleNamespace(  # type: ignore[attr-defined]
+            functional=types.SimpleNamespace(normalize=_torch_unavailable)
+        )
+        sys.modules["torch"] = torch_module
+
+    if "transformers" not in sys.modules:
+        try:
+            sys.modules["transformers"] = importlib.import_module("transformers")
+        except Exception:
+            pass
+    if "transformers" not in sys.modules:
+        transformers_module = types.ModuleType("transformers")
+
+        class _UnavailableAutoTokenizer:
+            @classmethod
+            def from_pretrained(cls, *args: Any, **kwargs: Any) -> Any:
+                del args, kwargs
+                raise RuntimeError(
+                    "transformers.AutoTokenizer is required for token-mode truncation "
+                    "or encoder-backed baselines, but `transformers` is not installed."
+                )
+
+        transformers_module.AutoTokenizer = _UnavailableAutoTokenizer  # type: ignore[attr-defined]
+        sys.modules["transformers"] = transformers_module
+
+
+def _populate_safe_memengine_function_exports() -> None:
+    """Expose all function symbols for complete baseline deployment without running package __init__."""
+    function_pkg = sys.modules.get("memengine.function")
+    if function_pkg is None:
+        return
+
+    # Complete list — covers every module referenced by any of the 11 baselines:
+    #   FU/ST/LT/GA/MG/RF (text-only) + MM/MMFU/NG/AUGUSTUS/UniversalRAG (multimodal)
+    for module_name in (
+        # --- text-only baselines ---
+        "memengine.function.Encoder",
+        "memengine.function.Retrieval",
+        "memengine.function.LLM",
+        "memengine.function.Judge",
+        "memengine.function.Reflector",
+        "memengine.function.Summarizer",
+        "memengine.function.Truncation",
+        "memengine.function.Trigger",
+        "memengine.function.Utilization",
+        "memengine.function.Forget",
+        # --- multimodal / graph / concept baselines ---
+        "memengine.function.MultiModalEncoder",
+        "memengine.function.MultiModalRetrieval",
+        "memengine.function.ConceptExtractor",
+        "memengine.function.ConceptBasedRetrieval",
+        "memengine.function.EntityExtractor",
+        "memengine.function.FactExtractor",
+        "memengine.function.UniversalRAGRouting",
+        "memengine.function.UniversalRAGRetrieval",
+    ):
+        try:
+            module = importlib.import_module(module_name)
+        except Exception:
+            # Some modules may depend on optional heavy deps (torch, transformers).
+            # Skip gracefully — they will fail loudly if the baseline actually needs them.
+            continue
+        for attr_name, value in vars(module).items():
+            if attr_name.startswith("_"):
+                continue
+            if not hasattr(function_pkg, attr_name):
+                setattr(function_pkg, attr_name, value)
+
+
+def create_memgallery_adapter(
+    baseline_name: str,
+    *,
+    config_overrides: dict[str, Any] | None = None,
+) -> MemGalleryNativeAdapter:
+    """
+    Instantiate a native Mem-Gallery adapter for a known baseline name.
+
+    Loads default_config + memengine from the Mem-Gallery benchmark tree.
+    """
+    if baseline_name not in MEMGALLERY_NATIVE_BASELINES:
+        raise KeyError(f"unknown Mem-Gallery baseline: {baseline_name!r}")
+    _ensure_memgallery_benchmark_on_path()
+    runtime_overrides = _default_memgallery_runtime_overrides(baseline_name)
+    if config_overrides:
+        runtime_overrides = {
+            **runtime_overrides,
+            **config_overrides,
+        }
+    return MemGalleryNativeAdapter.from_baseline(
+        baseline_name, config=runtime_overrides or None
+    )
+
+
+MEMGALLERY_NATIVE_REGISTRY: dict[str, Callable[..., MemGalleryNativeAdapter]] = {
+    name: partial(create_memgallery_adapter, name) for name in MEMGALLERY_NATIVE_BASELINES
+}
+
+EXTERNAL_ADAPTER_KEYS: frozenset[str] = frozenset({
+    "A-Mem", "MemoryOS", "Dummy",
+    "Mem0", "Mem0-Graph",
+    "SimpleMem", "Omni-SimpleMem",
+    "MemVerse",
+    "Zep",
+})
+
+
+def create_amem_adapter(**kwargs: Any) -> MemoryAdapter:
+    from eval_framework.memory_adapters.amem_v2 import AMemV2Adapter
+    return AMemV2Adapter(**kwargs)
+
+
+def create_memoryos_adapter(**kwargs: Any) -> MemoryOSAdapter:
+    return MemoryOSAdapter(**kwargs)
+
+
+def create_dummy_adapter(**kwargs: Any) -> DummyAdapter:
+    return DummyAdapter()
+
+
+def create_mem0_adapter(**kwargs: Any) -> MemoryAdapter:
+    from eval_framework.memory_adapters.mem0_adapter import Mem0Adapter
+    return Mem0Adapter(use_graph=False, **kwargs)
+
+
+def create_mem0_graph_adapter(**kwargs: Any) -> MemoryAdapter:
+    from eval_framework.memory_adapters.mem0_adapter import Mem0Adapter
+    return Mem0Adapter(use_graph=True, **kwargs)
+
+
+def create_simplemem_adapter(**kwargs: Any) -> MemoryAdapter:
+    from eval_framework.memory_adapters.simplemem_adapter import SimpleMemAdapter
+    return SimpleMemAdapter(mode="text", **kwargs)
+
+
+def create_omni_simplemem_adapter(**kwargs: Any) -> MemoryAdapter:
+    from eval_framework.memory_adapters.simplemem_adapter import SimpleMemAdapter
+    return SimpleMemAdapter(mode="omni", **kwargs)
+
+
+def create_memverse_adapter(**kwargs: Any) -> MemoryAdapter:
+    from eval_framework.memory_adapters.memverse_adapter import MemVerseAdapter
+    return MemVerseAdapter(**kwargs)
+
+
+def create_zep_adapter(**kwargs: Any) -> MemoryAdapter:
+    from eval_framework.memory_adapters.zep_adapter import ZepAdapter
+    return ZepAdapter(**kwargs)
+
+
+EXTERNAL_ADAPTER_REGISTRY: dict[str, Callable[..., MemoryAdapter]] = {
+    "A-Mem": create_amem_adapter,
+    "MemoryOS": create_memoryos_adapter,
+    "Dummy": create_dummy_adapter,
+    "Mem0": create_mem0_adapter,
+    "Mem0-Graph": create_mem0_graph_adapter,
+    "SimpleMem": create_simplemem_adapter,
+    "Omni-SimpleMem": create_omni_simplemem_adapter,
+    "MemVerse": create_memverse_adapter,
+    "Zep": create_zep_adapter,
+}
+
+
+def create_external_adapter(
+    name: str,
+    *,
+    config_overrides: dict[str, Any] | None = None,
+) -> MemoryAdapter:
+    """Instantiate an external adapter for a known baseline name."""
+    if name not in EXTERNAL_ADAPTER_KEYS:
+        raise KeyError(f"unknown external adapter: {name!r}")
+    return EXTERNAL_ADAPTER_REGISTRY[name](**(config_overrides or {}))

memory_adapters/simplemem_adapter.py

@@ -0,0 +1,156 @@
+"""Adapter for SimpleMem and Omni-SimpleMem baselines."""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+
+_DEFAULT_SOURCE = Path("/data1/toby/nips26/baselines/SimpleMem")
+
+
+class SimpleMemAdapter(MemoryAdapter):
+    """Adapter for SimpleMem (text mode) or Omni-SimpleMem (omni mode)."""
+
+    def __init__(
+        self,
+        *,
+        mode: str = "text",
+        source_root: str | os.PathLike[str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self._mode = mode  # "text" or "omni"
+        root = Path(source_root or _DEFAULT_SOURCE).resolve()
+        if str(root) not in sys.path:
+            sys.path.insert(0, str(root))
+
+        import simplemem_router as simplemem
+        self._simplemem = simplemem
+        self._mem: Any = None
+        self._session_id = ""
+        self._prev_snapshot_ids: set[str] = set()
+        self._stored_texts: list[dict[str, str]] = []
+        self._init_mem()
+
+    def _init_mem(self) -> None:
+        self._mem = self._simplemem.create(mode=self._mode, clear_db=True)
+        self._stored_texts = []
+
+    def reset(self) -> None:
+        if self._mem is not None:
+            try:
+                self._mem.close()
+            except Exception:
+                pass
+        self._init_mem()
+        self._prev_snapshot_ids = set()
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        self._session_id = turn.session_id
+        text = f"{turn.role}: {turn.text}"
+        for att in turn.attachments:
+            text += f"\n[{att.type}] {att.caption}"
+
+        mid = str(len(self._stored_texts))
+        if self._mode == "omni":
+            self._mem.add_text(text, tags=[f"session:{turn.session_id}"])
+        else:
+            speaker = "User" if turn.role == "user" else "Assistant"
+            ts = turn.timestamp or ""
+            self._mem.add_dialogue(speaker, text, ts)
+        self._stored_texts.append({"id": mid, "text": text, "session_id": turn.session_id})
+
+    def end_session(self, session_id: str) -> None:
+        self._session_id = session_id
+        if self._mode == "text":
+            try:
+                self._mem.finalize()
+            except Exception:
+                pass
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        return [
+            MemorySnapshotRecord(
+                memory_id=t["id"], text=t["text"],
+                session_id=t["session_id"], status="active",
+                source=f"SimpleMem-{self._mode}",
+                raw_backend_id=t["id"], raw_backend_type="simplemem",
+                metadata={},
+            )
+            for t in self._stored_texts
+        ]
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        current = self.snapshot_memories()
+        current_ids = {s.memory_id for s in current}
+        deltas = [
+            MemoryDeltaRecord(
+                session_id=session_id, op="add", text=s.text,
+                linked_previous=(), raw_backend_id=s.raw_backend_id,
+                metadata={"baseline": f"SimpleMem-{self._mode}"},
+            )
+            for s in current if s.memory_id not in self._prev_snapshot_ids
+        ]
+        self._prev_snapshot_ids = current_ids
+        return deltas
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        items: list[RetrievalItem] = []
+        try:
+            if self._mode == "omni":
+                result = self._mem.query(query, top_k=top_k)
+                if isinstance(result, list):
+                    for i, r in enumerate(result[:top_k]):
+                        text = r.get("text", str(r)) if isinstance(r, dict) else str(r)
+                        items.append(RetrievalItem(
+                            rank=i, memory_id=str(i), text=text,
+                            score=1.0 / (i + 1), raw_backend_id=None,
+                        ))
+            else:
+                answer = self._mem.ask(query)
+                if answer:
+                    items.append(RetrievalItem(
+                        rank=0, memory_id="answer", text=str(answer),
+                        score=1.0, raw_backend_id=None,
+                    ))
+        except Exception:
+            pass
+
+        if not items:
+            # Fallback: simple text search over stored memories
+            query_lower = query.lower()
+            scored = []
+            for t in self._stored_texts:
+                overlap = len(set(query_lower.split()) & set(t["text"].lower().split()))
+                scored.append((overlap, t))
+            scored.sort(key=lambda x: x[0], reverse=True)
+            for i, (sc, t) in enumerate(scored[:top_k]):
+                items.append(RetrievalItem(
+                    rank=i, memory_id=t["id"], text=t["text"],
+                    score=float(sc) / max(len(query.split()), 1),
+                    raw_backend_id=t["id"],
+                ))
+
+        return RetrievalRecord(
+            query=query, top_k=top_k, items=items[:top_k],
+            raw_trace={"baseline": f"SimpleMem-{self._mode}"},
+        )
+
+    def get_capabilities(self) -> dict[str, Any]:
+        name = "Omni-SimpleMem" if self._mode == "omni" else "SimpleMem"
+        return {
+            "backend": name, "baseline": name,
+            "available": self._mem is not None,
+            "delta_granularity": "per_turn",
+            "snapshot_mode": "full",
+        }

memory_adapters/zep_adapter.py

@@ -0,0 +1,122 @@
+"""Adapter for Zep memory system (community/self-hosted edition)."""
+
+from __future__ import annotations
+
+import os
+import uuid as _uuid
+from typing import Any
+
+from eval_framework.datasets.schemas import (
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+
+
+class ZepAdapter(MemoryAdapter):
+    """Adapter for Zep community edition (self-hosted)."""
+
+    def __init__(self, *, base_url: str | None = None, **kwargs: Any) -> None:
+        from zep_python import ZepClient
+
+        self._base_url = base_url or os.getenv("ZEP_BASE_URL", "http://localhost:8000")
+        self._client = ZepClient(base_url=self._base_url)
+        self._session_id = ""
+        self._thread_id = f"eval_{_uuid.uuid4().hex[:8]}"
+        self._prev_snapshot_ids: set[str] = set()
+
+    def reset(self) -> None:
+        try:
+            self._client.memory.delete_memory(self._thread_id)
+        except Exception:
+            pass
+        self._thread_id = f"eval_{_uuid.uuid4().hex[:8]}"
+        self._prev_snapshot_ids = set()
+
+    def ingest_turn(self, turn: NormalizedTurn) -> None:
+        from zep_python.memory import Memory
+        from zep_python.message import Message
+
+        self._session_id = turn.session_id
+        text = f"{turn.role}: {turn.text}"
+        for att in turn.attachments:
+            text += f"\n[{att.type}] {att.caption}"
+
+        role_type = "user" if turn.role == "user" else "ai"
+        msg = Message(role=turn.role, role_type=role_type, content=text)
+        memory = Memory(messages=[msg])
+        self._client.memory.add_memory(self._thread_id, memory)
+
+    def end_session(self, session_id: str) -> None:
+        self._session_id = session_id
+
+    def snapshot_memories(self) -> list[MemorySnapshotRecord]:
+        try:
+            memory = self._client.memory.get_memory(self._thread_id)
+        except Exception:
+            return []
+
+        rows: list[MemorySnapshotRecord] = []
+        if memory and memory.messages:
+            for i, msg in enumerate(memory.messages):
+                mid = str(getattr(msg, "uuid", i))
+                rows.append(MemorySnapshotRecord(
+                    memory_id=mid,
+                    text=msg.content or "",
+                    session_id=self._session_id,
+                    status="active",
+                    source="Zep",
+                    raw_backend_id=mid,
+                    raw_backend_type="zep_message",
+                    metadata={},
+                ))
+        return rows
+
+    def export_memory_delta(self, session_id: str) -> list[MemoryDeltaRecord]:
+        current = self.snapshot_memories()
+        current_ids = {s.memory_id for s in current}
+        deltas = [
+            MemoryDeltaRecord(
+                session_id=session_id, op="add", text=s.text,
+                linked_previous=(), raw_backend_id=s.raw_backend_id,
+                metadata={"baseline": "Zep"},
+            )
+            for s in current if s.memory_id not in self._prev_snapshot_ids
+        ]
+        self._prev_snapshot_ids = current_ids
+        return deltas
+
+    def retrieve(self, query: str, top_k: int) -> RetrievalRecord:
+        try:
+            results = self._client.memory.search_memory(
+                self._thread_id, query, limit=top_k,
+            )
+        except Exception:
+            results = []
+
+        items = [
+            RetrievalItem(
+                rank=i,
+                memory_id=str(getattr(r.message, "uuid", i)) if r.message else str(i),
+                text=r.message.content if r.message else str(r),
+                score=float(getattr(r, "score", 1.0 / (i + 1))),
+                raw_backend_id=str(getattr(r.message, "uuid", "")) if r.message else None,
+            )
+            for i, r in enumerate(results[:top_k])
+        ]
+        return RetrievalRecord(
+            query=query, top_k=top_k, items=items,
+            raw_trace={"baseline": "Zep"},
+        )
+
+    def get_capabilities(self) -> dict[str, Any]:
+        return {
+            "backend": "Zep",
+            "baseline": "Zep",
+            "available": True,
+            "delta_granularity": "snapshot_diff",
+            "snapshot_mode": "full",
+        }

openai_compat.py

@@ -0,0 +1,49 @@
+"""Compatibility helpers for OpenAI chat completions across model families."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def rewrite_chat_completion_kwargs(payload: dict[str, Any]) -> dict[str, Any]:
+    """Translate deprecated chat completion parameters for reasoning models."""
+    rewritten = dict(payload)
+    model = str(rewritten.get("model") or "")
+    if (
+        model.startswith("gpt-5")
+        and "max_tokens" in rewritten
+        and "max_completion_tokens" not in rewritten
+    ):
+        rewritten["max_completion_tokens"] = rewritten.pop("max_tokens")
+    return rewritten
+
+
+def patch_openai_chat_completions() -> bool:
+    """Monkeypatch the OpenAI SDK so GPT-5 chat calls accept legacy max_tokens."""
+    try:
+        from openai.resources.chat.completions.completions import Completions
+    except Exception:
+        return False
+
+    current = Completions.create
+    if getattr(current, "_eval_framework_patched", False):
+        return True
+
+    original_create = current
+
+    def _patched_create(self: Any, *args: Any, **kwargs: Any) -> Any:
+        rewritten = rewrite_chat_completion_kwargs(kwargs)
+        try:
+            return original_create(self, *args, **rewritten)
+        except Exception as exc:
+            if (
+                "Unsupported parameter: 'max_tokens'" in str(exc)
+                and "max_tokens" in kwargs
+            ):
+                retried = rewrite_chat_completion_kwargs(kwargs)
+                return original_create(self, *args, **retried)
+            raise
+
+    _patched_create._eval_framework_patched = True  # type: ignore[attr-defined]
+    Completions.create = _patched_create  # type: ignore[assignment]
+    return True

pipeline/__init__.py

@@ -0,0 +1 @@
+"""Session and checkpoint orchestration."""

pipeline/gold_state.py

@@ -0,0 +1,130 @@
+"""Cumulative gold memory state from staged memory-point annotations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Mapping, Sequence
+
+
+@dataclass(frozen=True)
+class GoldMemoryPoint:
+    memory_id: str
+    memory_content: str
+    memory_type: str
+    memory_source: str
+    is_update: bool
+    original_memories: tuple[str, ...]
+    importance: float
+    timestamp: str | None = None
+    update_type: str = ""
+
+
+@dataclass(frozen=True)
+class SessionGoldState:
+    session_id: str
+    cumulative_gold_memories: tuple[GoldMemoryPoint, ...]
+    session_new_memories: tuple[GoldMemoryPoint, ...]
+    session_update_memories: tuple[GoldMemoryPoint, ...]
+    session_interference_memories: tuple[GoldMemoryPoint, ...]
+
+
+def _as_str_tuple(val: Any) -> tuple[str, ...]:
+    if val is None:
+        return ()
+    if isinstance(val, str):
+        return (val,)
+    if isinstance(val, Sequence) and not isinstance(val, (str, bytes)):
+        out: list[str] = []
+        for x in val:
+            out.append(x if isinstance(x, str) else str(x))
+        return tuple(out)
+    return (str(val),)
+
+
+def _parse_is_update(raw: Mapping[str, Any]) -> bool:
+    v = raw.get("is_update")
+    if v is True:
+        return True
+    if v is False or v is None:
+        return False
+    if isinstance(v, str):
+        return v.strip().lower() == "true"
+    return bool(v)
+
+
+def _parse_importance(raw: Mapping[str, Any]) -> float:
+    value = raw.get("importance", 0.0)
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def gold_point_from_raw(raw: Mapping[str, Any]) -> GoldMemoryPoint:
+    mid = raw.get("memory_id")
+    content = raw.get("memory_content")
+    return GoldMemoryPoint(
+        memory_id=str(mid) if mid is not None else "",
+        memory_content=str(content) if content is not None else "",
+        memory_type=str(raw.get("memory_type", "")),
+        memory_source=str(raw.get("memory_source", "")),
+        is_update=_parse_is_update(raw),
+        original_memories=_as_str_tuple(raw.get("original_memories")),
+        importance=_parse_importance(raw),
+        timestamp=(
+            str(raw["timestamp"])
+            if raw.get("timestamp") is not None
+            else None
+        ),
+        update_type=str(raw.get("update_type", "") or ""),
+    )
+
+
+def build_session_gold_states(
+    ordered_session_ids: Sequence[str],
+    *,
+    s00_memory_points: Sequence[Mapping[str, Any]],
+    stage4_by_session_id: Mapping[str, Sequence[Mapping[str, Any]]],
+) -> tuple[SessionGoldState, ...]:
+    """Accumulate non-interference gold memories through sessions in order.
+
+    S00 is taken from the domain JSON session; later sessions prefer ``stage4``
+    rows when present, since those drive staged evaluation labels.
+    """
+    cumulative: list[GoldMemoryPoint] = []
+    states: list[SessionGoldState] = []
+
+    for sid in ordered_session_ids:
+        if sid == "S00":
+            raw_points: Sequence[Mapping[str, Any]] = s00_memory_points
+        else:
+            raw_points = stage4_by_session_id.get(sid)
+            if raw_points is None:
+                raw_points = ()
+
+        news: list[GoldMemoryPoint] = []
+        updates: list[GoldMemoryPoint] = []
+        interference: list[GoldMemoryPoint] = []
+
+        for raw in raw_points:
+            gp = gold_point_from_raw(raw)
+            if gp.memory_source == "interference":
+                interference.append(gp)
+                continue
+            cumulative.append(gp)
+            if gp.is_update:
+                updates.append(gp)
+            else:
+                news.append(gp)
+
+        states.append(
+            SessionGoldState(
+                session_id=sid,
+                cumulative_gold_memories=tuple(cumulative),
+                session_new_memories=tuple(news),
+                session_update_memories=tuple(updates),
+                session_interference_memories=tuple(interference),
+            )
+        )
+
+    return tuple(states)

pipeline/qa_runner.py

@@ -0,0 +1,59 @@
+"""Shared checkpoint QA: retrieval via adapter + answer from an injected callable.
+
+``AnswerFn`` may return either a plain ``str`` (legacy) or a
+``(str, list[str])`` tuple of ``(answer, cited_memories)``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Union
+
+from eval_framework.datasets.domain_a_v2 import NormalizedCheckpoint, NormalizedCheckpointQuestion
+from eval_framework.datasets.schemas import RetrievalRecord
+from eval_framework.memory_adapters.base import MemoryAdapter
+from eval_framework.pipeline.records import PipelineCheckpointQARecord
+
+# answer_fn may return str (legacy) or (str, list[str])
+AnswerResult = Union[str, tuple[str, list[str]]]
+AnswerFn = Callable[[NormalizedCheckpointQuestion, RetrievalRecord], AnswerResult]
+
+
+def run_checkpoint_qa_records(
+    adapter: MemoryAdapter,
+    *,
+    sample_id: str,
+    sample_uuid: str,
+    checkpoint: NormalizedCheckpoint,
+    top_k: int,
+    answer_fn: AnswerFn,
+) -> tuple[PipelineCheckpointQARecord, ...]:
+    """For each question, call ``retrieve`` then ``answer_fn`` (not ``adapter.answer``)."""
+    out: list[PipelineCheckpointQARecord] = []
+    for q in checkpoint.questions:
+        retrieval = adapter.retrieve(q.question, top_k)
+        result = answer_fn(q, retrieval)
+
+        if isinstance(result, tuple):
+            generated, cited = result
+        else:
+            generated, cited = result, []
+
+        out.append(
+            PipelineCheckpointQARecord(
+                sample_id=sample_id,
+                sample_uuid=sample_uuid,
+                checkpoint_id=checkpoint.checkpoint_id,
+                question=q.question,
+                gold_answer=q.gold_answer,
+                gold_evidence_memory_ids=q.gold_evidence_memory_ids,
+                gold_evidence_contents=q.gold_evidence_contents,
+                question_type=q.question_type,
+                question_type_abbrev=q.question_type_abbrev,
+                difficulty=q.difficulty,
+                retrieval=retrieval,
+                generated_answer=generated,
+                cited_memories=tuple(cited),
+            )
+        )
+    return tuple(out)

pipeline/records.py

@@ -0,0 +1,60 @@
+"""Pipeline-facing aliases and runtime record types emitted by the eval runner."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from eval_framework.datasets.schemas import (
+    Attachment,
+    MemoryDeltaRecord,
+    MemorySnapshotRecord,
+    NormalizedTurn,
+    RetrievalItem,
+    RetrievalRecord,
+    normalize_turn,
+)
+from eval_framework.pipeline.gold_state import SessionGoldState
+
+__all__ = [
+    "Attachment",
+    "MemoryDeltaRecord",
+    "MemorySnapshotRecord",
+    "NormalizedTurn",
+    "PipelineCheckpointQARecord",
+    "PipelineSessionRecord",
+    "RetrievalItem",
+    "RetrievalRecord",
+    "SessionGoldState",
+    "normalize_turn",
+]
+
+
+@dataclass(frozen=True)
+class PipelineSessionRecord:
+    """Normalized outputs after one dialogue session (one row per session)."""
+
+    sample_id: str
+    sample_uuid: str
+    session_id: str
+    memory_snapshot: tuple[MemorySnapshotRecord, ...]
+    memory_delta: tuple[MemoryDeltaRecord, ...]
+    gold_state: SessionGoldState
+
+
+@dataclass(frozen=True)
+class PipelineCheckpointQARecord:
+    """One checkpoint question: retrieval trace plus model-generated answer."""
+
+    sample_id: str
+    sample_uuid: str
+    checkpoint_id: str
+    question: str
+    gold_answer: str
+    gold_evidence_memory_ids: tuple[str, ...]
+    gold_evidence_contents: tuple[str, ...]
+    question_type: str
+    question_type_abbrev: str
+    difficulty: str
+    retrieval: RetrievalRecord
+    generated_answer: str
+    cited_memories: tuple[str, ...] = ()

pipeline/runner.py

@@ -0,0 +1,104 @@
+"""Session-by-session ingest, memory export, and checkpoint QA orchestration."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from eval_framework.datasets.domain_a_v2 import (
+    DomainAV2AcademicSample,
+    NormalizedCheckpointQuestion,
+)
+from eval_framework.memory_adapters.base import MemoryAdapter
+from eval_framework.pipeline.qa_runner import run_checkpoint_qa_records
+from eval_framework.pipeline.records import PipelineCheckpointQARecord, PipelineSessionRecord
+from eval_framework.datasets.schemas import RetrievalRecord
+
+
+def ensure_adapter_available(adapter: MemoryAdapter) -> None:
+    caps = adapter.get_capabilities()
+    if caps.get("available") is False:
+        backend = caps.get("backend", type(adapter).__name__)
+        detail = caps.get("integration_error") or caps.get(
+            "integration_status", "available=False"
+        )
+        raise RuntimeError(
+            f"Memory adapter {backend!r} is not available for pipeline runs: {detail}"
+        )
+
+
+def run_domain_a_v2_sample(
+    adapter: MemoryAdapter,
+    sample: DomainAV2AcademicSample,
+    *,
+    top_k: int = 5,
+    answer_fn: Callable | None = None,
+) -> tuple[tuple[PipelineSessionRecord, ...], tuple[PipelineCheckpointQARecord, ...]]:
+    """Run all sessions in order, emit one session record per session, then checkpoint QA when due."""
+    ensure_adapter_available(adapter)
+    if sample.normalized_checkpoints and answer_fn is None:
+        raise ValueError(
+            "answer_fn is required when the sample defines normalized checkpoints"
+        )
+
+    adapter.reset()
+    session_out: list[PipelineSessionRecord] = []
+    qa_out: list[PipelineCheckpointQARecord] = []
+    completed_sessions: set[str] = set()
+    session_order = {
+        session.session_id: index for index, session in enumerate(sample.sessions)
+    }
+
+    if len(sample.sessions) != len(sample.session_gold_states):
+        raise ValueError(
+            "sample.sessions and sample.session_gold_states length mismatch"
+        )
+
+    for sess, gold in zip(sample.sessions, sample.session_gold_states):
+        if sess.session_id != gold.session_id:
+            raise ValueError(
+                f"session / gold_state id mismatch: {sess.session_id!r} vs {gold.session_id!r}"
+            )
+        for turn in sess.turns:
+            adapter.ingest_turn(turn)
+        adapter.end_session(sess.session_id)
+
+        snapshot = tuple(adapter.snapshot_memories())
+        delta = tuple(adapter.export_memory_delta(sess.session_id))
+        session_out.append(
+            PipelineSessionRecord(
+                sample_id=sample.sample_id,
+                sample_uuid=sample.uuid,
+                session_id=sess.session_id,
+                memory_snapshot=snapshot,
+                memory_delta=delta,
+                gold_state=gold,
+            )
+        )
+        completed_sessions.add(sess.session_id)
+
+        for cp in sample.normalized_checkpoints:
+            covered = cp.covered_sessions
+            if not covered:
+                continue
+            missing = [sid for sid in covered if sid not in session_order]
+            if missing:
+                raise ValueError(
+                    f"checkpoint {cp.checkpoint_id!r} references unknown sessions: {missing}"
+                )
+            if not set(covered).issubset(completed_sessions):
+                continue
+            trigger_session_id = max(covered, key=session_order.__getitem__)
+            if sess.session_id != trigger_session_id:
+                continue
+            qa_out.extend(
+                run_checkpoint_qa_records(
+                    adapter,
+                    sample_id=sample.sample_id,
+                    sample_uuid=sample.uuid,
+                    checkpoint=cp,
+                    top_k=top_k,
+                    answer_fn=answer_fn,
+                )
+            )
+
+    return tuple(session_out), tuple(qa_out)