Add files using upload-large-folder tool

.gitattributes

@@ -6,3 +6,4 @@
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 ChatUniVi/eval/questions/scienceqa/problems.json filter=lfs diff=lfs merge=lfs -text
+runs/tubetoken_phase0/proposals_stride8_n64_bidir/lYwnXP3g050_4000_14000.npz filter=lfs diff=lfs merge=lfs -text

configs/config.py

@@ -53,6 +53,7 @@ parser.add_argument("--log_root",type=str,default='log', help="where to save log
 parser.add_argument("--checkpoint_root",type=str,default='checkpoints', help="where to save trained checkpoints during training")
 
 parser.add_argument("--visualization_root",type=str,default='visualization', help="where to save visualization result during test")
+parser.add_argument("--eval_splits",type=str,default='test_s,test_u,test_n', help="comma-separated eval splits for load_model.py: test_s,test_u,test_n")
 
 
 

runs/tubetoken_phase0/proposals_stride8_n64_bidir/lYwnXP3g050_4000_14000.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f09a2efd79256574c47624d11d5ae5ccff4e267abb961dc63375d862c8db958
+size 1418418

runs/tubetoken_phase_minus1/audit_full.log

@@ -0,0 +1,47 @@
+{
+  "area_unstable_expressions": 41,
+  "audio_keyword_expressions": 15890,
+  "audio_keyword_percent": 77.66753018231586,
+  "data_dir": "/workspace/SimToken/data",
+  "expressions_per_object": {
+    "ge2": 5836,
+    "ge3": 4206,
+    "max": 10,
+    "mean": 2.742125720412813,
+    "median": 3
+  },
+  "expressions_per_video": {
+    "ge2": 3521,
+    "ge3": 3381,
+    "max": 26,
+    "mean": 5.7243984331281474,
+    "median": 6.0
+  },
+  "h3_candidate_expressions": 18614,
+  "h3_candidate_objects": 5781,
+  "late_target_expressions": 0,
+  "mask_rows_audited": 20459,
+  "multi_expression_objects": 5836,
+  "multi_expression_videos": 3521,
+  "null_split_expressions": 1028,
+  "null_split_percent": 5.0246835133682,
+  "num_expressions": 20459,
+  "num_objects_vid_fid": 7461,
+  "num_videos": 3574,
+  "partial_target_expressions": 33,
+  "same_category_distractor_heuristic_expressions": 2563,
+  "same_category_distractor_heuristic_percent": 12.527494012415074,
+  "small_target_expressions": 10037,
+  "spatial_keyword_expressions": 5924,
+  "spatial_keyword_percent": 28.955471919448655,
+  "splits": {
+    "TODO": 25,
+    "test_n": 1028,
+    "test_s": 2288,
+    "test_u": 1656,
+    "train": 14113,
+    "val": 1349
+  }
+}
+
+Wrote audit files to: /workspace/SimToken/runs/tubetoken_phase_minus1/audit_full

runs/tubetoken_phase_minus1/audit_full/audit_report.md

@@ -0,0 +1,34 @@
+# TubeToken Phase -1 Audit
+
+- Expressions: 20459
+- Videos: 3574
+- Objects `(vid, fid)`: 7461
+- Splits: `{'val': 1349, 'train': 14113, 'test_s': 2288, 'TODO': 25, 'test_u': 1656, 'test_n': 1028}`
+
+## Multi-expression
+
+- Expressions/video mean: 5.724
+- Expressions/video median: 6.0
+- Videos with >=2 expressions: 3521
+- Expressions/object mean: 2.742
+- Objects with >=2 expressions: 5836
+- H3 candidate objects: 5781
+- H3 candidate expressions: 18614
+
+## Diagnostic Subsets
+
+- Null split expressions: 1028 (5.02%)
+- Audio-keyword expressions: 15890 (77.67%)
+- Spatial-keyword expressions: 5924 (28.96%)
+- Same-category distractor heuristic expressions: 2563 (12.53%)
+- Mask rows audited: 20459
+- Late-target expressions: 0
+- Small-target expressions: 10037
+- Partial-target expressions: 33
+- Area-unstable expressions: 41
+
+## Phase -1 H3 Decision Hint
+
+H3 can stay as a direct validation target: the data has multi-expression structure.
+
+Generated files: `audit_summary.json`, `audit_samples.csv`, `h3_candidates.csv`.

tools/audit_refavs.py

@@ -0,0 +1,371 @@
+#!/usr/bin/env python3
+"""Audit Ref-AVS style metadata for the TubeToken experiment plan.
+
+This script intentionally depends only on the dataset files. It does not import
+the training code, so it can run before model dependencies are fully settled.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import math
+import os
+from collections import Counter, defaultdict
+from pathlib import Path
+from statistics import mean, median
+from typing import Dict, Iterable, List, Optional, Tuple
+
+try:
+    from PIL import Image
+except Exception:  # pragma: no cover - only used as an environment fallback
+    Image = None
+
+
+AUDIO_KEYWORDS = (
+    "sound",
+    "sounding",
+    "making sound",
+    "longest sound",
+    "intermittent sound",
+    "silent",
+    "audio",
+    "heard",
+    "emitting",
+    "playing instrument",
+    "voice",
+    "speaking",
+    "talking",
+    "singing",
+    "barking",
+    "meowing",
+    "hitting",
+)
+
+SPATIAL_KEYWORDS = (
+    "left",
+    "right",
+    "top",
+    "bottom",
+    "front",
+    "back",
+    "behind",
+    "next to",
+    "near",
+    "far",
+    "middle",
+    "center",
+    "between",
+    "above",
+    "below",
+    "under",
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Audit Ref-AVS data for TubeToken Phase -1.")
+    parser.add_argument("--data_dir", type=Path, default=Path("data"))
+    parser.add_argument("--out_dir", type=Path, default=Path("runs/tubetoken_phase_minus1/audit"))
+    parser.add_argument("--frames", type=int, default=10)
+    parser.add_argument("--small_area", type=float, default=0.05)
+    parser.add_argument("--mask_sample_limit", type=int, default=0, help="0 means audit every row.")
+    return parser.parse_args()
+
+
+def read_metadata(path: Path) -> List[dict]:
+    with path.open("r", newline="") as f:
+        return list(csv.DictReader(f))
+
+
+def video_id(row: dict) -> str:
+    return row.get("vid") or row["uid"].rsplit("_", 2)[0]
+
+
+def fid_value(row: dict) -> str:
+    return str(row.get("fid", "")).strip()
+
+
+def object_key(row: dict) -> Tuple[str, str]:
+    return video_id(row), fid_value(row)
+
+
+def category_from_uid(row: dict) -> str:
+    vid = video_id(row)
+    uid = row.get("uid", "")
+    suffix = uid[len(vid) + 1 :] if uid.startswith(vid + "_") else uid.rsplit("_", 2)[-2]
+    if "_" in suffix:
+        return suffix.rsplit("_", 1)[0]
+    return suffix
+
+
+def has_any(text: str, keywords: Iterable[str]) -> bool:
+    text = text.lower()
+    return any(k in text for k in keywords)
+
+
+def mask_path(data_dir: Path, vid: str, fid: str, t: int) -> Path:
+    return data_dir / "gt_mask" / vid / f"fid_{fid}" / f"0000{t}.png"
+
+
+def read_binary_mask_stats(path: Path) -> Optional[Tuple[int, int, int]]:
+    if Image is None or not path.exists():
+        return None
+    with Image.open(path) as img:
+        gray = img.convert("L")
+        width, height = gray.size
+        hist = gray.histogram()
+        positive = sum(hist[1:])
+        return positive, width, height
+
+
+def row_mask_stats(data_dir: Path, row: dict, frames: int, small_area: float) -> dict:
+    vid = video_id(row)
+    fid = fid_value(row)
+    positives: List[int] = []
+    areas: List[float] = []
+    missing = 0
+    width = height = None
+
+    for t in range(frames):
+        stats = read_binary_mask_stats(mask_path(data_dir, vid, fid, t))
+        if stats is None:
+            missing += 1
+            positives.append(0)
+            areas.append(0.0)
+            continue
+        pos, width, height = stats
+        positives.append(pos)
+        denom = max(width * height, 1)
+        areas.append(pos / denom)
+
+    visible = [i for i, pos in enumerate(positives) if pos > 0]
+    visible_areas = [areas[i] for i in visible]
+    first_visible = min(visible) if visible else None
+    mean_visible_area = mean(visible_areas) if visible_areas else 0.0
+    mean_all_area = mean(areas) if areas else 0.0
+    area_cv = 0.0
+    if len(visible_areas) > 1 and mean_visible_area > 0:
+        var = sum((x - mean_visible_area) ** 2 for x in visible_areas) / len(visible_areas)
+        area_cv = math.sqrt(var) / mean_visible_area
+
+    return {
+        "visible_frames": len(visible),
+        "visible_ratio": len(visible) / frames,
+        "first_visible": first_visible,
+        "late_target": first_visible is not None and first_visible > 0.5 * frames,
+        "mean_visible_area": mean_visible_area,
+        "mean_all_area": mean_all_area,
+        "small_target": mean_visible_area > 0 and mean_visible_area < small_area,
+        "partial_target": 0 < len(visible) < 0.5 * frames,
+        "area_cv": area_cv,
+        "area_unstable": area_cv >= 1.0,
+        "missing_masks": missing,
+        "width": width,
+        "height": height,
+    }
+
+
+def pct(num: int, den: int) -> float:
+    return 0.0 if den == 0 else 100.0 * num / den
+
+
+def summarize_counts(values: List[int]) -> dict:
+    if not values:
+        return {"mean": 0, "median": 0, "max": 0, "ge2": 0, "ge3": 0}
+    return {
+        "mean": mean(values),
+        "median": median(values),
+        "max": max(values),
+        "ge2": sum(v >= 2 for v in values),
+        "ge3": sum(v >= 3 for v in values),
+    }
+
+
+def write_csv(path: Path, rows: List[dict], fieldnames: List[str]) -> None:
+    with path.open("w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for row in rows:
+            writer.writerow({k: row.get(k, "") for k in fieldnames})
+
+
+def main() -> None:
+    args = parse_args()
+    data_dir = args.data_dir
+    out_dir = args.out_dir
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    rows = read_metadata(data_dir / "metadata.csv")
+    if args.mask_sample_limit > 0:
+        mask_rows = rows[: args.mask_sample_limit]
+    else:
+        mask_rows = rows
+
+    by_split = Counter(row["split"] for row in rows)
+    by_video: Dict[str, List[dict]] = defaultdict(list)
+    by_object: Dict[Tuple[str, str], List[dict]] = defaultdict(list)
+    by_video_category: Dict[Tuple[str, str], set] = defaultdict(set)
+
+    enriched: List[dict] = []
+    for row in rows:
+        vid = video_id(row)
+        fid = fid_value(row)
+        category = category_from_uid(row)
+        expr = row.get("exp", "")
+        row2 = dict(row)
+        row2["vid"] = vid
+        row2["fid"] = fid
+        row2["category"] = category
+        row2["is_null_split"] = row.get("split") == "test_n"
+        row2["is_audio_keyword"] = has_any(expr, AUDIO_KEYWORDS)
+        row2["is_spatial_keyword"] = has_any(expr, SPATIAL_KEYWORDS)
+        by_video[vid].append(row2)
+        by_object[(vid, fid)].append(row2)
+        by_video_category[(vid, category)].add(fid)
+        enriched.append(row2)
+
+    mask_stats_by_uid: Dict[str, dict] = {}
+    for row in mask_rows:
+        uid = row["uid"]
+        mask_stats_by_uid[uid] = row_mask_stats(data_dir, row, args.frames, args.small_area)
+
+    sample_rows: List[dict] = []
+    for row in enriched:
+        stats = mask_stats_by_uid.get(row["uid"], {})
+        same_cat_fids = by_video_category[(row["vid"], row["category"])]
+        row2 = dict(row)
+        row2.update(stats)
+        row2["same_category_distractor_heuristic"] = len(same_cat_fids) >= 2
+        row2["multi_expr_video"] = len(by_video[row["vid"]]) >= 2
+        row2["multi_expr_object"] = len(by_object[(row["vid"], row["fid"])]) >= 2
+        row2["h3_candidate"] = row2["multi_expr_object"] and not row2["is_null_split"]
+        sample_rows.append(row2)
+
+    video_expr_counts = [len(v) for v in by_video.values()]
+    object_expr_counts = [len(v) for v in by_object.values()]
+    h3_objects = [k for k, v in by_object.items() if len(v) >= 2 and v[0]["split"] != "test_n"]
+    null_rows = [r for r in enriched if r["is_null_split"]]
+    audio_rows = [r for r in enriched if r["is_audio_keyword"]]
+    spatial_rows = [r for r in enriched if r["is_spatial_keyword"]]
+    same_cat_rows = [r for r in sample_rows if r.get("same_category_distractor_heuristic")]
+
+    audited_mask_rows = [r for r in sample_rows if "visible_ratio" in r]
+    late_rows = [r for r in audited_mask_rows if r.get("late_target")]
+    small_rows = [r for r in audited_mask_rows if r.get("small_target")]
+    partial_rows = [r for r in audited_mask_rows if r.get("partial_target")]
+    unstable_rows = [r for r in audited_mask_rows if r.get("area_unstable")]
+
+    summary = {
+        "data_dir": str(data_dir),
+        "num_expressions": len(rows),
+        "num_videos": len(by_video),
+        "num_objects_vid_fid": len(by_object),
+        "splits": dict(by_split),
+        "expressions_per_video": summarize_counts(video_expr_counts),
+        "expressions_per_object": summarize_counts(object_expr_counts),
+        "multi_expression_videos": sum(c >= 2 for c in video_expr_counts),
+        "multi_expression_objects": sum(c >= 2 for c in object_expr_counts),
+        "h3_candidate_objects": len(h3_objects),
+        "h3_candidate_expressions": sum(len(by_object[k]) for k in h3_objects),
+        "null_split_expressions": len(null_rows),
+        "null_split_percent": pct(len(null_rows), len(rows)),
+        "audio_keyword_expressions": len(audio_rows),
+        "audio_keyword_percent": pct(len(audio_rows), len(rows)),
+        "spatial_keyword_expressions": len(spatial_rows),
+        "spatial_keyword_percent": pct(len(spatial_rows), len(rows)),
+        "same_category_distractor_heuristic_expressions": len(same_cat_rows),
+        "same_category_distractor_heuristic_percent": pct(len(same_cat_rows), len(rows)),
+        "mask_rows_audited": len(audited_mask_rows),
+        "late_target_expressions": len(late_rows),
+        "small_target_expressions": len(small_rows),
+        "partial_target_expressions": len(partial_rows),
+        "area_unstable_expressions": len(unstable_rows),
+    }
+
+    with (out_dir / "audit_summary.json").open("w") as f:
+        json.dump(summary, f, indent=2, sort_keys=True)
+
+    fields = [
+        "uid",
+        "vid",
+        "split",
+        "fid",
+        "category",
+        "exp",
+        "is_null_split",
+        "is_audio_keyword",
+        "is_spatial_keyword",
+        "multi_expr_video",
+        "multi_expr_object",
+        "h3_candidate",
+        "same_category_distractor_heuristic",
+        "visible_frames",
+        "visible_ratio",
+        "first_visible",
+        "late_target",
+        "mean_visible_area",
+        "mean_all_area",
+        "small_target",
+        "partial_target",
+        "area_cv",
+        "area_unstable",
+        "missing_masks",
+        "width",
+        "height",
+    ]
+    write_csv(out_dir / "audit_samples.csv", sample_rows, fields)
+
+    h3_rows = [r for r in sample_rows if r.get("h3_candidate")]
+    write_csv(out_dir / "h3_candidates.csv", h3_rows, fields)
+
+    md = [
+        "# TubeToken Phase -1 Audit",
+        "",
+        f"- Expressions: {summary['num_expressions']}",
+        f"- Videos: {summary['num_videos']}",
+        f"- Objects `(vid, fid)`: {summary['num_objects_vid_fid']}",
+        f"- Splits: `{dict(by_split)}`",
+        "",
+        "## Multi-expression",
+        "",
+        f"- Expressions/video mean: {summary['expressions_per_video']['mean']:.3f}",
+        f"- Expressions/video median: {summary['expressions_per_video']['median']}",
+        f"- Videos with >=2 expressions: {summary['multi_expression_videos']}",
+        f"- Expressions/object mean: {summary['expressions_per_object']['mean']:.3f}",
+        f"- Objects with >=2 expressions: {summary['multi_expression_objects']}",
+        f"- H3 candidate objects: {summary['h3_candidate_objects']}",
+        f"- H3 candidate expressions: {summary['h3_candidate_expressions']}",
+        "",
+        "## Diagnostic Subsets",
+        "",
+        f"- Null split expressions: {summary['null_split_expressions']} ({summary['null_split_percent']:.2f}%)",
+        f"- Audio-keyword expressions: {summary['audio_keyword_expressions']} ({summary['audio_keyword_percent']:.2f}%)",
+        f"- Spatial-keyword expressions: {summary['spatial_keyword_expressions']} ({summary['spatial_keyword_percent']:.2f}%)",
+        f"- Same-category distractor heuristic expressions: {summary['same_category_distractor_heuristic_expressions']} ({summary['same_category_distractor_heuristic_percent']:.2f}%)",
+        f"- Mask rows audited: {summary['mask_rows_audited']}",
+        f"- Late-target expressions: {summary['late_target_expressions']}",
+        f"- Small-target expressions: {summary['small_target_expressions']}",
+        f"- Partial-target expressions: {summary['partial_target_expressions']}",
+        f"- Area-unstable expressions: {summary['area_unstable_expressions']}",
+        "",
+        "## Phase -1 H3 Decision Hint",
+        "",
+    ]
+    epv = summary["expressions_per_video"]["mean"]
+    if epv > 1.5 and summary["h3_candidate_objects"] > 0:
+        md.append("H3 can stay as a direct validation target: the data has multi-expression structure.")
+    elif summary["h3_candidate_objects"] > 0:
+        md.append("H3 should be treated as diagnostic: multi-expression objects exist, but average expressions/video is limited.")
+    else:
+        md.append("H3 should be downgraded: this audit did not find same-object multi-expression candidates.")
+    md.append("")
+    md.append("Generated files: `audit_summary.json`, `audit_samples.csv`, `h3_candidates.csv`.")
+
+    (out_dir / "audit_report.md").write_text("\n".join(md) + "\n")
+    print(json.dumps(summary, indent=2, sort_keys=True))
+    print(f"\nWrote audit files to: {out_dir}")
+
+
+if __name__ == "__main__":
+    main()

tools/tubetoken/evaluate_oracle_refine_sam2.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Evaluate bbox-only SAM2 refinement for oracle proposal tubes."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import numpy as np
+import torch
+from PIL import Image
+from tqdm import tqdm
+
+from phase0_common import (
+    bbox_from_mask,
+    bool_field,
+    evaluate_tube_jf,
+    load_audit_rows,
+    load_gt_tube,
+    read_metadata,
+    fid_value,
+    video_id,
+    write_json,
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Evaluate oracle proposal bbox-only SAM2 refinement.")
+    parser.add_argument("--data_dir", type=Path, required=True)
+    parser.add_argument("--proposal_dir", type=Path, required=True)
+    parser.add_argument("--phase0_samples", type=Path, required=True)
+    parser.add_argument("--out_dir", type=Path, required=True)
+    parser.add_argument("--audit_csv", type=Path, default=None)
+    parser.add_argument("--splits", type=str, default="test_s,test_u")
+    parser.add_argument("--sam2_repo", type=Path, default=None)
+    parser.add_argument("--model_cfg", type=str, default="configs/sam2.1/sam2.1_hiera_l.yaml")
+    parser.add_argument("--checkpoint", type=Path, required=True)
+    parser.add_argument("--device", type=str, default="cuda")
+    parser.add_argument("--frames", type=int, default=10)
+    parser.add_argument("--limit_samples", type=int, default=0)
+    return parser.parse_args()
+
+
+def import_sam2(repo: Optional[Path]):
+    if repo is not None:
+        sys.path.insert(0, str(repo))
+    from sam2.build_sam import build_sam2
+    from sam2.sam2_image_predictor import SAM2ImagePredictor
+
+    return build_sam2, SAM2ImagePredictor
+
+
+def load_rgb(path: Path) -> np.ndarray:
+    with Image.open(path) as img:
+        return np.array(img.convert("RGB"))
+
+
+def load_phase0_samples(path: Path) -> Dict[str, dict]:
+    with path.open("r", newline="") as f:
+        return {row["uid"]: row for row in csv.DictReader(f)}
+
+
+def nearest_box(tube: np.ndarray, t: int) -> Optional[np.ndarray]:
+    boxes = []
+    for idx in range(tube.shape[0]):
+        box = bbox_from_mask(tube[idx])
+        if box is not None:
+            boxes.append((idx, np.array(box, dtype=np.float32)))
+    if not boxes:
+        return None
+    _, box = min(boxes, key=lambda item: abs(item[0] - t))
+    return box
+
+
+def predict_box_mask(predictor, image: np.ndarray, box: np.ndarray) -> np.ndarray:
+    predictor.set_image(image)
+    masks, scores, _ = predictor.predict(box=box, multimask_output=False)
+    masks = np.asarray(masks)
+    if masks.ndim == 4:
+        masks = masks[0]
+    if masks.ndim == 3:
+        masks = masks[0]
+    return masks > 0
+
+
+def sample_subsets(row: dict, audit: Dict[str, dict]) -> List[str]:
+    out = ["all", row["split"]]
+    audit_row = audit.get(row["uid"])
+    for key, name in [
+        ("small_target", "small"),
+        ("partial_target", "partial"),
+        ("area_unstable", "area_unstable"),
+        ("late_target", "late_target"),
+        ("is_audio_keyword", "audio_keyword"),
+        ("is_spatial_keyword", "spatial_keyword"),
+        ("same_category_distractor_heuristic", "same_category"),
+        ("h3_candidate", "h3_candidate"),
+    ]:
+        if bool_field(audit_row, key):
+            out.append(name)
+    return out
+
+
+def empty_bucket() -> dict:
+    return {"count": 0, "refined_j": 0.0, "refined_f": 0.0, "refined_jf": 0.0}
+
+
+def add_bucket(bucket: dict, sample: dict) -> None:
+    bucket["count"] += 1
+    bucket["refined_j"] += sample["refined_j"]
+    bucket["refined_f"] += sample["refined_f"]
+    bucket["refined_jf"] += sample["refined_jf"]
+
+
+def finalize(bucket: dict) -> dict:
+    out = dict(bucket)
+    if bucket["count"]:
+        for key in ["refined_j", "refined_f", "refined_jf"]:
+            out[key] = bucket[key] / bucket["count"]
+    return out
+
+
+def main() -> None:
+    args = parse_args()
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    build_sam2, SAM2ImagePredictor = import_sam2(args.sam2_repo)
+    model = build_sam2(args.model_cfg, str(args.checkpoint), device=args.device)
+    predictor = SAM2ImagePredictor(model)
+
+    splits = [s.strip() for s in args.splits.split(",") if s.strip()]
+    rows = read_metadata(args.data_dir, splits)
+    if args.limit_samples:
+        rows = rows[: args.limit_samples]
+    phase0_samples = load_phase0_samples(args.phase0_samples)
+    audit = load_audit_rows(args.audit_csv) if args.audit_csv else {}
+
+    out_rows: List[dict] = []
+    summary = defaultdict(empty_bucket)
+
+    for row in tqdm(rows, desc="Oracle bbox-only SAM2 refinement"):
+        phase0 = phase0_samples[row["uid"]]
+        best_idx = int(phase0["best_idx"])
+        if best_idx < 0:
+            continue
+        vid = video_id(row)
+        proposals = np.load(args.proposal_dir / f"{vid}.npz")["masks"].astype(bool)
+        if best_idx >= proposals.shape[0]:
+            continue
+        oracle_tube = proposals[best_idx]
+        refined_masks = []
+
+        with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16, enabled=args.device.startswith("cuda")):
+            for t in range(args.frames):
+                box = bbox_from_mask(oracle_tube[t])
+                if box is None:
+                    box = nearest_box(oracle_tube, t)
+                if box is None:
+                    refined_masks.append(np.zeros_like(oracle_tube[t], dtype=bool))
+                    continue
+                image = load_rgb(args.data_dir / "media" / vid / "frames" / f"{t}.jpg")
+                refined_masks.append(predict_box_mask(predictor, image, np.asarray(box, dtype=np.float32)))
+
+        refined_tube = np.stack(refined_masks, axis=0)
+        gt = load_gt_tube(args.data_dir, vid, fid_value(row), args.frames)
+        j, f, jf = evaluate_tube_jf(refined_tube, gt)
+        sample = {
+            "uid": row["uid"],
+            "vid": vid,
+            "split": row["split"],
+            "fid": fid_value(row),
+            "best_idx": best_idx,
+            "refined_j": j,
+            "refined_f": f,
+            "refined_jf": jf,
+        }
+        out_rows.append(sample)
+        for subset in sample_subsets(row, audit):
+            add_bucket(summary[subset], sample)
+
+    with (args.out_dir / "sample_metrics.csv").open("w", newline="") as f:
+        fieldnames = list(out_rows[0].keys()) if out_rows else []
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(out_rows)
+
+    final_summary = {name: finalize(bucket) for name, bucket in sorted(summary.items())}
+    write_json(args.out_dir / "summary.json", final_summary)
+
+    md = ["# TubeToken Phase 0 Oracle Refined Evaluation", ""]
+    for name, metrics in final_summary.items():
+        if metrics["count"] == 0:
+            continue
+        md.append(f"- {name}: n={metrics['count']}, Refined J&F={metrics['refined_jf']:.4f}")
+    (args.out_dir / "report.md").write_text("\n".join(md) + "\n")
+    print("\n".join(md))
+
+
+if __name__ == "__main__":
+    main()