scripts/compare_arl2_patch_vs_frame.py

"""Quantitative arl2_patch vs frame-level ARL2 comparison on generated mp4s.

Same seed (0) and same prompts -> frame-aligned, directly comparable. Reuses
the same motion-magnitude / sharpness / rel-L2 diagnostics as
scripts/compare_gen_videos.py, adapted to the stage_b_{ckpt}_{split} directory
layout used by run_gen_arl2_6layer_3chunk_teacher_flow_all.sh and
run_gen_arl2_6layer_3chunk_patch_teacher_flow_all.sh.
"""
import argparse
import glob
import os

from torchvision.io import read_video


def vid_stats(path):
    v, _, _ = read_video(path, pts_unit="sec", output_format="TCHW")
    v = v.float() / 255.0
    dt = (v[1:] - v[:-1]).abs().mean().item()          # temporal motion magnitude
    dh = (v[..., 1:, :] - v[..., :-1, :]).pow(2).mean()  # spatial high-freq energy
    dw = (v[..., :, 1:] - v[..., :, :-1]).pow(2).mean()
    sharp = (dh + dw).item()
    return v, dt, sharp


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--frame_root", default="workspace/outputs/gen_arl2_6layer_3chunk_teacher_flow")
    ap.add_argument("--patch_root", default="workspace/outputs/gen_arl2_6layer_3chunk_patch_teacher_flow")
    ap.add_argument("--out", default="workspace/markdown/arl2_patch_vs_frame_comparison.md")
    ap.add_argument("--checkpoints", nargs="+", default=["best", "ema", "final"])
    args = ap.parse_args()

    L = ["# ARL2 patch (8-patch softmax+MLP inter) vs frame-level ARL2, teacher_flow objective\n",
         "Same seed=0, same prompts, same 832x480/21-latent-frame/81-decoded-frame protocol -> "
         "frame-aligned videos, directly comparable. rel-L2 = ||patch-frame||/||frame|| over the "
         "decoded video (higher = more different from the frame-level control).\n"]
    for ckpt in args.checkpoints:
        L.append(f"\n## checkpoint = {ckpt}\n")
        L.append("| set | prompt (truncated) | rel-L2 patch-vs-frame | motion-mag (patch / frame) | sharpness (patch / frame) |")
        L.append("|---|---|---|---|---|")
        for split in ["train", "ood"]:
            fdir = os.path.join(args.frame_root, f"stage_b_{ckpt}_{split}")
            pdir = os.path.join(args.patch_root, f"stage_b_{ckpt}_{split}")
            rels, dt_p, dt_f, sh_p, sh_f = [], [], [], [], []
            for fp in sorted(glob.glob(os.path.join(fdir, "*.mp4"))):
                name = os.path.basename(fp)
                pp = os.path.join(pdir, name)
                if not os.path.exists(pp):
                    continue
                vf, dtf, shf = vid_stats(fp)
                vp, dtp, shp = vid_stats(pp)
                n = min(vf.shape[0], vp.shape[0])
                rel = ((vp[:n] - vf[:n]).norm() / (vf[:n].norm() + 1e-8)).item()
                rels.append(rel); dt_p.append(dtp); dt_f.append(dtf); sh_p.append(shp); sh_f.append(shf)
                L.append(f"| {split} | {name[:42]} | {rel*100:.2f}% | {dtp:.4f} / {dtf:.4f} | {shp:.4f} / {shf:.4f} |")
            if rels:
                m = lambda x: sum(x) / len(x)
                L.append(f"| **{split} MEAN** | ({len(rels)} prompts) | **{m(rels)*100:.2f}%** | "
                          f"{m(dt_p):.4f} / {m(dt_f):.4f} | {m(sh_p):.4f} / {m(sh_f):.4f} |")

    L.append("\n**Read:** motion-mag (mean |frame_t+1 - frame_t|) higher = more/sharper motion, "
              "less = more static/blurred-together. sharpness (spatial gradient energy) higher = "
              "more spatial detail. rel-L2 just measures how much the two checkpoints diverge; it "
              "does not by itself say which is better -- read it together with motion-mag/sharpness "
              "and a visual check.\n")
    os.makedirs(os.path.dirname(args.out), exist_ok=True)
    with open(args.out, "w") as f:
        f.write("\n".join(L) + "\n")
    print("wrote", args.out)
    print("\n".join(L))


if __name__ == "__main__":
    main()