| """Eyeball what Model V actually outputs — to diagnose a weak ProcessBench score. |
| |
| python scripts/inspect_verifier.py --adapter runs/verifier_v --n 6 --only error |
| |
| Prints, per example: gold first-error index, V's parsed prediction, and the raw |
| critique. Reveals whether V is collapsing to -1, mislocating, off-by-one, or |
| rambling/truncating. |
| """ |
| import argparse |
| import sys |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) |
|
|
| from mathcompose.common.chat import verifier_messages |
| from mathcompose.data.schema import parse_verifier_output |
|
|
|
|
| def main() -> int: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--base-id", default="Qwen/Qwen2.5-Math-1.5B-Instruct") |
| ap.add_argument("--adapter", default=None) |
| ap.add_argument("--split", default="gsm8k") |
| ap.add_argument("--n", type=int, default=6) |
| ap.add_argument("--only", choices=["error", "correct", "both"], default="error") |
| ap.add_argument("--max-new-tokens", type=int, default=768) |
| args = ap.parse_args() |
|
|
| from datasets import load_dataset |
| from mathcompose.eval.runners import HFRunner |
|
|
| ds = load_dataset("Qwen/ProcessBench", split=args.split).shuffle(seed=0) |
| runner = HFRunner(args.base_id, adapter_path=args.adapter) |
|
|
| shown = 0 |
| for row in ds: |
| gold = int(row["label"]) |
| if args.only == "error" and gold == -1: |
| continue |
| if args.only == "correct" and gold != -1: |
| continue |
| out = runner.chat(verifier_messages(row["problem"], list(row["steps"])), |
| n=1, temperature=0.0, max_new_tokens=args.max_new_tokens)[0] |
| pred = parse_verifier_output(out).first_error_index |
| flag = "HIT" if pred == gold else "miss" |
| print(f"\n{'=' * 70}\n[{flag}] gold={gold} pred={pred} (#steps={len(row['steps'])})") |
| print("-" * 70) |
| print(out.strip()[:1200]) |
| shown += 1 |
| if shown >= args.n: |
| break |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
