code/run_benchmark.sh

#!/usr/bin/env bash
# Scores a trained checkpoint through the real harness and archives the result.
#
# Everything here goes through bench.ts, not a Python loop, so the model faces the
# canonical index-addressed scrambles and is scored by the same engine + Kociemba
# path as every LLM on the board. The output is an ordinary results row.
#
#   ./run_benchmark.sh <checkpoint> [n-per-depth]
set -euo pipefail

CHECKPOINT="${1:?usage: run_benchmark.sh <checkpoint-dir-or-hub-id> [n]}"
N="${2:-200}"
PORT="${PORT:-8077}"
PY="${PY:-python3}"
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
STAMP="$(date -u +%Y-%m-%d)"

# n is large on purpose. The n=10 convention exists because API calls cost money;
# local inference is free, so the sampling noise that makes a 50% and an 80% cell
# statistically indistinguishable at n=10 can simply be removed.
echo "serving $CHECKPOINT on :$PORT"
"$PY" "$REPO_ROOT/training/serve.py" --checkpoint "$CHECKPOINT" --port "$PORT" &
SERVER=$!
trap 'kill $SERVER 2>/dev/null || true' EXIT

for _ in $(seq 1 60); do
  curl -sS --max-time 2 "http://127.0.0.1:$PORT/v1/models" >/dev/null 2>&1 && break
  sleep 2
done

cd "$REPO_ROOT"
mkdir -p results
# One depth per invocation with seed = depth * 1000. Passing several depths at once
# would derive them all from a single base seed and silently stop matching the
# canonical scrambles every other model was scored against.
for DEPTH in 3 6 10 15 20 25 30 40 50 100; do
  OUT="results/tiny-cube-d${DEPTH}-${STAMP}.jsonl"
  echo "=== depth $DEPTH (seed $((DEPTH * 1000)), n=$N) ==="
  OPENROUTER_BASE_URL="http://127.0.0.1:$PORT/v1" OPENROUTER_API_KEY=local \
    npx tsx scripts/bench.ts \
      --models local/tiny-cube --depths "$DEPTH" --seed "$((DEPTH * 1000))" \
      --n "$N" --m 1 --concurrency 4 --no-upload \
      --timeout-ms 60000 --out "$OUT"
done

echo
echo "=== summary ==="
"$PY" - <<'PYEOF'
import glob, json
from collections import defaultdict
by = defaultdict(lambda: [0, 0])
for path in glob.glob("results/tiny-cube-d*.jsonl"):
    for line in open(path):
        r = json.loads(line)
        by[r["depth"]][0] += bool(r.get("solved"))
        by[r["depth"]][1] += 1
print(f"{'depth':>6} {'solved':>8} {'n':>6} {'rate':>8}")
for d in sorted(by):
    ok, n = by[d]
    print(f"{d:>6} {ok:>8} {n:>6} {ok/n:>7.1%}")
PYEOF

echo
echo "Next: back the JSONL into D1 and push it to R2 (see CLAUDE.md 'Where run artefacts live'):"
echo "  python3 scripts/jsonl-to-sql.py out.sql results/tiny-cube-d*.jsonl"
echo "  cd web && wrangler d1 execute rubiksbench --remote --file=../out.sql"
echo "  cd web && for f in ../results/tiny-cube-d*.jsonl; do wrangler r2 object put \\"
echo "      rubiksbench-logs/$STAMP/runs/\$(basename \$f) --file=\$f --remote; done"