File size: 2,762 Bytes
bce3b06 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | #!/usr/bin/env bash
# Scores a trained checkpoint through the real harness and archives the result.
#
# Everything here goes through bench.ts, not a Python loop, so the model faces the
# canonical index-addressed scrambles and is scored by the same engine + Kociemba
# path as every LLM on the board. The output is an ordinary results row.
#
# ./run_benchmark.sh <checkpoint> [n-per-depth]
set -euo pipefail
CHECKPOINT="${1:?usage: run_benchmark.sh <checkpoint-dir-or-hub-id> [n]}"
N="${2:-200}"
PORT="${PORT:-8077}"
PY="${PY:-python3}"
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
STAMP="$(date -u +%Y-%m-%d)"
# n is large on purpose. The n=10 convention exists because API calls cost money;
# local inference is free, so the sampling noise that makes a 50% and an 80% cell
# statistically indistinguishable at n=10 can simply be removed.
echo "serving $CHECKPOINT on :$PORT"
"$PY" "$REPO_ROOT/training/serve.py" --checkpoint "$CHECKPOINT" --port "$PORT" &
SERVER=$!
trap 'kill $SERVER 2>/dev/null || true' EXIT
for _ in $(seq 1 60); do
curl -sS --max-time 2 "http://127.0.0.1:$PORT/v1/models" >/dev/null 2>&1 && break
sleep 2
done
cd "$REPO_ROOT"
mkdir -p results
# One depth per invocation with seed = depth * 1000. Passing several depths at once
# would derive them all from a single base seed and silently stop matching the
# canonical scrambles every other model was scored against.
for DEPTH in 3 6 10 15 20 25 30 40 50 100; do
OUT="results/tiny-cube-d${DEPTH}-${STAMP}.jsonl"
echo "=== depth $DEPTH (seed $((DEPTH * 1000)), n=$N) ==="
OPENROUTER_BASE_URL="http://127.0.0.1:$PORT/v1" OPENROUTER_API_KEY=local \
npx tsx scripts/bench.ts \
--models local/tiny-cube --depths "$DEPTH" --seed "$((DEPTH * 1000))" \
--n "$N" --m 1 --concurrency 4 --no-upload \
--timeout-ms 60000 --out "$OUT"
done
echo
echo "=== summary ==="
"$PY" - <<'PYEOF'
import glob, json
from collections import defaultdict
by = defaultdict(lambda: [0, 0])
for path in glob.glob("results/tiny-cube-d*.jsonl"):
for line in open(path):
r = json.loads(line)
by[r["depth"]][0] += bool(r.get("solved"))
by[r["depth"]][1] += 1
print(f"{'depth':>6} {'solved':>8} {'n':>6} {'rate':>8}")
for d in sorted(by):
ok, n = by[d]
print(f"{d:>6} {ok:>8} {n:>6} {ok/n:>7.1%}")
PYEOF
echo
echo "Next: back the JSONL into D1 and push it to R2 (see CLAUDE.md 'Where run artefacts live'):"
echo " python3 scripts/jsonl-to-sql.py out.sql results/tiny-cube-d*.jsonl"
echo " cd web && wrangler d1 execute rubiksbench --remote --file=../out.sql"
echo " cd web && for f in ../results/tiny-cube-d*.jsonl; do wrangler r2 object put \\"
echo " rubiksbench-logs/$STAMP/runs/\$(basename \$f) --file=\$f --remote; done"
|