File size: 1,712 Bytes
bce3b06 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | """
Regression test for prompt parsing.
Worth having as a standing test rather than a one-off check, because a wrong
parse is invisible: it still yields a syntactically valid 54-character state, so
the model is simply asked to solve a different cube and the damage shows up only
as an unexplained low benchmark score. The fixture is generated by the real
renderer (scripts/print-prompt-pairs.ts), so it also fails if the prompt layout
ever changes.
python -m pytest test_serve.py # or: python test_serve.py
"""
import json
from pathlib import Path
from serve import extract_state
FIXTURE = Path(__file__).parent / "fixtures" / "prompt_pairs.json"
def load():
return json.loads(FIXTURE.read_text())
def test_extracts_state_from_real_prompts():
for case in load():
got = extract_state(case["prompt"])
assert got == case["facelets"], (
f"depth {case['depth']} index {case['index']}:\n"
f" want {case['facelets']}\n got {got}")
def test_rejects_prose_without_a_net():
"""The prompt's prose alone contains plenty of face letters -- naming the
colours up front and the move vocabulary at the end. Parsing must fail loudly
on text with no diagram rather than assembling a state out of that prose."""
prose = "Use U, D, L, R, F, or B. U=White (Up), R=Red (Right), F=Green (Front)."
try:
extract_state(prose)
except ValueError:
return
raise AssertionError("expected ValueError on prompt with no net diagram")
if __name__ == "__main__":
test_extracts_state_from_real_prompts()
test_rejects_prose_without_a_net()
print(f"ok: parsed all {len(load())} fixture prompts; rejected prose-only input")
|