File size: 1,290 Bytes
047277d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | """Code-execution verify loop (M1c) — with a scripted runner (no model/network)."""
from mathcompose.eval.code_verify import verify_with_code
from mathcompose.eval.parse import PARSE_FAIL
class _ScriptedRunner:
"""Emits a code block first, then boxes an index after seeing execution output."""
def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
saw_exec = any("Execution results" in m["content"] for m in messages if m["role"] == "user")
if saw_exec:
return ["The output shows 5, contradicting the step. \\boxed{2}"]
return ["Check paragraph 2.\n```python\nprint(2+3)\n```"]
class _ImmediateRunner:
def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
return ["All steps verify. \\boxed{-1}"]
class _GarbageRunner:
def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
return ["hmm no answer here"]
def test_executes_then_boxes():
assert verify_with_code(_ScriptedRunner(), "2+3", ["a", "2+3=4", "c"], max_rounds=3) == 2
def test_immediate_box_no_code():
assert verify_with_code(_ImmediateRunner(), "p", ["a", "b"]) == -1
def test_unparseable_returns_sentinel():
assert verify_with_code(_GarbageRunner(), "p", ["a", "b"], max_rounds=2) == PARSE_FAIL
|