File size: 1,257 Bytes
014b14c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | """Smoke #8: composition math (gap, recovery) with fake callbacks."""
from mathcompose.eval.compose import evaluate_composition, _weighted_vote
def test_weighted_vote_rescues_minority_correct():
answers = ["4", "4", "7", "7", "7"]
correct = [True, True, False, False, False]
# plain vote -> "7" (wrong)
assert _weighted_vote(answers, [1] * 5, correct) is False
# V-weighted (correct ones weighted high) -> "4" (right)
assert _weighted_vote(answers, [0.9, 0.9, 0.1, 0.1, 0.1], correct) is True
def test_composition_gap_and_recovery():
samples = [r"\boxed{4} #0", r"\boxed{4} #1", r"\boxed{7} #2", r"\boxed{7} #3", r"\boxed{7} #4"]
vmap = {0: 0.9, 1: 0.9, 2: 0.1, 3: 0.1, 4: 0.1}
def solve_fn(_):
return samples
def v_score_fn(_, steps):
joined = " ".join(steps)
for i in range(len(samples)):
if f"#{i}" in joined:
return vmap[i]
return 0.0
res = evaluate_composition([("q", "4")], solve_fn, v_score_fn, progress=False)
assert res["maj@n"] == 0.0
assert res["oracle"] == 1.0
assert res["weighted_maj"] == 1.0
assert res["best_of_n"] == 1.0
assert res["generator_verifier_gap"] == 1.0
assert res["gap_recovered_by_V"] == 1.0
|