feat(bench): forbidden_tools + tool_violations_gte — strict-toolban / procedural-compliance primitive (BFCL V4 / τ²-bench / IFBench anchor)

Pure bench-side compliance constraint over agent tool calls — no Rust
needed since this measures what the agent CHOSE to invoke, not what
the engine executed.

- Level / CompiledLevel gain an optional forbidden_tools: list[str].
- EpisodeSignals gains tools_called: dict[str, int] and tool_violations:
int. Tracked in eval_core.run_level by decoding each cmd's repr
(Rust enum stringifies Command::VariantName { … }) into snake_case
via _cmd_tool_name, so scripted and live-model policies are graded
by the same rule.
- New predicate tool_violations_gte: N for use in fail_condition
(typical: tool_violations_gte: 1 = zero-tolerance allowlist).
- _PHRASES translation: tool_violations_gte: 1 reads as 'you used a
forbidden tool (instant fail)'.
- 7 tests in tests/test_forbidden_tools.py cover cmd-repr decoder,
predicate semantics, schema roundtrip, and end-to-end live-engine
enforcement (violator policy LOSSES, allowed-only stall stays 0).

Unblocks B3 strict-toolban-fidelity-under-pressure + standalone
proc-strict-toolban-fidelity + proc-no-attack-passive-only +
proc-only-build-no-combat + proc-only-defend-no-attack (6+ packs from
one PR per SCENARIO_BACKLOG.md priority list).

openra_bench/eval_core.py

@@ -78,6 +78,31 @@ class EpisodeResult:
     reward_vector: dict = field(default_factory=dict)
 
 
+_CMD_NAME_RE = __import__("re").compile(r"Command::([A-Z][A-Za-z0-9]*)")
+
+
+def _cmd_tool_name(cmd: Any) -> str | None:
+    """Decode the snake_case tool name from a Command repr.
+
+    The Rust pyo3 Command enum stringifies as ``Command::VariantName { … }``
+    or ``Command::VariantName``. We extract the variant and convert to
+    snake_case to match the bench's tool-name vocabulary (and the
+    `tools:` / `forbidden_tools:` allowlist keys in YAML). Returns
+    ``None`` for anything that doesn't match — defensive; never raises.
+    """
+    m = _CMD_NAME_RE.search(repr(cmd))
+    if not m:
+        return None
+    variant = m.group(1)
+    # CamelCase → snake_case (MoveUnits → move_units, AttackUnit → attack_unit)
+    out: list[str] = []
+    for i, ch in enumerate(variant):
+        if i and ch.isupper():
+            out.append("_")
+        out.append(ch.lower())
+    return "".join(out)
+
+
 def scripted_explore_agent(render_state: dict, Command: Any) -> list:
     """Baseline reference agent: walk every unit toward the nearest
     unexplored frontier cell. Exercises the move path; a useful
@@ -203,9 +228,23 @@ def run_level(
         interrupt_mode = bool(enabled_sig) and raw_env is not None and hasattr(
             raw_env, "step_until_event"
         )
+        # Strict-toolban / procedural-compliance accounting: any cmd whose
+        # tool name is in compiled.forbidden_tools increments
+        # signals.tool_violations (read by the tool_violations_gte
+        # predicate). Tracked here so scripted and live-model policies
+        # are graded by the exact same rule.
+        forbidden = {str(t).lower() for t in (compiled.forbidden_tools or [])}
         for turns in range(1, compiled.max_turns + 1):
             rs = adapter.render_state()
             cmds = agent_fn(rs, env.Command) or [env.Command.observe()]
+            for _cmd in cmds:
+                _tn = _cmd_tool_name(_cmd)
+                if _tn:
+                    adapter.signals.tools_called[_tn] = (
+                        adapter.signals.tools_called.get(_tn, 0) + 1
+                    )
+                    if _tn in forbidden:
+                        adapter.signals.tool_violations += 1
             if not conceded:
                 conceded = any("Surrender" in repr(c) for c in cmds)
             interrupt = None

openra_bench/game_knowledge.py

@@ -201,6 +201,11 @@ _PHRASES: dict[str, Any] = {
         + "+".join((v or {}).get("types", []))
         + f" at the base near ({(v or {}).get('x')},{(v or {}).get('y')})"
     ),
+    "tool_violations_gte": lambda v: (
+        "you used a forbidden tool (instant fail)"
+        if int(v) <= 1
+        else f"you used a forbidden tool ≥{v} times"
+    ),
 }
 
 

openra_bench/rust_adapter.py

@@ -131,6 +131,15 @@ class EpisodeSignals:
     # waypoint_sequence's ordered-visit progress, keyed by sequence id).
     # Reset for free: EpisodeSignals is reconstructed each episode.
     seq_progress: dict = field(default_factory=dict)
+    # Per-episode tool-use accounting for the strict-toolban / procedural-
+    # compliance family. tools_called counts each tool name the agent
+    # invoked this episode; tool_violations counts how many of those calls
+    # were on the scenario's forbidden_tools list. The `tool_violations_gte`
+    # predicate reads from here (typically as a fail clause). Tracking is
+    # bench-side (see eval_core.run_level), so scripted policies are
+    # graded by the same rule as live models.
+    tools_called: dict[str, int] = field(default_factory=dict)
+    tool_violations: int = 0
     # Outcome is synthesized (Rust has no result field): a scenario is
     # "won" when all enemy buildings have been discovered AND/OR all
     # enemy units neutralized — refined per-scenario in Phase 2 rubrics.

openra_bench/scenarios/schema.py

@@ -110,6 +110,12 @@ class Level(BaseModel):
         description="Per-level economy budget (overrides pack default; "
         "engine default 5000 when unset everywhere).",
     )
+    # Procedural-compliance family: any agent command whose tool name is
+    # in this list increments signals.tool_violations (bench-side track,
+    # see eval_core). Use as a fail clause via the
+    # `tool_violations_gte` predicate — usually `tool_violations_gte: 1`
+    # for a strict zero-tolerance allowlist. Empty list = no constraint.
+    forbidden_tools: list[str] = Field(default_factory=list)
 
 
 class ScenarioConfig(BaseModel):
@@ -155,6 +161,7 @@ class CompiledLevel(BaseModel):
     fog_mode: str = "vision"
     config_name: str | None = None
     objective_coords: Literal["exact", "relative"] = "exact"
+    forbidden_tools: list[str] = Field(default_factory=list)
 
 
 class ScenarioPack(BaseModel):
@@ -222,6 +229,7 @@ class ScenarioPack(BaseModel):
             else self.starting_cash,
             map_supported=map_supported,
             objective_coords=lvl.objective_coords,
+            forbidden_tools=list(lvl.forbidden_tools or []),
         )
 
     def config_names(self) -> list[str]:

openra_bench/scenarios/win_conditions.py

@@ -189,6 +189,16 @@ _PREDICATES: dict[str, Callable[[WinContext, Any], bool]] = {
         <= float(v.get("radius", 5)) ** 2
     )
     >= int(v.get("count", 1)),
+    # Procedural-compliance / strict-toolban family: triggers when the
+    # agent has invoked tools on the level's `forbidden_tools` list at
+    # least N times this episode. Typical use is in a fail clause as
+    # `tool_violations_gte: 1` (zero-tolerance), but >1 also lets a pack
+    # tolerate up to N slips before failing. The counter is maintained
+    # by eval_core.run_level — see ScenarioPack/Level.forbidden_tools.
+    "tool_violations_gte": lambda c, v: getattr(
+        c.signals, "tool_violations", 0
+    )
+    >= int(v),
 }
 
 LEAF_KEYS = frozenset(_PREDICATES)

tests/test_forbidden_tools.py

@@ -0,0 +1,170 @@
+"""forbidden_tools / tool_violations: bench-side procedural-compliance.
+
+Tests the new schema field + signal + predicate end-to-end. The
+real-world anchor is BFCL V4 / τ²-bench / IFBench: the agent has a
+strict allowlist and any disallowed tool call must trip a fail clause.
+
+Three things must hold:
+
+1. The cmd-repr → tool-name decoder maps every Command variant to its
+   snake_case allowlist key (move_units, attack_unit, …).
+2. eval_core.run_level increments signals.tool_violations exactly once
+   per disallowed call, regardless of whether the policy is scripted or
+   wrapped by ModelAgent.
+3. The `tool_violations_gte` predicate reads that counter and fails the
+   episode (typical use: `tool_violations_gte: 1` in fail_condition).
+"""
+
+from __future__ import annotations
+
+import types
+from pathlib import Path
+
+import pytest
+
+from openra_bench.eval_core import _cmd_tool_name, run_level
+from openra_bench.scenarios import load_pack
+from openra_bench.scenarios.loader import compile_level
+from openra_bench.scenarios.win_conditions import WinContext, evaluate
+
+PACKS = Path(__file__).parent.parent / "openra_bench" / "scenarios" / "packs"
+
+
+# ── 1. cmd-repr decoder ───────────────────────────────────────────────────
+
+
+def test_cmd_tool_name_extracts_snake_case_from_command_repr():
+    """The Rust Command enum stringifies as `Command::VariantName { … }`
+    or `Command::VariantName`. _cmd_tool_name must produce the bench's
+    snake_case tool key for every variant — covers the cmd type names
+    actually observed live on the engine in this session."""
+    # MoveUnits with payload
+    class _Stub:
+        def __init__(self, r: str):
+            self.r = r
+
+        def __repr__(self) -> str:
+            return self.r
+
+    assert _cmd_tool_name(_Stub('Command::MoveUnits { unit_ids: ["1"], '
+                                'target_x: 10, target_y: 10 }')) == "move_units"
+    # bare variant
+    assert _cmd_tool_name(_Stub("Command::Observe")) == "observe"
+    # multi-word variant
+    assert _cmd_tool_name(_Stub('Command::AttackUnit { unit_ids: ["1"], '
+                                'target_id: "2" }')) == "attack_unit"
+    assert _cmd_tool_name(_Stub('Command::AttackMove { … }')) == "attack_move"
+    assert _cmd_tool_name(_Stub('Command::Stop { unit_ids: ["1"] }')) == "stop"
+    assert _cmd_tool_name(_Stub("Command::Surrender")) == "surrender"
+    assert _cmd_tool_name(_Stub('Command::SetStance { stance: 2 }')) == "set_stance"
+    assert _cmd_tool_name(_Stub('Command::PlaceBuilding { … }')) == "place_building"
+    # Non-Command reprs → None, never raise
+    assert _cmd_tool_name(_Stub("something else")) is None
+    assert _cmd_tool_name(_Stub("")) is None
+
+
+# ── 2. predicate reads from signals.tool_violations ───────────────────────
+
+
+def test_tool_violations_gte_predicate_reads_signals():
+    """tool_violations_gte: N is true iff signals.tool_violations >= N."""
+    sig = types.SimpleNamespace(tool_violations=0)
+    ctx = WinContext(signals=sig, render_state={"units_summary": []})
+    assert evaluate({"tool_violations_gte": 1}, ctx) is False
+    sig.tool_violations = 1
+    assert evaluate({"tool_violations_gte": 1}, ctx) is True
+    sig.tool_violations = 5
+    assert evaluate({"tool_violations_gte": 3}, ctx) is True
+    assert evaluate({"tool_violations_gte": 6}, ctx) is False
+
+
+def test_tool_violations_default_zero_for_legacy_signals():
+    """Signals that predate the field (e.g. raw types.SimpleNamespace
+    in unit tests) should default to 0, not raise."""
+    sig = types.SimpleNamespace()  # no tool_violations attr
+    ctx = WinContext(signals=sig, render_state={"units_summary": []})
+    assert evaluate({"tool_violations_gte": 1}, ctx) is False
+
+
+# ── 3. _PHRASES has a translation (suite invariant elsewhere too) ─────────
+
+
+def test_tool_violations_gte_has_a_phrase_translation():
+    from openra_bench.game_knowledge import _PHRASES
+
+    assert "tool_violations_gte" in _PHRASES
+    p = _PHRASES["tool_violations_gte"]
+    assert "forbidden tool" in p(1)
+    assert "≥3" in p(3)
+
+
+# ── 4. forbidden_tools roundtrips through schema ──────────────────────────
+
+
+def test_forbidden_tools_roundtrips_through_compiled_level():
+    """A pack that declares forbidden_tools on a Level surfaces it on
+    the CompiledLevel. CompiledLevel.forbidden_tools is what
+    eval_core.run_level reads."""
+    # Use an existing pack and inject forbidden_tools at the Level model
+    # level (we don't need to write a new YAML to verify roundtripping).
+    pack = load_pack(PACKS / "custom-map-no-enemy.yaml")
+    pack.levels["easy"].forbidden_tools = ["attack_unit", "attack_move"]
+    c = compile_level(pack, "easy")
+    assert c.forbidden_tools == ["attack_unit", "attack_move"]
+
+
+# ── 5. live engine: forbidden cmds increment counter; allowed don't ───────
+
+
+def test_forbidden_tools_counted_live_via_run_level():
+    """The end-to-end test: a scripted policy that issues an
+    attack_unit (forbidden) on every turn must accumulate
+    tool_violations > 0 by episode end, and the tool_violations_gte:1
+    fail clause must fire ⇒ outcome == 'loss'."""
+    pytest.importorskip("openra_train")
+
+    pack = load_pack(PACKS / "custom-map-no-enemy.yaml")
+    # Hot-patch the easy level: forbid attack_unit + put it in fail.
+    pack.levels["easy"].forbidden_tools = ["attack_unit", "attack_move"]
+    from openra_bench.scenarios.win_conditions import WinCondition
+
+    pack.levels["easy"].fail_condition = WinCondition(
+        any_of=[
+            {"after_ticks": 10_000},
+            {"tool_violations_gte": 1},
+        ]
+    )
+    c = compile_level(pack, "easy")
+    assert c.forbidden_tools == ["attack_unit", "attack_move"]
+
+    def violator(rs, Command):
+        # observe + a single forbidden attack_unit on a non-existent
+        # target id (the engine warns, but the bench tracks the tool
+        # name BEFORE the engine evaluates it — so the violation
+        # counts even when the engine refuses the order).
+        return [Command.observe(), Command.attack_unit(["1"], target_id="99999")]
+
+    res = run_level(c, violator, seed=1)
+    # bench-side accounting must have flagged ≥1 forbidden call …
+    assert res.signals.tool_violations >= 1, res.signals.tools_called
+    # … and the fail predicate must have fired.
+    assert res.outcome == "loss", res.outcome
+
+
+def test_allowed_tools_dont_increment_violations():
+    """The same harness with NO forbidden_tools and only allowed cmds
+    must yield tool_violations == 0 (no false positives)."""
+    pytest.importorskip("openra_train")
+
+    pack = load_pack(PACKS / "custom-map-no-enemy.yaml")
+    c = compile_level(pack, "easy")
+    assert c.forbidden_tools == []
+
+    def stall(rs, Command):
+        return [Command.observe()]
+
+    res = run_level(c, stall, seed=1)
+    assert res.signals.tool_violations == 0
+    # Tools_called should at least have observe (the bench's default
+    # fallback and the policy's explicit call).
+    assert "observe" in res.signals.tools_called