Upload folder using huggingface_hub

README.md

@@ -253,9 +253,7 @@ This performs:
 - GRPO continuation from the SFT adapter.
 - Mixed reward shaping where env-derived reward remains active and local shaping can be down-weighted/up-weighted via scales.
 - Optional complexity curriculum (`easy_to_full`) that starts with stronger scaffold/local signals and anneals to env-dominant reward later.
-- Stability-first optimization defaults (cosine schedule + warmup + grad clipping + higher GRPO KL beta) and optional guardrails:
-  - `--reward-ema-decay 0..1` smooths the *env* reward channel (defaults come from `--training-preset`).
-  - omit `--no-stability-tripwire` to enable early stopping when logs show repeated “env reward down + loss up” (GRPO) or repeated loss blow-ups (SFT).
+- Stability-first optimization defaults (cosine schedule + warmup + grad clipping + higher GRPO KL beta). Optional `--reward-ema-decay 0..1` smooths the *env* reward channel (defaults come from `--training-preset`). Training always runs the full `max_*_steps` (no early-stop callbacks).
 
 Recommended model strategy for hackathon iteration speed:
 - Start with `--model-preset small_iter_fast` (`unsloth/Qwen2.5-3B-Instruct`) + QLoRA.

inference.py

@@ -1,9 +1,15 @@
 """
-Baseline runner for the Ghostexec submission.
+Baseline runner for the Ghostexec OpenEnv submission.
 
-This script queries a chat model through the OpenAI client, sends its decision
-to the environment server, and prints machine-readable lines expected by simple
-evaluators/log parsers.
+Links (keep these in sync when you change the env):
+  - **openenv.yaml** — `name`, `port`, `tasks[].id`, `tasks[].grader`, `max_steps`, `difficulties`
+  - **graders.py** — episode-level scores in (0.01, 0.99); symbols referenced by `tasks[].grader`
+  - **scenarios/*.json** — fixtures named in each task description in `openenv.yaml`
+  - **server/** — FastAPI app from `openenv.yaml` `app:` (`server.app:app`)
+
+This script calls the deployed/local env over HTTP (`/reset`, `/step`), queries an LLM via the
+OpenAI-compatible HF router, then aggregates step rewards with the **same** grader functions
+used for OpenEnv validation (must match `openenv.yaml` task table).
 """
 
 from __future__ import annotations
@@ -11,6 +17,8 @@ from __future__ import annotations
 import argparse
 import json
 import os
+import re
+from pathlib import Path
 from typing import Any, Iterable
 
 import requests
@@ -23,11 +31,14 @@ except ImportError:
     from graders import dinner_disaster_grader, monday_morning_grader, phase2_core_grader
     from models import GhostexecAction
 
+REPO_ROOT = Path(__file__).resolve().parent
+OPENENV_SPEC = REPO_ROOT / "openenv.yaml"
 
 API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
 MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct")
 HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
-ENV_URL = os.getenv("ENV_URL", "http://localhost:7860").rstrip("/")
+# Default matches openenv.yaml `port: 8000` and `uv run server` / Spaces proxy.
+ENV_URL = os.getenv("ENV_URL", "http://127.0.0.1:8000").rstrip("/")
 TASK_OVERRIDE = os.getenv("TASK_NAME", "").strip()
 BENCHMARK = "ghostexec"
 
@@ -44,6 +55,77 @@ TASK_TO_GRADER = {
     "dinner_disaster": dinner_disaster_grader,
 }
 
+_GRADER_TO_SYMBOL = {
+    phase2_core_grader: "graders.phase2_core_grader",
+    monday_morning_grader: "graders.monday_morning_grader",
+    dinner_disaster_grader: "graders.dinner_disaster_grader",
+}
+
+
+def load_openenv_task_rows(spec_path: Path) -> list[dict[str, str]]:
+    """Parse task `id` + `grader` from openenv.yaml without requiring PyYAML."""
+    if not spec_path.is_file():
+        return []
+    rows: list[dict[str, str]] = []
+    cur: dict[str, str] | None = None
+    for raw in spec_path.read_text(encoding="utf-8").splitlines():
+        line = raw.rstrip()
+        m_id = re.match(r"^\s*-\s+id:\s*(\S+)\s*$", line)
+        if m_id:
+            if cur and cur.get("id"):
+                rows.append(cur)
+            cur = {"id": m_id.group(1).strip()}
+            continue
+        if cur is not None:
+            m_gr = re.match(r"^\s+grader:\s*(\S+)\s*$", line)
+            if m_gr:
+                cur["grader"] = m_gr.group(1).strip()
+    if cur and cur.get("id"):
+        rows.append(cur)
+    return rows
+
+
+def openenv_max_steps(spec_path: Path) -> int | None:
+    if not spec_path.is_file():
+        return None
+    m = re.search(r"(?m)^max_steps:\s*(\d+)\s*$", spec_path.read_text(encoding="utf-8"))
+    return int(m.group(1)) if m else None
+
+
+def verify_openenv_alignment(spec_path: Path = OPENENV_SPEC) -> list[str]:
+    """Return human-readable warnings if inference tables drift from openenv.yaml."""
+    warnings: list[str] = []
+    rows = load_openenv_task_rows(spec_path)
+    if not rows:
+        warnings.append(f"Could not read tasks from {spec_path} — skipping alignment check.")
+        return warnings
+
+    yaml_ids = [r["id"] for r in rows]
+    if tuple(yaml_ids) != TASK_SETS["all"]:
+        warnings.append(
+            f"openenv.yaml task order/ids {yaml_ids!r} != inference TASK_SETS['all'] {list(TASK_SETS['all'])!r}"
+        )
+
+    for row in rows:
+        tid = row["id"]
+        gref = row.get("grader", "")
+        fn = TASK_TO_GRADER.get(tid)
+        if fn is None:
+            warnings.append(f"openenv.yaml task {tid!r} has no TASK_TO_GRADER entry in inference.py")
+            continue
+        expected = _GRADER_TO_SYMBOL.get(fn)
+        if expected and gref and gref != expected:
+            warnings.append(
+                f"Task {tid!r}: openenv.yaml grader {gref!r} != inference mapping {expected!r}"
+            )
+
+    for tid in TASK_SETS["all"]:
+        if tid not in yaml_ids:
+            warnings.append(f"inference TASK_SETS includes {tid!r} but openenv.yaml has no such task id")
+
+    return warnings
+
+
 SYSTEM_MESSAGE = """
 You are acting as an AI Chief-of-Staff assistant in Ghostexec.
 
@@ -78,8 +160,12 @@ Rules:
 """.strip()
 
 
-def emit_start(task_name: str) -> None:
-    print(f"[START] task={task_name} env={BENCHMARK} model={MODEL_NAME}", flush=True)
+def emit_start(task_name: str, max_steps_hint: int | None) -> None:
+    ms = f" max_steps={max_steps_hint}" if max_steps_hint is not None else ""
+    print(
+        f"[START] task={task_name} env={BENCHMARK} model={MODEL_NAME} env_url={ENV_URL}{ms}",
+        flush=True,
+    )
 
 
 def emit_step(step_no: int, action_text: str, reward: float, done: bool, error: str | None) -> None:
@@ -202,13 +288,13 @@ def final_score(task_name: str, rewards: list[float]) -> float:
     return float(grader({"rewards": rewards}))
 
 
-def run_one_task(llm: Any, task_name: str) -> None:
+def run_one_task(llm: Any, task_name: str, *, max_steps_hint: int | None) -> None:
     rewards: list[float] = []
     steps_taken = 0
     score = 0.0
     success = False
 
-    emit_start(task_name)
+    emit_start(task_name, max_steps_hint)
 
     try:
         result = fetch_reset(task_name)
@@ -247,18 +333,57 @@ def run_one_task(llm: Any, task_name: str) -> None:
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description="Run the Ghostexec baseline agent")
+    parser = argparse.ArgumentParser(
+        description="Run the Ghostexec baseline agent (HTTP env + HF OpenAI-compatible router)."
+    )
     parser.add_argument(
         "--difficulty",
         choices=["easy", "medium", "hard", "all"],
         default="all",
-        help="Which task subset to run",
+        help="Which task subset to run (mirrors openenv.yaml difficulties / tasks).",
+    )
+    parser.add_argument(
+        "--env-url",
+        default="",
+        help="Override Ghostexec HTTP base URL (else ENV_URL env or default 127.0.0.1:8000).",
+    )
+    parser.add_argument(
+        "--list-tasks",
+        action="store_true",
+        help="Print tasks parsed from openenv.yaml and exit.",
+    )
+    parser.add_argument(
+        "--check-alignment",
+        action="store_true",
+        help="Verify inference.py TASK_TO_GRADER matches openenv.yaml; print warnings and exit 1 if drift.",
     )
     args = parser.parse_args()
 
+    global ENV_URL
+    if args.env_url.strip():
+        ENV_URL = args.env_url.strip().rstrip("/")
+
+    if args.list_tasks:
+        for row in load_openenv_task_rows(OPENENV_SPEC):
+            print(row.get("id", ""), "->", row.get("grader", "?"))
+        return
+
+    drift = verify_openenv_alignment(OPENENV_SPEC)
+    for w in drift:
+        print(f"[openenv] {w}", flush=True)
+
+    if args.check_alignment:
+        hard = [x for x in drift if not x.startswith("Could not read")]
+        if hard:
+            for x in hard:
+                print(f"[ALIGNMENT ERROR] {x}", flush=True)
+            raise SystemExit(1)
+        return
+
+    max_steps_hint = openenv_max_steps(OPENENV_SPEC)
     llm = client()
     for task_name in choose_tasks(args.difficulty):
-        run_one_task(llm, task_name)
+        run_one_task(llm, task_name, max_steps_hint=max_steps_hint)
 
 
 if __name__ == "__main__":

scripts/train_sft_then_grpo.py

@@ -67,115 +67,6 @@ TRAINING_PRESETS: dict[str, dict[str, float | int | str]] = {
 }
 
 
-def _as_float(x: object | None) -> float | None:
-    if x is None:
-        return None
-    try:
-        return float(x)
-    except Exception:
-        return None
-
-
-class StabilityTripwire(TrainerCallback):
-    """Stop training when logs show sustained reward collapse + loss blow-up."""
-
-    def __init__(
-        self,
-        *,
-        min_step: int,
-        reward_key: str,
-        loss_key: str,
-        reward_drop: float,
-        loss_spike: float,
-        bad_streak: int,
-    ) -> None:
-        self.min_step = min_step
-        self.reward_key = reward_key
-        self.loss_key = loss_key
-        self.reward_drop = reward_drop
-        self.loss_spike = loss_spike
-        self.bad_streak = bad_streak
-        self._best_reward: float | None = None
-        self._best_loss: float | None = None
-        self._streak = 0
-
-    def on_log(self, args, state, control, logs=None, **kw):  # type: ignore[no-untyped-def]
-        logs = logs or {}
-        step = int(getattr(state, "global_step", 0) or 0)
-        if step < self.min_step:
-            return control
-
-        r = _as_float(logs.get(self.reward_key))
-        loss = _as_float(logs.get(self.loss_key))
-
-        reward_bad = False
-        loss_bad = False
-
-        if r is not None:
-            if self._best_reward is None or r > self._best_reward:
-                self._best_reward = r
-            elif self._best_reward is not None and self._best_reward - r >= self.reward_drop:
-                reward_bad = True
-
-        if loss is not None:
-            if self._best_loss is None or loss < self._best_loss:
-                self._best_loss = loss
-            elif self._best_loss is not None and loss - self._best_loss >= self.loss_spike:
-                loss_bad = True
-
-        bad = reward_bad and loss_bad and r is not None and loss is not None
-
-        if bad:
-            self._streak += 1
-        else:
-            self._streak = 0
-
-        if self._streak >= self.bad_streak:
-            print(
-                f"[STABILITY] stopping: sustained instability "
-                f"(best {self.reward_key}={self._best_reward}, best loss={self._best_loss}, streak={self._streak})."
-            )
-            control.should_training_stop = True
-        return control
-
-
-class LossSpikeTripwire(TrainerCallback):
-    """SFT guardrail: stop if loss repeatedly blows up vs the best-so-far."""
-
-    def __init__(self, *, min_step: int, loss_key: str, loss_spike: float, bad_streak: int) -> None:
-        self.min_step = min_step
-        self.loss_key = loss_key
-        self.loss_spike = loss_spike
-        self.bad_streak = bad_streak
-        self._best_loss: float | None = None
-        self._streak = 0
-
-    def on_log(self, args, state, control, logs=None, **kw):  # type: ignore[no-untyped-def]
-        logs = logs or {}
-        step = int(getattr(state, "global_step", 0) or 0)
-        if step < self.min_step:
-            return control
-
-        loss = _as_float(logs.get(self.loss_key))
-        if loss is None:
-            return control
-
-        if self._best_loss is None or loss < self._best_loss:
-            self._best_loss = loss
-            self._streak = 0
-            return control
-
-        if self._best_loss is not None and loss - self._best_loss >= self.loss_spike:
-            self._streak += 1
-        else:
-            self._streak = 0
-
-        if self._streak >= self.bad_streak:
-            print(f"[STABILITY] stopping SFT: repeated loss spikes (best={self._best_loss}, streak={self._streak}).")
-            control.should_training_stop = True
-        return control
-
-
 def _extract_briefing(reset_payload: dict[str, Any]) -> str:
     obs = reset_payload.get("observation", reset_payload)
     if isinstance(obs, dict):
@@ -251,7 +142,6 @@ def run_sft_then_grpo(
     grpo_grad_accum: int,
     grpo_beta: float,
     reward_ema_decay: float,
-    stability_tripwire: bool,
 ) -> None:
     try:
         from datasets import load_dataset
@@ -315,16 +205,6 @@ def run_sft_then_grpo(
         dataset_text_field="prompt",
         formatting_func=lambda ex: [f"{p}\n\n{c}" for p, c in zip(ex["prompt"], ex["completion"])],
     )
-    if stability_tripwire:
-        sft_trainer.add_callback(
-            LossSpikeTripwire(
-                min_step=max(10, max_sft_steps // 6),
-                loss_key="loss",
-                loss_spike=0.85,
-                bad_streak=4,
-            )
-        )
-
     sft_before = _trainable_lora_sum_abs(policy)
     sft_trainer.train()
     sft_after = _trainable_lora_sum_abs(sft_trainer.model)
@@ -482,25 +362,13 @@ def run_sft_then_grpo(
         adam_beta2=0.95,
         report_to=[],
     )
-    grpo_callbacks = [_ProgressCallback()]
-    if stability_tripwire:
-        grpo_callbacks.append(
-            StabilityTripwire(
-                min_step=max(15, max_grpo_steps // 8),
-                reward_key="rewards/env_reward/mean",
-                loss_key="loss",
-                reward_drop=0.12,
-                loss_spike=0.35,
-                bad_streak=3,
-            )
-        )
     grpo_trainer = GRPOTrainer(
         model=sft_trainer.model,
         processing_class=tokenizer,
         reward_funcs=[env_reward, format_reward, semantic_action_reward, anti_idle_reward],
         train_dataset=ds,
         args=grpo_cfg,
-        callbacks=grpo_callbacks,
+        callbacks=[_ProgressCallback()],
     )
     grpo_before = _trainable_lora_sum_abs(sft_trainer.model)
     grpo_trainer.train()
@@ -556,11 +424,6 @@ def main() -> None:
         default=0.60,
         help="Fraction of GRPO steps used to ramp from easy scaffold to full env weighting.",
     )
-    parser.add_argument(
-        "--no-stability-tripwire",
-        action="store_true",
-        help="Disable oscillation/collapse early-stop guardrails (not recommended).",
-    )
     parser.add_argument(
         "--reward-ema-decay",
         type=float,
@@ -599,7 +462,6 @@ def main() -> None:
         sft_samples = args.sft_samples
     if args.reward_ema_decay >= 0.0:
         reward_ema_decay = float(args.reward_ema_decay)
-    stability_tripwire = not args.no_stability_tripwire
     print(f"Model preset: {args.model_preset} -> {model_name}")
     print(
         "Training preset:"
@@ -633,7 +495,6 @@ def main() -> None:
         grpo_grad_accum=grpo_grad_accum,
         grpo_beta=grpo_beta,
         reward_ema_decay=reward_ema_decay,
-        stability_tripwire=stability_tripwire,
     )