task1, task2 evaluated

agents/task1.py

@@ -0,0 +1,113 @@
+"""Agents for Task 1: Function + Vulnerability Identification."""
+
+import random as _random
+from typing import Any, Dict, List
+
+from tasks.task1 import Task1Environment
+from env.schemas import Action, ActionType
+from data.data_loader import load_contracts, get_function_by_name
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Helpers
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _parse_fn_list(result_text: str) -> List[str]:
+    """Parse 'Functions in X: f1, f2, f3' into [f1, f2, f3]."""
+    if ": " in result_text:
+        return [f.strip() for f in result_text.split(": ", 1)[-1].split(", ") if f.strip()]
+    return []
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Task 1 agents
+# ─────────────────────────────────────────────────────────────────────────────
+
+def oracle_t1(env: Task1Environment, seed: int, verbose: bool = False) -> Dict[str, Any]:
+    """Submits exact ground-truth function + vulnerability type → score = 1.0."""
+    r        = env.reset(seed=seed)
+    obs      = r.observation
+    fn_name  = env.state().target_function
+    contracts = load_contracts()
+    vuln_issue = ""
+    for c in contracts:
+        fn = get_function_by_name(c, fn_name)
+        if fn and fn.get("vulnerable"):
+            vuln_issue = fn["vulnerability_details"]["issue"]
+            break
+    if verbose:
+        print(f"    {obs.contract_name}.{fn_name}()  [{vuln_issue}]")
+    env.step(Action(action_type=ActionType.LIST_FUNCTIONS))
+    env.step(Action(action_type=ActionType.GET_FUNCTION_CODE,
+                    params={"function_name": fn_name}))
+    result = env.step(Action(action_type=ActionType.SUBMIT,
+                              params={"function_name": fn_name,
+                                      "vulnerability_type": vuln_issue}))
+    v = result.reward.value
+    score = 1.0 if v >= 4.9 else (0.5 if v >= 0.9 else 0.0)
+    return {"seed": seed, "contract": obs.contract_name, "target_function": fn_name,
+            "vulnerability": vuln_issue, "grader_score": score,
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def partial_t1(env: Task1Environment, seed: int) -> Dict[str, Any]:
+    """Correct function, 'unknown' vuln type → score = 0.5."""
+    env.reset(seed=seed)
+    fn_name = env.state().target_function
+    result  = env.step(Action(action_type=ActionType.SUBMIT,
+                               params={"function_name": fn_name, "vulnerability_type": "unknown"}))
+    v = result.reward.value
+    return {"seed": seed, "grader_score": 0.5 if v >= 0.9 else 0.0,
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def random_t1(env: Task1Environment, seed: int) -> Dict[str, Any]:
+    """Genuine random agent: random browse then submits a random function + random vuln type.
+    
+    Uses a seeded RNG (offset from episode seed) so results are reproducible.
+    Expected score: low (~0–5%) since must randomly hit both right function and right keyword, plus
+    can submit only once per episode.
+    """
+    rng = _random.Random(seed ^ 0x5A1AD)  # different RNG stream from episode seed
+    env.reset(seed=seed)
+
+    # Step 1: list functions to get real candidates
+    s = env.step(Action(action_type=ActionType.LIST_FUNCTIONS))
+    fns = _parse_fn_list(s.observation.last_action_result or "")
+    if not fns:
+        fns = ["deposit", "withdraw", "constructor"]  # fallback
+
+    # Step 2: do 1–2 random browse actions (not repeated)
+    browse_pool = [
+        (ActionType.GET_FILE_METADATA,  {}),
+        (ActionType.GET_CALL_GRAPH,     {}),
+        (ActionType.GET_STATE_VARIABLE, {}),
+    ]
+    _random.Random(seed).shuffle(browse_pool)  # deterministic order
+    for at, params in browse_pool[:rng.randint(1, 2)]:
+        env.step(Action(action_type=at, params=params))
+
+    # Step 3: submit a random function from the real list, random vuln type
+    random_fn = rng.choice(fns)
+    vuln_pool  = [
+        "bad logic", "incorrect check", "overflow", "no guard", "wrong order",
+        "missing event", "unprotected", "stale data", "unsafe cast",
+    ]
+    random_vuln = rng.choice(vuln_pool)
+    result = env.step(Action(action_type=ActionType.SUBMIT,
+                              params={"function_name": random_fn,
+                                      "vulnerability_type": random_vuln}))
+    v = result.reward.value
+    score = 1.0 if v >= 4.9 else (0.5 if v >= 0.9 else 0.0)
+    return {"seed": seed, "grader_score": score, "submitted_fn": random_fn,
+            "submitted_vuln": random_vuln,
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def floor_t1(env: Task1Environment, seed: int) -> Dict[str, Any]:
+    """Always submits 'constructor' → guaranteed score = 0.0."""
+    env.reset(seed=seed)
+    result = env.step(Action(action_type=ActionType.SUBMIT,
+                              params={"function_name": "constructor",
+                                      "vulnerability_type": "reentrancy"}))
+    return {"seed": seed, "grader_score": 0.0,
+            "cumulative_reward": result.observation.cumulative_reward}

agents/task2.py

@@ -0,0 +1,105 @@
+"""Agents for Task2: Property Discovery"""
+
+import random as _random
+from typing import Any, Dict, List
+
+from tasks.task2 import Task2Environment
+from env.schemas import Action, ActionType
+from data.data_loader import load_contracts, get_function_by_name
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Task 2 agents
+# ─────────────────────────────────────────────────────────────────────────────
+
+def oracle_t2(env: Task2Environment, seed: int, verbose: bool = False) -> Dict[str, Any]:
+    """Submits ground-truth in natural langugage (English) → score ≥ 0.70."""
+    r        = env.reset(seed=seed)
+    obs      = r.observation
+    fn_name  = obs.extra["target_function"]
+    contract = obs.contract_name
+    contracts = load_contracts()
+    gt_text = ""
+    for c in contracts:
+        if c["contract_name"] == contract:
+            fn = get_function_by_name(c, fn_name)
+            if fn and fn.get("property"):
+                gt_text = fn["property"]
+            break
+    if verbose:
+        print(f"    {contract}.{fn_name}()")
+    env.step(Action(action_type=ActionType.GET_FUNCTION_CODE))
+    result = env.step(Action(action_type=ActionType.SUBMIT_PROPERTY,
+                              params={"property": gt_text}))
+    r_val  = result.reward.value
+    score  = round(r_val / 5.0, 4) if r_val > 0 else 0.0
+    return {"seed": seed, "contract": contract, "function": fn_name,
+            "grader_score": score,
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def partial_t2(env: Task2Environment, seed: int) -> Dict[str, Any]:
+    """Submits only the function's short NatSpec comment — partial credit."""
+    r   = env.reset(seed=seed)
+    obs = r.observation
+    contracts = load_contracts()
+    comment = ""
+    for c in contracts:
+        if c["contract_name"] == obs.contract_name:
+            fn = get_function_by_name(c, obs.extra["target_function"])
+            if fn:
+                comment = fn.get("comment", "")
+            break
+    result = env.step(Action(action_type=ActionType.SUBMIT_PROPERTY,
+                              params={"property": comment}))
+    r_val = result.reward.value
+    return {"seed": seed, "grader_score": round(r_val / 5.0, 4) if r_val > 0 else 0.0,
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def random_t2(env: Task2Environment, seed: int) -> Dict[str, Any]:
+    """Genuine random agent: random browse then submits a generic property template.
+
+    The submitted text contains high-frequency words that are unlikely to match
+    task-specific key phrases. Expected score: near 0 (coincidental matches only).
+    Uses a seeded RNG for reproducibility.
+    """
+    rng = _random.Random(seed ^ 0xBEEF1)
+
+    r   = env.reset(seed=seed)
+    obs = r.observation
+    fn_name = obs.extra.get("target_function", "this function")
+
+    # Random browse: pick 1–2 actions at random
+    browse_pool = [
+        ActionType.GET_FILE_NATSPEC,
+        ActionType.GET_RELATED_FUNCTIONS,
+        ActionType.GET_SIGNATURE,
+    ]
+    rng.shuffle(browse_pool)
+    for at in browse_pool[:rng.randint(1, 2)]:
+        env.step(Action(action_type=at))
+
+    # Submit a randomly assembled generic property (won't match specific key phrases)
+    templates = [
+        f"The {fn_name} operation completes the intended computation on the input data.",
+        f"When {fn_name} executes, it processes the provided arguments and updates the contract.",
+        f"The {fn_name} function validates inputs and performs the expected operation.",
+        f"Calling {fn_name} causes the contract to execute its designated logic.",
+        f"{fn_name} runs when invoked and modifies internal state as designed.",
+    ]
+    prop = rng.choice(templates)
+    result = env.step(Action(action_type=ActionType.SUBMIT_PROPERTY,
+                              params={"property": prop}))
+    r_val = result.reward.value
+    return {"seed": seed, "grader_score": round(r_val / 5.0, 4) if r_val > 0 else 0.0,
+            "submitted": prop[:60],
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def floor_t2(env: Task2Environment, seed: int) -> Dict[str, Any]:
+    """Submits empty string → score = 0.0 guaranteed."""
+    env.reset(seed=seed)
+    result = env.step(Action(action_type=ActionType.SUBMIT_PROPERTY,
+                              params={"property": ""}))
+    return {"seed": seed, "grader_score": 0.0,
+            "cumulative_reward": result.observation.cumulative_reward}

agents/task3.py

@@ -0,0 +1,106 @@
+""" Agents for Task3 : Rule Checking for a function """
+
+import json
+import random as _random
+from typing import Any, Dict, List
+
+from tasks.task3 import Task3Environment
+from env.schemas import Action, ActionType
+from data.data_loader import load_contracts, get_function_by_name
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Helpers
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _parse_fn_list(result_text: str) -> List[str]:
+    """Parse 'Functions in X: f1, f2, f3' into [f1, f2, f3]."""
+    if ": " in result_text:
+        return [f.strip() for f in result_text.split(": ", 1)[-1].split(", ") if f.strip()]
+    return []
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Task 3 agents
+# ─────────────────────────────────────────────────────────────────────────────
+
+def oracle_t3(env: Task3Environment, seed: int, verbose: bool = False) -> Dict[str, Any]:
+    """Submits exact target function → score = 1.0."""
+    r        = env.reset(seed=seed)
+    obs      = r.observation
+    fn_name  = env.state().target_function
+    contract = obs.contract_name
+    if verbose:
+        prop = obs.extra.get("property_english", "")[:60]
+        print(f"    {contract}.{fn_name}()  \"{prop}\"")
+    env.step(Action(action_type=ActionType.GET_PROPERTY_SPECIFICATION))
+    env.step(Action(action_type=ActionType.LIST_FUNCTIONS))
+    result = env.step(Action(action_type=ActionType.SUBMIT_FUNCTION,
+                              params={"function_name": fn_name}))
+    v = result.reward.value
+    score = 1.0 if v >= 4.9 else (0.3 if v >= 1.0 else 0.0)
+    return {"seed": seed, "contract": contract, "target_function": fn_name,
+            "grader_score": score,
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def subfunction_t3(env: Task3Environment, seed: int) -> Dict[str, Any]:
+    """Submits the first partial-credit subfunction if one exists, else 'constructor'."""
+    r   = env.reset(seed=seed)
+    obs = r.observation
+    contracts = load_contracts()
+    partial_fns = []
+    for c in contracts:
+        if c["contract_name"] == obs.contract_name:
+            fn = get_function_by_name(c, env.state().target_function)
+            if fn:
+                partial_fns = fn.get("task3", {}).get("partial_credit_functions", [])
+            break
+    submit_name = partial_fns[0] if partial_fns else "constructor"
+    result = env.step(Action(action_type=ActionType.SUBMIT_FUNCTION,
+                              params={"function_name": submit_name}))
+    v = result.reward.value
+    score = 1.0 if v >= 4.9 else (0.3 if v >= 1.0 else 0.0)
+    return {"seed": seed, "grader_score": score, "submitted": submit_name,
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def random_t3(env: Task3Environment, seed: int) -> Dict[str, Any]:
+    """Genuine random agent: lists functions, picks one at random, submits.
+
+    With N functions per contract and 1 target, expected score ≈ 1/N ≈ 0.20–0.25.
+    Uses a seeded RNG for reproducibility.
+    """
+    rng = _random.Random(seed ^ 0xCAFE1)
+    env.reset(seed=seed)
+
+    # Step 1: get function list (necessary to pick a real candidate)
+    s   = env.step(Action(action_type=ActionType.LIST_FUNCTIONS))
+    fns = _parse_fn_list(s.observation.last_action_result or "")
+    if not fns:
+        fns = ["constructor"]
+
+    # Step 2: optionally do 1 cheap browse action (formalized or call_graph)
+    browse_options = [
+        (ActionType.GET_PROPERTY_SPECIFICATION, {}),
+        (ActionType.GET_CALL_GRAPH,          {}),
+    ]
+    at, params = rng.choice(browse_options)
+    env.step(Action(action_type=at, params=params))
+
+    # Step 3: submit a uniformly random function from the real list
+    chosen = rng.choice(fns)
+    result = env.step(Action(action_type=ActionType.SUBMIT_FUNCTION,
+                              params={"function_name": chosen}))
+    v = result.reward.value
+    score = 1.0 if v >= 4.9 else (0.3 if v >= 1.0 else 0.0)
+    return {"seed": seed, "grader_score": score, "submitted": chosen,
+            "cumulative_reward": result.observation.cumulative_reward}
+
+
+def floor_t3(env: Task3Environment, seed: int) -> Dict[str, Any]:
+    """Always submits 'constructor' → guaranteed score = 0.0."""
+    env.reset(seed=seed)
+    result = env.step(Action(action_type=ActionType.SUBMIT_FUNCTION,
+                              params={"function_name": "constructor"}))
+    return {"seed": seed, "grader_score": 0.0,
+            "cumulative_reward": result.observation.cumulative_reward}

app.py

@@ -22,9 +22,9 @@ from fastapi import FastAPI, HTTPException, Query
 from pydantic import BaseModel
 
 from env.schemas import Action, ActionType, TaskInfo
-from tasks.task1.environment import Task1Environment
-from tasks.task2.environment import Task2Environment
-from tasks.task3.environment import Task3Environment
+from tasks.task1 import Task1Environment
+from tasks.task2 import Task2Environment
+from tasks.task3 import Task3Environment
 
 # ─────────────────────────────────────────────────────────────────────────────
 # App
@@ -191,7 +191,7 @@ def action_space(task_id: str = "task1_vuln_detection"):
                 {"type": "get_function_natspec",  "params": {},                         "reward": -0.08, "description": "Read NatSpec + expected behaviour"},
                 {"type": "get_file_natspec",      "params": {},                         "reward": -0.03, "description": "Read contract-level NatSpec"},
                 {"type": "get_related_functions", "params": {},                         "reward": -0.06, "description": "List caller/callee functions with summaries"},
-                {"type": "get_io",                "params": {},                         "reward": -0.04, "description": "Get structured I/O + expected behaviour"},
+                {"type": "get_signature",                "params": {},                         "reward": -0.04, "description": "Get structured I/O + expected behaviour"},
                 {"type": "get_similar_rule",      "params": {},                         "reward": -0.20, "description": "Get a similar property from another contract"},
                 {"type": "submit_property",       "params": {"property": "string"},     "reward": "0.0–5.0 (scored)", "description": "Submit property. ONE attempt. Ends episode."},
             ],

data/contracts.json

@@ -4827,7 +4827,7 @@
         ]
     },
     {
-        "contract_name": "StableDebtToken",
+        "contract_name": "StableDebtToken_OLD",
         "file_name": "StableDebtToken.sol",
         "metadata": {
             "license": "agpl-3.0",
@@ -5380,7 +5380,7 @@
         ]
     },
     {
-        "contract_name": "ATokenVault",
+        "contract_name": "ATokenVault_OLD",
         "file_name": "ATokenVault_old.sol",
         "metadata": {
             "license": "MIT",

data/data_loader.py

@@ -15,7 +15,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 DATA_DIR = os.path.join(os.path.dirname(__file__))
 DEFAULT_CONTRACTS_FILE = os.path.join(DATA_DIR, "contracts.json")
-DEFAULT_VUNERABILITIES_FILE = os.path.join(DATA_DIR, "vulnerabilities.json")
+DEFAULT_CSV_PATH = os.path.join(DATA_DIR, "properties.csv")
 
 
 # ────────────────────────────────────────────────────────────────
@@ -62,11 +62,6 @@ def list_state_variable_names(contract: Dict[str, Any]) -> List[str]:
 # Task 1 helpers
 # ────────────────────────────────────────────────────────────────
 
-def load_vulnerabilities(path: str = DEFAULT_VUNERABILITIES_FILE) -> List[Dict[str, Any]]:
-    """Load and return all vulnerability entries from the JSON dataset."""
-    with open(path, "r") as f:
-        return json.load(f)
-
 def get_all_vulnerable_entries(
     contracts: List[Dict[str, Any]],
 ) -> List[Tuple[Dict[str, Any], Dict[str, Any]]]:

env/schemas.py

@@ -130,7 +130,7 @@ class ResetResult(BaseModel):
 class StateResult(BaseModel):
     task_id: str
     contract_name: str
-    target_function: Optional[str] = None   # hidden in real eval, exposed here for debugging
+    target_function: str   # hidden in real eval, exposed here for debugging
     step_count: int
     cumulative_reward: float
     done: bool

eval.py

@@ -3,197 +3,44 @@ eval.py
 -------
 Evaluation harness for all three tasks.
 
-Runs oracle / partial / baseline agents, verifying score orderings and
-that reward shaping is meaningful across the trajectory.
+Runs four agent tiers per task:
+  oracle     – always submits the ground-truth answer       (upper bound)
+  partial    – right category, wrong detail                 (partial credit)
+  random     – genuine random exploration + random submit   (random baseline)
+  floor      – always submits a guaranteed-wrong answer     (lower bound)
 
 Usage:
-  python eval.py                        # all tasks, 8 episodes each
-  python eval.py --task 1|2|3           # single task
+  python eval.py                          # all tasks, 8 episodes each
+  python eval.py --task 1|2|3             # single task
   python eval.py --episodes 16 --verbose
   python eval.py --out results.json
 """
 
 import argparse
 import json
+import random as _random
 from typing import Any, Dict, List
 
-from tasks.task1.environment import Task1Environment
-from tasks.task2.environment import Task2Environment
-from tasks.task3.environment import Task3Environment
-from env.schemas import Action, ActionType
+from tasks.task1 import Task1Environment
+from tasks.task2 import Task2Environment
+from tasks.task3 import Task3Environment
+from agents.task1 import oracle_t1, partial_t1, random_t1, floor_t1
+from agents.task2 import oracle_t2, partial_t2, random_t2, floor_t2
+from agents.task3 import oracle_t3, subfunction_t3, random_t3, floor_t3
 from data.data_loader import (
     load_contracts,
-    get_function_by_name,
     get_all_vulnerable_entries,
     get_all_property_entries,
     get_all_task3_entries,
 )
 
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Task 1 agents
-# ─────────────────────────────────────────────────────────────────────────────
-
-def oracle_t1(env: Task1Environment, seed: int, verbose: bool = False) -> Dict[str, Any]:
-    """Submits the exact ground-truth function + vulnerability → score = 1.0."""
-    r   = env.reset(seed=seed)
-    obs = r.observation
-    fn_name = env.state().target_function
-    contracts = load_contracts()
-    vuln_issue = ""
-    for c in contracts:
-        fn = get_function_by_name(c, fn_name)
-        if fn and fn.get("vulnerable"):
-            vuln_issue = fn["vulnerability_details"]["issue"]
-            break
-    if verbose:
-        print(f"    {obs.contract_name}.{fn_name}()  [{vuln_issue}]")
-    env.step(Action(action_type=ActionType.LIST_FUNCTIONS))
-    env.step(Action(action_type=ActionType.GET_FUNCTION_CODE,
-                    params={"function_name": fn_name}))
-    result = env.step(Action(action_type=ActionType.SUBMIT,
-                              params={"function_name": fn_name,
-                                      "vulnerability_type": vuln_issue}))
-    v = result.reward.value
-    score = 1.0 if v >= 4.9 else (0.5 if v >= 0.9 else 0.0)
-    return {"seed": seed, "contract": obs.contract_name, "target_function": fn_name,
-            "vulnerability": vuln_issue, "grader_score": score,
-            "cumulative_reward": result.observation.cumulative_reward}
-
-
-def partial_t1(env: Task1Environment, seed: int) -> Dict[str, Any]:
-    """Right function, 'unknown' vuln type → score = 0.5."""
-    env.reset(seed=seed)
-    fn_name = env.state().target_function
-    result = env.step(Action(action_type=ActionType.SUBMIT,
-                              params={"function_name": fn_name, "vulnerability_type": "unknown"}))
-    v = result.reward.value
-    return {"seed": seed, "grader_score": 0.5 if v >= 0.9 else 0.0,
-            "cumulative_reward": result.observation.cumulative_reward}
-
-
-def wrong_t1(env: Task1Environment, seed: int) -> Dict[str, Any]:
-    """Always submits 'constructor' → score = 0.0."""
-    env.reset(seed=seed)
-    result = env.step(Action(action_type=ActionType.SUBMIT,
-                              params={"function_name": "constructor",
-                                      "vulnerability_type": "reentrancy"}))
-    return {"seed": seed, "grader_score": 0.0,
-            "cumulative_reward": result.observation.cumulative_reward}
-
-
 # ─────────────────────────────────────────────────────────────────────────────
-# Task 2 agents
-# ─────────────────────────────────────────────────────────────────────────────
-
-def oracle_t2(env: Task2Environment, seed: int, verbose: bool = False) -> Dict[str, Any]:
-    """Submits ground-truth natural_language → score ≥ 0.70."""
-    r   = env.reset(seed=seed)
-    obs = r.observation
-    fn_name  = obs.extra["target_function"]
-    contract = obs.contract_name
-    contracts = load_contracts()
-    gt_text = ""
-    for c in contracts:
-        if c["contract_name"] == contract:
-            fn = get_function_by_name(c, fn_name)
-            if fn and fn.get("property"):
-                gt_text = fn["property"]["natural_language"]
-            break
-    if verbose:
-        print(f"    {contract}.{fn_name}()")
-    env.step(Action(action_type=ActionType.GET_FUNCTION_CODE))
-    result = env.step(Action(action_type=ActionType.SUBMIT_PROPERTY,
-                              params={"property": gt_text}))
-    r_val = result.reward.value
-    score = round(r_val / 5.0, 4) if r_val > 0 else 0.0
-    return {"seed": seed, "contract": contract, "function": fn_name,
-            "grader_score": score, "cumulative_reward": result.observation.cumulative_reward}
-
-
-def partial_t2(env: Task2Environment, seed: int) -> Dict[str, Any]:
-    """Submits the function's NatSpec comment — partial credit."""
-    r   = env.reset(seed=seed)
-    obs = r.observation
-    contracts = load_contracts()
-    comment = ""
-    for c in contracts:
-        if c["contract_name"] == obs.contract_name:
-            fn = get_function_by_name(c, obs.extra["target_function"])
-            if fn:
-                comment = fn.get("comment", "")
-            break
-    result = env.step(Action(action_type=ActionType.SUBMIT_PROPERTY,
-                              params={"property": comment}))
-    r_val = result.reward.value
-    return {"seed": seed, "grader_score": round(r_val / 5.0, 4) if r_val > 0 else 0.0,
-            "cumulative_reward": result.observation.cumulative_reward}
-
-
-def empty_t2(env: Task2Environment, seed: int) -> Dict[str, Any]:
-    """Submits empty string → score = 0.0."""
-    env.reset(seed=seed)
-    result = env.step(Action(action_type=ActionType.SUBMIT_PROPERTY, params={"property": ""}))
-    return {"seed": seed, "grader_score": 0.0,
-            "cumulative_reward": result.observation.cumulative_reward}
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Task 3 agents
+# Evaluation runners
 # ─────────────────────────────────────────────────────────────────────────────
 
-def oracle_t3(env: Task3Environment, seed: int, verbose: bool = False) -> Dict[str, Any]:
-    """Always submits the exact target function → score = 1.0."""
-    r   = env.reset(seed=seed)
-    obs = r.observation
-    fn_name  = env.state().target_function
-    contract = obs.contract_name
-    if verbose:
-        prop = obs.extra.get("property_english", "")[:60]
-        print(f"    {contract}.{fn_name}()  \"{prop}\"")
-    env.step(Action(action_type=ActionType.GET_PROPERTY_SPECIFICATION))
-    env.step(Action(action_type=ActionType.LIST_FUNCTIONS))
-    result = env.step(Action(action_type=ActionType.SUBMIT_FUNCTION,
-                              params={"function_name": fn_name}))
-    v = result.reward.value
-    score = 1.0 if v >= 4.9 else (0.3 if v >= 1.0 else 0.0)
-    return {"seed": seed, "contract": contract, "target_function": fn_name,
-            "grader_score": score, "cumulative_reward": result.observation.cumulative_reward}
-
-
-def subfunction_t3(env: Task3Environment, seed: int) -> Dict[str, Any]:
-    """Submits the first partial-credit subfunction if it exists, else 'constructor'."""
-    r   = env.reset(seed=seed)
-    obs = r.observation
-    contracts = load_contracts()
-    partial_fns = []
-    for c in contracts:
-        if c["contract_name"] == obs.contract_name:
-            fn = get_function_by_name(c, env.state().target_function)
-            if fn:
-                partial_fns = fn.get("task3", {}).get("partial_credit_functions", [])
-            break
-    submit_name = partial_fns[0] if partial_fns else "constructor"
-    result = env.step(Action(action_type=ActionType.SUBMIT_FUNCTION,
-                              params={"function_name": submit_name}))
-    v = result.reward.value
-    score = 1.0 if v >= 4.9 else (0.3 if v >= 1.0 else 0.0)
-    return {"seed": seed, "grader_score": score, "submitted": submit_name,
-            "cumulative_reward": result.observation.cumulative_reward}
-
-
-def wrong_t3(env: Task3Environment, seed: int) -> Dict[str, Any]:
-    """Always submits 'constructor' → score = 0.0."""
-    env.reset(seed=seed)
-    result = env.step(Action(action_type=ActionType.SUBMIT_FUNCTION,
-                              params={"function_name": "constructor"}))
-    return {"seed": seed, "grader_score": 0.0,
-            "cumulative_reward": result.observation.cumulative_reward}
-
+def _avg(episodes: List[Dict[str, Any]], key: str = "grader_score") -> float:
+    return sum(e[key] for e in episodes) / len(episodes) if episodes else 0.0
 
-# ─────────────────────────────────────────────────────────────────────────────
-# Evaluation runners
-# ─────────────────────────────────────────────────────────────────────────────
 
 def run_task1_eval(n: int, seed_offset: int, verbose: bool) -> Dict[str, Any]:
     print("\n" + "=" * 64)
@@ -204,27 +51,37 @@ def run_task1_eval(n: int, seed_offset: int, verbose: bool) -> Dict[str, Any]:
           f"{len(get_all_vulnerable_entries(contracts))} vulnerable functions\n")
     env = Task1Environment()
 
-    print("▶ Oracle (correct function + correct vuln type → 1.0):")
+    # Oracle
+    print("▶ Oracle  (correct function + correct vuln → 1.0):")
     oracle_eps = []
     for i in range(n):
         ep = oracle_t1(env, seed_offset + i, verbose)
         oracle_eps.append(ep)
         print(f"  seed={ep['seed']:3d}  {ep['contract']:12s}.{ep['target_function']:18s}"
               f"  score={ep['grader_score']:.1f}  reward={ep['cumulative_reward']:+.2f}")
-    oracle_avg   = sum(e["grader_score"] for e in oracle_eps) / n
-    oracle_avg_r = sum(e["cumulative_reward"] for e in oracle_eps) / n
-    print(f"\n  Oracle avg: {oracle_avg:.3f}  reward: {oracle_avg_r:+.2f}")
+    oracle_avg = _avg(oracle_eps)
+    print(f"\n  Oracle   avg: {oracle_avg:.3f}")
 
-    print("\n▶ Partial (right function, wrong vuln → 0.5):")
+    # Partial
+    print("\n▶ Partial (correct function, 'unknown' vuln → 0.5):")
     partial_eps = [partial_t1(env, seed_offset + i) for i in range(n)]
-    partial_avg = sum(e["grader_score"] for e in partial_eps) / n
-    print(f"  Partial avg: {partial_avg:.3f}")
-
-    print("\n▶ Wrong (always 'constructor' → 0.0):")
-    wrong_eps = [wrong_t1(env, seed_offset + i) for i in range(n)]
-    wrong_avg = sum(e["grader_score"] for e in wrong_eps) / n
-    print(f"  Wrong avg: {wrong_avg:.3f}")
-
+    partial_avg = _avg(partial_eps)
+    print(f"  Partial  avg: {partial_avg:.3f}")
+
+    # Random
+    print("\n▶ Random  (random fn from list + random vuln type):")
+    random_eps = [random_t1(env, seed_offset + i) for i in range(n)]
+    random_avg = _avg(random_eps)
+    submitted  = [(e.get("submitted_fn", "?"), e.get("submitted_vuln", "?")) for e in random_eps]
+    print(f"  Random   avg: {random_avg:.3f}  submissions: {submitted}")
+
+    # Floor
+    print("\n▶ Floor   (always 'constructor' → 0.0):")
+    floor_eps = [floor_t1(env, seed_offset + i) for i in range(n)]
+    floor_avg = _avg(floor_eps)
+    print(f"  Floor    avg: {floor_avg:.3f}")
+
+    # Vulnerability type coverage
     vuln_seen: Dict[str, int] = {}
     for ep in oracle_eps:
         v = ep.get("vulnerability", "unknown")
@@ -233,16 +90,19 @@ def run_task1_eval(n: int, seed_offset: int, verbose: bool) -> Dict[str, Any]:
     for v in sorted(vuln_seen):
         print(f"  {vuln_seen[v]:2d}×  {v}")
 
-    assert oracle_avg == 1.0
-    assert partial_avg == 0.5
-    assert wrong_avg == 0.0
-    print("\n  ✅ Task 1: oracle(1.0) > partial(0.5) > wrong(0.0)")
+    assert oracle_avg == 1.0,  f"Oracle avg {oracle_avg:.3f} should be 1.0"
+    assert partial_avg == 0.5, f"Partial avg {partial_avg:.3f} should be 0.5"
+    assert floor_avg == 0.0,   f"Floor avg {floor_avg:.3f} should be 0.0"
+    assert oracle_avg >= random_avg >= floor_avg, \
+        f"Score ordering violated: oracle={oracle_avg}, random={random_avg}, floor={floor_avg}"
+    print(f"\n  ✅ Task 1: oracle(1.0) ≥ partial(0.5) ≥ random({random_avg:.3f}) ≥ floor(0.0)")
 
     return {
         "task_id": "task1_vuln_detection",
-        "oracle":  {"avg_score": oracle_avg,  "avg_reward": oracle_avg_r, "episodes": oracle_eps},
+        "oracle":  {"avg_score": oracle_avg,  "episodes": oracle_eps},
         "partial": {"avg_score": partial_avg, "episodes": partial_eps},
-        "wrong":   {"avg_score": wrong_avg,   "episodes": wrong_eps},
+        "random":  {"avg_score": random_avg,  "episodes": random_eps},
+        "floor":   {"avg_score": floor_avg,   "episodes": floor_eps},
         "vuln_coverage": vuln_seen,
     }
 
@@ -255,7 +115,8 @@ def run_task2_eval(n: int, seed_offset: int, verbose: bool) -> Dict[str, Any]:
     print(f"  Dataset: {len(get_all_property_entries(contracts))} property entries\n")
     env = Task2Environment()
 
-    print("▶ Oracle (submits ground-truth natural language):")
+    # Oracle
+    print("▶ Oracle  (submits ground-truth natural language):")
     oracle_eps = []
     for i in range(n):
         ep = oracle_t2(env, seed_offset + i, verbose)
@@ -263,31 +124,40 @@ def run_task2_eval(n: int, seed_offset: int, verbose: bool) -> Dict[str, Any]:
         icon = "✅" if ep["grader_score"] >= 0.65 else "⚠️ "
         print(f"  {icon} seed={ep['seed']:3d}  {ep['contract']:12s}.{ep['function']:18s}"
               f"  score={ep['grader_score']:.3f}  reward={ep['cumulative_reward']:+.2f}")
-    oracle_avg   = sum(e["grader_score"] for e in oracle_eps) / n
-    oracle_avg_r = sum(e["cumulative_reward"] for e in oracle_eps) / n
-    print(f"\n  Oracle avg: {oracle_avg:.3f}  reward: {oracle_avg_r:+.2f}")
+    oracle_avg = _avg(oracle_eps)
+    print(f"\n  Oracle   avg: {oracle_avg:.3f}")
 
+    # Partial
     print("\n▶ Partial (submits NatSpec comment):")
-    partial_eps   = [partial_t2(env, seed_offset + i) for i in range(n)]
-    partial_avg   = sum(e["grader_score"] for e in partial_eps) / n
-    partial_avg_r = sum(e["cumulative_reward"] for e in partial_eps) / n
-    print(f"  Partial avg: {partial_avg:.3f}  reward: {partial_avg_r:+.2f}")
-
-    print("\n▶ Empty (submits nothing → 0.0):")
-    empty_eps = [empty_t2(env, seed_offset + i) for i in range(n)]
-    empty_avg = sum(e["grader_score"] for e in empty_eps) / n
-    print(f"  Empty avg: {empty_avg:.3f}")
-
-    assert oracle_avg > 0.60
-    assert oracle_avg > partial_avg
-    assert empty_avg == 0.0
-    print(f"\n  ✅ Task 2: oracle({oracle_avg:.3f}) > partial({partial_avg:.3f}) > empty(0.0)")
+    partial_eps = [partial_t2(env, seed_offset + i) for i in range(n)]
+    partial_avg = _avg(partial_eps)
+    print(f"  Partial  avg: {partial_avg:.3f}")
+
+    # Random
+    print("\n▶ Random  (random browse + generic property template):")
+    random_eps = [random_t2(env, seed_offset + i) for i in range(n)]
+    random_avg = _avg(random_eps)
+    print(f"  Random   avg: {random_avg:.3f}")
+
+    # Floor
+    print("\n▶ Floor   (submits empty string → 0.0):")
+    floor_eps = [floor_t2(env, seed_offset + i) for i in range(n)]
+    floor_avg = _avg(floor_eps)
+    print(f"  Floor    avg: {floor_avg:.3f}")
+
+    assert oracle_avg > 0.60,  f"Oracle avg {oracle_avg:.3f} should be > 0.60"
+    assert oracle_avg > partial_avg >= floor_avg, \
+        "Score ordering violated: oracle > partial >= floor"
+    assert floor_avg == 0.0, f"Floor avg {floor_avg:.3f} should be 0.0"
+    print(f"\n  ✅ Task 2: oracle({oracle_avg:.3f}) > partial({partial_avg:.3f})"
+          f" ≥ random({random_avg:.3f}) ≥ floor(0.0)")
 
     return {
         "task_id": "task2_property_discovery",
-        "oracle":  {"avg_score": oracle_avg,  "avg_reward": oracle_avg_r, "episodes": oracle_eps},
-        "partial": {"avg_score": partial_avg, "avg_reward": partial_avg_r, "episodes": partial_eps},
-        "empty":   {"avg_score": empty_avg,   "episodes": empty_eps},
+        "oracle":  {"avg_score": oracle_avg,  "episodes": oracle_eps},
+        "partial": {"avg_score": partial_avg, "episodes": partial_eps},
+        "random":  {"avg_score": random_avg,  "episodes": random_eps},
+        "floor":   {"avg_score": floor_avg,   "episodes": floor_eps},
     }
 
 
@@ -299,40 +169,50 @@ def run_task3_eval(n: int, seed_offset: int, verbose: bool) -> Dict[str, Any]:
     print(f"  Dataset: {len(get_all_task3_entries(contracts))} rule-check episodes\n")
     env = Task3Environment()
 
-    print("▶ Oracle (submits exact target function → 1.0):")
+    # Oracle
+    print("▶ Oracle      (exact target function → 1.0):")
     oracle_eps = []
     for i in range(n):
         ep = oracle_t3(env, seed_offset + i, verbose)
         oracle_eps.append(ep)
         print(f"  seed={ep['seed']:3d}  {ep['contract']:12s}.{ep['target_function']:18s}"
               f"  score={ep['grader_score']:.1f}  reward={ep['cumulative_reward']:+.2f}")
-    oracle_avg   = sum(e["grader_score"] for e in oracle_eps) / n
-    oracle_avg_r = sum(e["cumulative_reward"] for e in oracle_eps) / n
-    print(f"\n  Oracle avg: {oracle_avg:.3f}  reward: {oracle_avg_r:+.2f}")
-
-    print("\n▶ Subfunction (partial-credit callee or fallback to wrong):")
-    sub_eps   = [subfunction_t3(env, seed_offset + i) for i in range(n)]
-    sub_avg   = sum(e["grader_score"] for e in sub_eps) / n
-    sub_avg_r = sum(e["cumulative_reward"] for e in sub_eps) / n
-    submitted = list({e.get("submitted", "?") for e in sub_eps})
-    print(f"  Subfunction avg: {sub_avg:.3f}  reward: {sub_avg_r:+.2f}  "
-          f"submitted fns: {submitted}")
-
-    print("\n▶ Wrong (always 'constructor' → 0.0):")
-    wrong_eps = [wrong_t3(env, seed_offset + i) for i in range(n)]
-    wrong_avg = sum(e["grader_score"] for e in wrong_eps) / n
-    print(f"  Wrong avg: {wrong_avg:.3f}")
-
-    assert oracle_avg == 1.0
-    assert 0.0 <= sub_avg <= oracle_avg
-    assert wrong_avg == 0.0
-    print(f"\n  ✅ Task 3: oracle(1.0) ≥ subfunction({sub_avg:.3f}) > wrong(0.0)")
+    oracle_avg = _avg(oracle_eps)
+    print(f"\n  Oracle       avg: {oracle_avg:.3f}")
+
+    # Subfunction (partial credit)
+    print("\n▶ Subfunction (partial-credit callee if exists, else constructor):")
+    sub_eps = [subfunction_t3(env, seed_offset + i) for i in range(n)]
+    sub_avg = _avg(sub_eps)
+    submitted_sub = list({e.get("submitted", "?") for e in sub_eps})
+    print(f"  Subfunction  avg: {sub_avg:.3f}  submitted: {submitted_sub}")
+
+    # Random
+    print("\n▶ Random      (lists functions, submits uniformly random one):")
+    random_eps = [random_t3(env, seed_offset + i) for i in range(n)]
+    random_avg = _avg(random_eps)
+    submitted_rand = [e.get("submitted", "?") for e in random_eps]
+    print(f"  Random       avg: {random_avg:.3f}  submitted: {submitted_rand}")
+
+    # Floor
+    print("\n▶ Floor       (always 'constructor' → 0.0):")
+    floor_eps = [floor_t3(env, seed_offset + i) for i in range(n)]
+    floor_avg = _avg(floor_eps)
+    print(f"  Floor        avg: {floor_avg:.3f}")
+
+    assert oracle_avg == 1.0, f"Oracle avg {oracle_avg:.3f} should be 1.0"
+    assert floor_avg == 0.0,  f"Floor avg {floor_avg:.3f} should be 0.0"
+    assert oracle_avg >= random_avg >= floor_avg, \
+        f"Score ordering violated: oracle={oracle_avg}, random={random_avg}, floor={floor_avg}"
+    print(f"\n  ✅ Task 3: oracle(1.0) ≥ subfunction({sub_avg:.3f})"
+          f" ≥ random({random_avg:.3f}) ≥ floor(0.0)")
 
     return {
         "task_id": "task3_rule_checker",
-        "oracle":      {"avg_score": oracle_avg,  "avg_reward": oracle_avg_r, "episodes": oracle_eps},
-        "subfunction": {"avg_score": sub_avg,     "avg_reward": sub_avg_r,   "episodes": sub_eps},
-        "wrong":       {"avg_score": wrong_avg,   "episodes": wrong_eps},
+        "oracle":      {"avg_score": oracle_avg, "episodes": oracle_eps},
+        "subfunction": {"avg_score": sub_avg,    "episodes": sub_eps},
+        "random":      {"avg_score": random_avg, "episodes": random_eps},
+        "floor":       {"avg_score": floor_avg,  "episodes": floor_eps},
     }
 
 
@@ -342,13 +222,18 @@ def run_task3_eval(n: int, seed_offset: int, verbose: bool) -> Dict[str, Any]:
 
 def main():
     parser = argparse.ArgumentParser(
-        description="Evaluate Task 1, 2, and/or 3 of the SC Audit RL Environment"
+        description="Evaluate Task 1, 2, and/or 3 — oracle / partial / random / floor"
     )
-    parser.add_argument("--episodes", type=int, default=8)
-    parser.add_argument("--seed",     type=int, default=42)
-    parser.add_argument("--task",     choices=["1", "2", "3", "all"], default="all")
-    parser.add_argument("--verbose",  action="store_true")
-    parser.add_argument("--out",      default="eval_results.json")
+    parser.add_argument("--episodes", type=int,   default=8,
+                        help="Episodes per agent tier (default: 8)")
+    parser.add_argument("--seed",     type=int,   default=42,
+                        help="Starting RNG seed (default: 42)")
+    parser.add_argument("--task",     choices=["1", "2", "3", "all"], default="all",
+                        help="Which task(s) from [1, 2, 3] to evaluate (default: all)")
+    parser.add_argument("--verbose",  action="store_true",
+                        help="Print per-episode target details for oracle agents")
+    parser.add_argument("--out",      default="eval_results.json",
+                        help="Output JSON file (default: eval_results.json)")
     args = parser.parse_args()
 
     report: Dict[str, Any] = {"num_episodes": args.episodes, "seed_offset": args.seed}
@@ -360,14 +245,16 @@ def main():
     if args.task in ("3", "all"):
         report["task3"] = run_task3_eval(args.episodes, args.seed, args.verbose)
 
+    # ── Summary ──────────────────────────────────────────────────────────────
     print("\n" + "=" * 64)
     print("EVALUATION COMPLETE")
     print("=" * 64)
-    for label, key, tiers in [
-        ("Task 1", "task1", ["oracle", "partial", "wrong"]),
-        ("Task 2", "task2", ["oracle", "partial", "empty"]),
-        ("Task 3", "task3", ["oracle", "subfunction", "wrong"]),
-    ]:
+    rows = [
+        ("Task 1", "task1", ["oracle", "partial", "random", "floor"]),
+        ("Task 2", "task2", ["oracle", "partial", "random", "floor"]),
+        ("Task 3", "task3", ["oracle", "subfunction", "random", "floor"]),
+    ]
+    for label, key, tiers in rows:
         if key in report:
             scores = "  ".join(
                 f"{t}={report[key][t]['avg_score']:.3f}" for t in tiers

inference.py

@@ -1,36 +1,40 @@
 """
 inference.py
 ------------
-Baseline inference script for the Smart Contract Audit RL Environment.
-Implements Task 1 (Vulnerability Detection), Task 2 (Property Discovery),
-and Task 3 (Rule Checker).
+Baseline inference script — Smart Contract Audit RL Environment.
 
-Environment variables:
-  API_BASE_URL   – LLM API endpoint   (e.g. https://api.openai.com/v1)
-  MODEL_NAME     – model identifier   (e.g. gpt-4o-mini)
-  HF_TOKEN       – API key
+Implements agents for all three tasks using the OpenAI-compatible client.
+Emits mandatory structured stdout in the OpenEnv format.
+
+MANDATORY ENV VARS:
+  API_BASE_URL   LLM API endpoint   (default: https://api.openai.com/v1)
+  MODEL_NAME     Model identifier   (default: gpt-4o-mini)
+  HF_TOKEN       API key / HF token
+
+MANDATORY STDOUT FORMAT (per episode):
+  [START] task=<id> env=smart-contract-audit model=<model>
+  [STEP]  step=<n> action=<str> reward=<0.00> done=<true|false> error=<str|null>
+  [END]   success=<true|false> steps=<n> score=<0.000> rewards=<r1,r2,...>
 
 Usage:
   python inference.py
 
 Output:
-  Per-task scores printed to stdout.
-  Final baseline scores written to baseline_scores.json.
-
-Runtime: < 5 minutes on 3 episodes per task with gpt-4o-mini.
+  Structured stdout per episode, plus baseline_scores.json summary.
 """
 
+import asyncio
 import json
 import os
 import sys
 import time
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from openai import OpenAI
 
-from tasks.task1.environment import Task1Environment
-from tasks.task2.environment import Task2Environment
-from tasks.task3.environment import Task3Environment
+from tasks.task1 import Task1Environment
+from tasks.task2 import Task2Environment
+from tasks.task3 import Task3Environment
 from env.schemas import Action, ActionType
 from utils import T1_SYSTEM, T2_SYSTEM, T3_SYSTEM
 
@@ -38,83 +42,152 @@ from utils import T1_SYSTEM, T2_SYSTEM, T3_SYSTEM
 # Configuration
 # ─────────────────────────────────────────────────────────────────────────────
 
-API_BASE_URL = os.environ.get("API_BASE_URL", "https://api.openai.com/v1")
-MODEL_NAME   = os.environ.get("MODEL_NAME",   "gpt-4o-mini")
-HF_TOKEN     = os.environ.get("HF_TOKEN",     "")
+API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
+MODEL_NAME   = os.getenv("MODEL_NAME",   "gpt-4o-mini")
+HF_TOKEN     = os.getenv("HF_TOKEN",     "")
 
 if not HF_TOKEN:
-    print("WARNING: HF_TOKEN not set. API calls may fail.", file=sys.stderr)
+    print("[WARN] HF_TOKEN not set — API calls may fail.", file=sys.stderr)
+
+# Benchmark / environment identifier (constant for this env)
+ENV_BENCHMARK = "smart-contract-audit"
+
+# Episodes per task
+NUM_EPISODES = 3
+SEED_BASE    = 42
+
+# Max steps per task
+MAX_STEPS_T1 = 15
+MAX_STEPS_T2 = 10
+MAX_STEPS_T3 = 12
 
-MAX_STEPS_T1  = 15
-MAX_STEPS_T2  = 10
-NUM_EPISODES  = 3
-SEED_BASE_T1  = 42
-SEED_BASE_T2  = 10
+# A grader_score >= this is considered a "success" for the [END] line
+SUCCESS_SCORE_THRESHOLD = 0.5
 
 client = OpenAI(api_key=HF_TOKEN, base_url=API_BASE_URL)
 
+
 # ─────────────────────────────────────────────────────────────────────────────
-# Task 1 agent
+# Mandatory stdout helpers
 # ─────────────────────────────────────────────────────────────────────────────
 
+def log_start(task: str, env: str, model: str) -> None:
+    """Emit the [START] line — one per episode."""
+    print(f"[START] task={task} env={env} model={model}", flush=True)
+
+
+def log_step(
+    step: int,
+    action: str,
+    reward: float,
+    done: bool,
+    error: Optional[str] = None,
+) -> None:
+    """Emit a [STEP] line — one per env.step() call."""
+    error_val = error if error else "null"
+    print(
+        f"[STEP] step={step} action={action} "
+        f"reward={reward:.2f} done={str(done).lower()} error={error_val}",
+        flush=True,
+    )
+
+
+def log_end(
+    success: bool,
+    steps: int,
+    score: float,
+    rewards: List[float],
+) -> None:
+    """Emit the [END] line — one per episode, always emitted."""
+    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
+    print(
+        f"[END] success={str(success).lower()} steps={steps} "
+        f"score={score:.3f} rewards={rewards_str}",
+        flush=True,
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Task 1 — Targeted Vulnerability Detection
+# ──────────────���──────────────────────────────────────────────────────────────
 
 def _t1_user_msg(obs: Dict[str, Any]) -> str:
     return (
         f"Contract: {obs['contract_name']}\n"
         f"Description: {obs['contract_description']}\n"
-        f"Step: {obs['step_count']} | Reward: {obs['cumulative_reward']:.2f}\n\n"
-        f"Last action: {obs['last_action'] or 'None'}\n"
-        f"Result: {obs['last_action_result'] or 'Episode started.'}"
+        f"Step: {obs['step_count']} | Reward so far: {obs['cumulative_reward']:.2f}\n\n"
+        f"Last action : {obs['last_action'] or 'None'}\n"
+        f"Last result : {obs['last_action_result'] or 'Episode just started.'}"
     )
 
 
-def run_t1_episode(env: Task1Environment, seed: int, ep: int) -> Dict[str, Any]:
-    r = env.reset(seed=seed)
+def _run_t1_episode(env: Task1Environment, seed: int, ep_num: int) -> Dict[str, Any]:
+    """Run one Task 1 episode; emit [START]/[STEP]/[END]."""
+    r   = env.reset(seed=seed)
     obs = r.observation.model_dump()
-    print(f"    ep={ep} seed={seed} contract={obs['contract_name']}")
-
-    messages = [{"role": "system", "content": T1_SYSTEM}]
-    grader_score = 0.0
-    cum_reward   = 0.0
-
-    for step in range(MAX_STEPS_T1):
-        messages.append({"role": "user", "content": _t1_user_msg(obs)})
-        try:
-            resp = client.chat.completions.create(
-                model=MODEL_NAME, messages=messages,
-                max_tokens=200, temperature=0.0,
-            )
-            raw = resp.choices[0].message.content.strip()
-        except Exception as e:
-            print(f"      LLM error: {e}", file=sys.stderr)
-            break
-
-        try:
-            parsed = json.loads(raw)
-            at     = ActionType(parsed["action"])
-            params = parsed.get("params", {})
-        except Exception:
-            at, params = ActionType.LIST_FUNCTIONS, {}
-
-        messages.append({"role": "assistant", "content": raw})
-        result = env.step(Action(action_type=at, params=params))
-        obs    = result.observation.model_dump()
-        print(f"      step {step+1:2d}: {at.value:25s} r={result.reward.value:+.2f}")
-
-        if result.done:
-            v = result.reward.value
-            grader_score = 1.0 if v >= 4.9 else (0.5 if v >= 0.9 else 0.0)
-            cum_reward   = obs["cumulative_reward"]
-            break
-        time.sleep(0.3)
-
-    print(f"      → grader_score={grader_score:.1f}  cum_reward={cum_reward:.2f}")
-    return {"episode": ep, "seed": seed, "contract": obs["contract_name"],
-            "grader_score": grader_score, "cumulative_reward": cum_reward}
+
+    log_start(task="task1_vuln_detection", env=ENV_BENCHMARK, model=MODEL_NAME)
+
+    messages      = [{"role": "system", "content": T1_SYSTEM}]
+    step_rewards: List[float] = []
+    grader_score  = 0.0
+    steps_taken   = 0
+    error_msg: Optional[str] = None
+
+    try:
+        for step in range(1, MAX_STEPS_T1 + 1):
+            messages.append({"role": "user", "content": _t1_user_msg(obs)})
+            try:
+                resp = client.chat.completions.create(
+                    model=MODEL_NAME, messages=messages,
+                    max_tokens=200, temperature=0.0,
+                )
+                raw = resp.choices[0].message.content.strip() # type: ignore
+                error_msg = None
+            except Exception as e:
+                raw = ""
+                error_msg = str(e)[:80]
+                print(f"[DEBUG] T1 LLM error ep={ep_num} step={step}: {e}", file=sys.stderr)
+
+            try:
+                parsed = json.loads(raw)
+                at     = ActionType(parsed["action"])
+                params = parsed.get("params", {})
+            except Exception:
+                at, params = ActionType.LIST_FUNCTIONS, {}
+
+            messages.append({"role": "assistant", "content": raw})
+            result = env.step(Action(action_type=at, params=params))
+            obs    = result.observation.model_dump()
+            r_val  = result.reward.value
+            done   = result.done
+
+            step_rewards.append(r_val)
+            steps_taken = step
+            log_step(step=step, action=at.value, reward=r_val, done=done, error=error_msg)
+
+            if done:
+                v = r_val
+                grader_score = 1.0 if v >= 4.9 else (0.5 if v >= 0.9 else 0.0)
+                break
+
+            time.sleep(0.3)
+
+    finally:
+        success = grader_score >= SUCCESS_SCORE_THRESHOLD
+        log_end(success=success, steps=steps_taken, score=grader_score, rewards=step_rewards)
+
+    return {
+        "episode":          ep_num,
+        "seed":             seed,
+        "contract":         obs["contract_name"],
+        "grader_score":     grader_score,
+        "cumulative_reward": obs["cumulative_reward"],
+    }
 
 
 # ──────────────────────────────────────────���──────────────────────────────────
-# Task 2 agent
+# Task 2 — Property Discovery
 # ─────────────────────────────────────────────────────────────────────────────
 
 
@@ -124,198 +197,246 @@ def _t2_user_msg(obs: Dict[str, Any]) -> str:
         f"Contract : {obs['contract_name']}\n"
         f"Function : {extra.get('target_function', '?')}  "
         f"({extra.get('target_signature', '')})\n"
-        f"Step: {obs['step_count']} | Reward: {obs['cumulative_reward']:.2f}\n\n"
-        f"Last action: {obs['last_action'] or 'None'}\n"
-        f"Result:\n{obs['last_action_result'] or 'Episode started — begin exploring.'}"
+        f"Step: {obs['step_count']} | Reward so far: {obs['cumulative_reward']:.2f}\n\n"
+        f"Last action : {obs['last_action'] or 'None'}\n"
+        f"Last result :\n{obs['last_action_result'] or 'Episode just started.'}"
     )
 
 
-def run_t2_episode(env: Task2Environment, seed: int, ep: int) -> Dict[str, Any]:
-    r = env.reset(seed=seed)
+def _run_t2_episode(env: Task2Environment, seed: int, ep_num: int) -> Dict[str, Any]:
+    """Run one Task 2 episode; emit [START]/[STEP]/[END]."""
+    r   = env.reset(seed=seed)
     obs = r.observation.model_dump()
     fn  = obs["extra"].get("target_function", "?")
-    print(f"    ep={ep} seed={seed}  {obs['contract_name']}.{fn}()")
-
-    messages = [{"role": "system", "content": T2_SYSTEM}]
-    grader_score = 0.0
-    cum_reward   = 0.0
-
-    for step in range(MAX_STEPS_T2):
-        messages.append({"role": "user", "content": _t2_user_msg(obs)})
-        try:
-            resp = client.chat.completions.create(
-                model=MODEL_NAME, messages=messages,
-                max_tokens=400, temperature=0.0,
-            )
-            raw = resp.choices[0].message.content.strip()
-        except Exception as e:
-            print(f"      LLM error: {e}", file=sys.stderr)
-            break
-
-        try:
-            parsed = json.loads(raw)
-            at     = ActionType(parsed["action"])
-            params = parsed.get("params", {})
-        except Exception:
-            at, params = ActionType.GET_FUNCTION_CODE, {}
-
-        messages.append({"role": "assistant", "content": raw})
-        result = env.step(Action(action_type=at, params=params))
-        obs    = result.observation.model_dump()
-        r_val  = result.reward.value
-        print(f"      step {step+1:2d}: {at.value:25s} r={r_val:+.2f}")
-
-        if result.done:
-            grader_score = round(r_val / 5.0, 3) if r_val > 0 else 0.0
-            cum_reward   = obs["cumulative_reward"]
-            break
-        time.sleep(0.3)
-
-    print(f"      → grader_score={grader_score:.3f}  cum_reward={cum_reward:.2f}")
-    return {"episode": ep, "seed": seed,
-            "contract": obs["contract_name"], "function": fn,
-            "grader_score": grader_score, "cumulative_reward": cum_reward}
+
+    log_start(task="task2_property_discovery", env=ENV_BENCHMARK, model=MODEL_NAME)
+
+    messages      = [{"role": "system", "content": T2_SYSTEM}]
+    step_rewards: List[float] = []
+    grader_score  = 0.0
+    steps_taken   = 0
+    error_msg: Optional[str] = None
+
+    try:
+        for step in range(1, MAX_STEPS_T2 + 1):
+            messages.append({"role": "user", "content": _t2_user_msg(obs)})
+            try:
+                resp = client.chat.completions.create(
+                    model=MODEL_NAME, messages=messages,
+                    max_tokens=400, temperature=0.0,
+                )
+                raw = resp.choices[0].message.content.strip() # type: ignore
+                error_msg = None
+            except Exception as e:
+                raw = ""
+                error_msg = str(e)[:80]
+                print(f"[DEBUG] T2 LLM error ep={ep_num} step={step}: {e}", file=sys.stderr)
+
+            try:
+                parsed = json.loads(raw)
+                at     = ActionType(parsed["action"])
+                params = parsed.get("params", {})
+            except Exception:
+                at, params = ActionType.GET_FUNCTION_CODE, {}
+
+            messages.append({"role": "assistant", "content": raw})
+            result = env.step(Action(action_type=at, params=params))
+            obs    = result.observation.model_dump()
+            r_val  = result.reward.value
+            done   = result.done
+
+            step_rewards.append(r_val)
+            steps_taken = step
+            log_step(step=step, action=at.value, reward=r_val, done=done, error=error_msg)
+
+            if done:
+                grader_score = round(r_val / 5.0, 3) if r_val > 0 else 0.0
+                break
+
+            time.sleep(0.3)
+
+    finally:
+        success = grader_score >= SUCCESS_SCORE_THRESHOLD
+        log_end(success=success, steps=steps_taken, score=grader_score, rewards=step_rewards)
+
+    return {
+        "episode":          ep_num,
+        "seed":             seed,
+        "contract":         obs["contract_name"],
+        "function":         fn,
+        "grader_score":     grader_score,
+        "cumulative_reward": obs["cumulative_reward"],
+    }
 
 
 # ─────────────────────────────────────────────────────────────────────────────
-# Task runners
+# Task 3 — Rule Checker
 # ─────────────────────────────────────────────────────────────────────────────
 
-def run_task1(n: int = NUM_EPISODES) -> Dict[str, Any]:
-    print("\n" + "="*60)
-    print("TASK 1: Targeted Vulnerability Detection")
-    print("="*60)
-    env = Task1Environment()
-    episodes = [run_t1_episode(env, SEED_BASE_T1 + i, i+1) for i in range(n)]
-    avg_s  = sum(e["grader_score"] for e in episodes) / n
-    avg_r  = sum(e["cumulative_reward"] for e in episodes) / n
-    print(f"\n  Avg grader score  : {avg_s:.3f}")
-    print(f"  Avg cum reward    : {avg_r:.2f}")
-    return {"task_id": "task1_vuln_detection", "name": "Targeted Vulnerability Detection",
-            "status": "active", "num_episodes": n, "episodes": episodes,
-            "avg_grader_score": avg_s, "avg_cumulative_reward": avg_r}
-
-
-def run_task2(n: int = NUM_EPISODES) -> Dict[str, Any]:
-    print("\n" + "="*60)
-    print("TASK 2: Property Discovery")
-    print("="*60)
-    env = Task2Environment()
-    episodes = [run_t2_episode(env, SEED_BASE_T2 + i, i+1) for i in range(n)]
-    avg_s  = sum(e["grader_score"] for e in episodes) / n
-    avg_r  = sum(e["cumulative_reward"] for e in episodes) / n
-    print(f"\n  Avg grader score  : {avg_s:.3f}")
-    print(f"  Avg cum reward    : {avg_r:.2f}")
-    return {"task_id": "task2_property_discovery", "name": "Property Discovery",
-            "status": "active", "num_episodes": n, "episodes": episodes,
-            "avg_grader_score": avg_s, "avg_cumulative_reward": avg_r}
-
 
 def _t3_user_msg(obs: Dict[str, Any]) -> str:
     extra = obs.get("extra", {})
     return (
         f"Contract  : {obs['contract_name']}\n"
-        f"Property  : {extra.get('property_english', '(no property)')}\n"
-        f"Step: {obs['step_count']} | Reward: {obs['cumulative_reward']:.2f}\n\n"
-        f"Last action: {obs['last_action'] or 'None'}\n"
-        f"Result:\n{obs['last_action_result'] or 'Episode started.'}"
+        f"Property  : {extra.get('property_english', '(none)')}\n"
+        f"Step: {obs['step_count']} | Reward so far: {obs['cumulative_reward']:.2f}\n\n"
+        f"Last action : {obs['last_action'] or 'None'}\n"
+        f"Last result :\n{obs['last_action_result'] or 'Episode just started.'}"
     )
 
 
-def run_t3_episode(env: Task3Environment, seed: int, ep: int) -> Dict[str, Any]:
+def _run_t3_episode(env: Task3Environment, seed: int, ep_num: int) -> Dict[str, Any]:
+    """Run one Task 3 episode; emit [START]/[STEP]/[END]."""
     r   = env.reset(seed=seed)
     obs = r.observation.model_dump()
-    prop_preview = obs['extra'].get('property_english', '')[:55]
-    print(f"    ep={ep} seed={seed}  {obs['contract_name']}  \"{prop_preview}...\"")
-
-    messages = [{"role": "system", "content": T3_SYSTEM}]
-    grader_score = 0.0
-    cum_reward   = 0.0
-
-    for step in range(15):
-        messages.append({"role": "user", "content": _t3_user_msg(obs)})
-        try:
-            resp = client.chat.completions.create(
-                model=MODEL_NAME, messages=messages,
-                max_tokens=200, temperature=0.0,
-            )
-            raw = resp.choices[0].message.content.strip()
-        except Exception as e:
-            print(f"      LLM error: {e}", file=sys.stderr)
-            break
-
-        try:
-            parsed = json.loads(raw)
-            at     = ActionType(parsed["action"])
-            params = parsed.get("params", {})
-        except Exception:
-            at, params = ActionType.LIST_FUNCTIONS, {}
-
-        messages.append({"role": "assistant", "content": raw})
-        result = env.step(Action(action_type=at, params=params))
-        obs    = result.observation.model_dump()
-        print(f"      step {step+1:2d}: {at.value:28s} r={result.reward.value:+.2f}")
-
-        if result.done:
-            v = result.reward.value
-            grader_score = 1.0 if v >= 4.9 else (0.3 if v >= 1.0 else 0.0)
-            cum_reward   = obs["cumulative_reward"]
-            break
-        time.sleep(0.3)
-
-    print(f"      → grader_score={grader_score:.1f}  cum_reward={cum_reward:.2f}")
-    return {"episode": ep, "seed": seed, "contract": obs["contract_name"],
-            "grader_score": grader_score, "cumulative_reward": cum_reward}
+
+    log_start(task="task3_rule_checker", env=ENV_BENCHMARK, model=MODEL_NAME)
+
+    messages      = [{"role": "system", "content": T3_SYSTEM}]
+    step_rewards: List[float] = []
+    grader_score  = 0.0
+    steps_taken   = 0
+    error_msg: Optional[str] = None
+
+    try:
+        for step in range(1, MAX_STEPS_T3 + 1):
+            messages.append({"role": "user", "content": _t3_user_msg(obs)})
+            try:
+                resp = client.chat.completions.create(
+                    model=MODEL_NAME, messages=messages,
+                    max_tokens=200, temperature=0.0,
+                )
+                raw = resp.choices[0].message.content.strip()   # type: ignore
+                error_msg = None
+            except Exception as e:
+                raw = ""
+                error_msg = str(e)[:80]
+                print(f"[DEBUG] T3 LLM error ep={ep_num} step={step}: {e}", file=sys.stderr)
+
+            try:
+                parsed = json.loads(raw)
+                at     = ActionType(parsed["action"])
+                params = parsed.get("params", {})
+            except Exception:
+                at, params = ActionType.LIST_FUNCTIONS, {}
+
+            messages.append({"role": "assistant", "content": raw})
+            result = env.step(Action(action_type=at, params=params))
+            obs    = result.observation.model_dump()
+            r_val  = result.reward.value
+            done   = result.done
+
+            step_rewards.append(r_val)
+            steps_taken = step
+            log_step(step=step, action=at.value, reward=r_val, done=done, error=error_msg)
+
+            if done:
+                v = r_val
+                grader_score = 1.0 if v >= 4.9 else (0.3 if v >= 1.0 else 0.0)
+                break
+
+            time.sleep(0.3)
+
+    finally:
+        success = grader_score >= SUCCESS_SCORE_THRESHOLD
+        log_end(success=success, steps=steps_taken, score=grader_score, rewards=step_rewards)
+
+    return {
+        "episode":          ep_num,
+        "seed":             seed,
+        "contract":         obs["contract_name"],
+        "grader_score":     grader_score,
+        "cumulative_reward": obs["cumulative_reward"],
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Task runners
+# ─────────────────────────────────────────────────────────────────────────────
+
+def run_task1(n: int = NUM_EPISODES) -> Dict[str, Any]:
+    print("\n" + "="*60, flush=True)
+    print("TASK 1: Targeted Vulnerability Detection", flush=True)
+    print("="*60, flush=True)
+    env      = Task1Environment()
+    episodes = [_run_t1_episode(env, SEED_BASE + i, i + 1) for i in range(n)]
+    avg_s    = sum(e["grader_score"] for e in episodes) / n
+    avg_r    = sum(e["cumulative_reward"] for e in episodes) / n
+    print(f"\n  Avg grader score : {avg_s:.3f}", flush=True)
+    print(f"  Avg cum reward   : {avg_r:.2f}", flush=True)
+    return {
+        "task_id": "task1_vuln_detection", "name": "Targeted Vulnerability Detection",
+        "status": "active", "num_episodes": n, "episodes": episodes,
+        "avg_grader_score": avg_s, "avg_cumulative_reward": avg_r,
+    }
+
+
+def run_task2(n: int = NUM_EPISODES) -> Dict[str, Any]:
+    print("\n" + "="*60, flush=True)
+    print("TASK 2: Property Discovery", flush=True)
+    print("="*60, flush=True)
+    env      = Task2Environment()
+    episodes = [_run_t2_episode(env, SEED_BASE + i, i + 1) for i in range(n)]
+    avg_s    = sum(e["grader_score"] for e in episodes) / n
+    avg_r    = sum(e["cumulative_reward"] for e in episodes) / n
+    print(f"\n  Avg grader score : {avg_s:.3f}", flush=True)
+    print(f"  Avg cum reward   : {avg_r:.2f}", flush=True)
+    return {
+        "task_id": "task2_property_discovery", "name": "Property Discovery",
+        "status": "active", "num_episodes": n, "episodes": episodes,
+        "avg_grader_score": avg_s, "avg_cumulative_reward": avg_r,
+    }
 
 
 def run_task3(n: int = NUM_EPISODES) -> Dict[str, Any]:
-    print("\n" + "="*60)
-    print("TASK 3: Rule Checker")
-    print("="*60)
-    env = Task3Environment()
-    episodes = [run_t3_episode(env, 42 + i, i + 1) for i in range(n)]
-    avg_s = sum(e["grader_score"] for e in episodes) / n
-    avg_r = sum(e["cumulative_reward"] for e in episodes) / n
-    print(f"\n  Avg grader score  : {avg_s:.3f}")
-    print(f"  Avg cum reward    : {avg_r:.2f}")
-    return {"task_id": "task3_rule_checker", "name": "Rule Checker",
-            "status": "active", "num_episodes": n, "episodes": episodes,
-            "avg_grader_score": avg_s, "avg_cumulative_reward": avg_r}
+    print("\n" + "="*60, flush=True)
+    print("TASK 3: Rule Checker", flush=True)
+    print("="*60, flush=True)
+    env      = Task3Environment()
+    episodes = [_run_t3_episode(env, SEED_BASE + i, i + 1) for i in range(n)]
+    avg_s    = sum(e["grader_score"] for e in episodes) / n
+    avg_r    = sum(e["cumulative_reward"] for e in episodes) / n
+    print(f"\n  Avg grader score : {avg_s:.3f}", flush=True)
+    print(f"  Avg cum reward   : {avg_r:.2f}", flush=True)
+    return {
+        "task_id": "task3_rule_checker", "name": "Rule Checker",
+        "status": "active", "num_episodes": n, "episodes": episodes,
+        "avg_grader_score": avg_s, "avg_cumulative_reward": avg_r,
+    }
 
 
 # ─────────────────────────────────────────────────────────────────────────────
 # Main
 # ─────────────────────────────────────────────────────────────────────────────
 
-def main():
-    print("Smart Contract Audit RL Environment — Baseline Inference")
-    print(f"Model: {MODEL_NAME} | Base URL: {API_BASE_URL}")
+async def main() -> None:
+    """Async entry point (wraps sync env calls; asyncio.run() expected by caller)."""
+    print("Smart Contract Audit RL Environment — Baseline Inference", flush=True)
+    print(f"Model: {MODEL_NAME} | Base URL: {API_BASE_URL}", flush=True)
 
     t1 = run_task1(NUM_EPISODES)
     t2 = run_task2(NUM_EPISODES)
     t3 = run_task3(NUM_EPISODES)
 
     results = {
-        "model": MODEL_NAME, "base_url": API_BASE_URL,
-        "tasks": [t1, t2, t3],
+        "model":    MODEL_NAME,
+        "base_url": API_BASE_URL,
+        "tasks":    [t1, t2, t3],
     }
-
-    active  = results["tasks"]
-    overall = sum(t["avg_grader_score"] for t in active) / len(active)
+    overall = sum(t["avg_grader_score"] for t in results["tasks"]) / 3
     results["overall_avg_score"] = overall
 
-    print("\n" + "="*60)
-    print("BASELINE SUMMARY")
-    print("="*60)
+    print("\n" + "="*60, flush=True)
+    print("BASELINE SUMMARY", flush=True)
+    print("="*60, flush=True)
     for t in results["tasks"]:
-        icon = "✅" if t["status"] == "active" else "⏳"
-        print(f"  {icon} {t['name']:40s}: {t['avg_grader_score']:.3f}")
-    print(f"\n  Overall (active tasks): {overall:.3f}")
+        print(f"  ✅ {t['name']:40s}: {t['avg_grader_score']:.3f}", flush=True)
+    print(f"\n  Overall avg grader score: {overall:.3f}", flush=True)
 
     with open("baseline_scores.json", "w") as f:
         json.dump(results, f, indent=2)
-    print("\n  Scores written to baseline_scores.json")
+    print("\n  Scores written to baseline_scores.json", flush=True)
 
 
 if __name__ == "__main__":
-    main()
+    asyncio.run(main())

openenv.yaml

@@ -5,7 +5,7 @@ description: >
   Agents interact with real-world Solidity contract data from Certora-audited
   projects, practising three real audit tasks: vulnerability detection,
   property discovery, and rule checking.
-author: "SmartAudit Team"
+author: "Codex47"
 license: MIT
 
 tasks:
@@ -73,7 +73,7 @@ action_space:
     get_function_natspec:  {params: {}, reward: -0.08}
     get_file_natspec:      {params: {}, reward: -0.03}
     get_related_functions: {params: {}, reward: -0.06}
-    get_signature:         {params: {}, reward: -0.04}
+    get_io:                {params: {}, reward: -0.04}
     get_similar_rule:      {params: {}, reward: -0.20}
     submit_property:       {params: {property: string}, reward: "0.0-5.0 keyword-weighted, one attempt"}
   task3:
@@ -82,7 +82,7 @@ action_space:
     get_function_code:       {params: {function_name: string},       reward: -0.10}
     get_state_variable:      {params: {variable_name: "string opt"}, reward: -0.05}
     get_call_graph:          {params: {},                            reward: -0.08}
-    get_property_specification: {params: {},                            reward: -0.03}
+    get_formalized_property: {params: {},                            reward: -0.03}
     submit_function:         {params: {function_name: string},       reward: "+5.0 / +1.5 / -1.5, one attempt"}
 
 reward:
@@ -135,4 +135,4 @@ interface:
   python:
     reset: "env.reset(seed=None) -> ResetResult"
     step:  "env.step(action)     -> StepResult"
-    state: "env.state()          -> StateResult"
+    state: "env.state()          -> StateResult"

tasks/task1/environment.py

@@ -27,14 +27,7 @@ from __future__ import annotations
 import random
 from typing import Any, Dict, List, Optional, Set
 
-from data.data_loader import (
-    load_contracts,
-    sample_episode,
-    get_function_by_name,
-    get_state_variable_by_name,
-    list_function_names,
-    list_state_variable_names,
-)
+from data.data_loader import load_contracts, sample_episode
 from env.base_env import BaseEnv
 from env.schemas import (
     Action,
@@ -138,7 +131,7 @@ class Task1Environment(BaseEnv):
         return StateResult(
             task_id=TASK_ID,
             contract_name=self._contract.get("contract_name", ""),
-            target_function=self._target_fn.get("name"),
+            target_function=self._target_fn.get("name", ""),
             step_count=self._step_count,
             cumulative_reward=self._cumulative_reward,
             done=self._done,

tasks/task1/grader.py

@@ -10,14 +10,6 @@ Deterministic grader. Score range: 0.0 – 1.0
 from __future__ import annotations
 from typing import Dict
 from utils import SemanticMatcher
-from data.data_loader import load_vulnerabilities
-
-def match_vuln_keywords(submitted: str, expected: str) -> bool:
-    """Checks if the submitted vulnerability type matches the expected one using keyword matching."""
-    for types in load_vulnerabilities():
-        if types["vulnerability"] == expected:
-            return SemanticMatcher().match(types["terms"], submitted)
-    return False
 
 class Task1Grader:
     def __init__(self, target_function: str, vulnerability_issue: str) -> None:
@@ -27,7 +19,7 @@ class Task1Grader:
     def grade_submission(self, submitted_function: str, submitted_vuln_type: str) -> float:
         if submitted_function.strip().lower() != self.target_function:
             return 0.0
-        return 1.0 if match_vuln_keywords(submitted_vuln_type, self.vulnerability_issue) else 0.5
+        return 1.0 if SemanticMatcher().match(self.vulnerability_issue, submitted_vuln_type) else 0.5
 
     def reward_for_score(self, score: float) -> float:
         if score == 1.0: return 5.0

tasks/task2/actions.py

@@ -124,7 +124,7 @@ def submit_property(ctx: Any, qkey: str, params: Dict) -> Tuple[str, Reward]:
     score, confidence = ctx._grader.grade(submitted_text)
     reward = round(score * 5.0, 4)
 
-    msg = f'Score: {score:.2f}/1.00 → Confidence: {confidence:.2f}\n'    
+    msg = f'Score: {score:.2f}/1.00 → Confidence: {confidence}\n'    
     return msg, Reward(
         value=reward,
         reason=f"Property submission score={score:.3f}",

tasks/task2/environment.py

@@ -134,7 +134,7 @@ class Task2Environment(BaseEnv):
         return StateResult(
             task_id=TASK_ID,
             contract_name=self._contract.get("contract_name", ""),
-            target_function=self._target_fn.get("name"),
+            target_function=self._target_fn.get("name", ""),
             step_count=self._step_count,
             cumulative_reward=self._cum_reward,
             done=self._done,

tasks/task3/environment.py

@@ -35,7 +35,7 @@ from __future__ import annotations
 
 import random
 from typing import Any, Dict, List, Optional, Set
-import actions
+from tasks.task3 import actions
 
 from data.data_loader import load_contracts, sample_task3_episode
 from env.base_env import BaseEnv
@@ -83,6 +83,8 @@ class Task3Environment(BaseEnv):
         self._seen:       Set[str] = set()
 
     # ── OpenEnv interface ─────────────────────────────────────────────────────
+    
+    # ! Need to change alot here
 
     def reset(self, seed: Optional[int] = None) -> ResetResult:
         if seed is not None:
@@ -94,8 +96,8 @@ class Task3Environment(BaseEnv):
         t3 = self._target_fn["task3"]
         self._grader = Task3Grader(
             target_function=self._target_fn["name"],
-            partial_credit_functions=t3.get("partial_credit_functions", []),
-            property_english=t3.get("property_english", ""),
+            partial_credit_functions=t3.get("partial_credit_functions", []),        # ! doesn't exists
+            property_english=t3.get("property_english", ""),        # ! doesn't exist
         )
         self._step_count = 0
         self._cum_reward = 0.0
@@ -142,7 +144,7 @@ class Task3Environment(BaseEnv):
         return StateResult(
             task_id=TASK_ID,
             contract_name=self._contract.get("contract_name", ""),
-            target_function=self._target_fn.get("name"),
+            target_function=self._target_fn.get("name", ""),
             step_count=self._step_count,
             cumulative_reward=self._cum_reward,
             done=self._done,

utils/prompts.py

@@ -18,7 +18,16 @@ Given a contract, identify the ONE vulnerable function and its vulnerability typ
    timestamp dependence, denial of service, unchecked return value
 4. Submit when confident
 
-Respond ONLY with valid JSON. No explanation, no markdown."""
+Respond ONLY with valid JSON. No explanation, no markdown.
+
+## Evaluation Strategy:
+Your output vulnerability_type will be compared to ground truth using a deterministic semantic matcher with 
+three weighted components:
+- Lexical Jaccard (20%) - overlap of lemmatized, stopword‑removed tokens.
+- Synonym Jaccard (25%) - overlap after expanding each word with WordNet synonyms.
+- Semantic cosine (55%) - sentence‑embedding similarity (all‑MiniLM‑L6‑v2).
+Match Threshold: score ≥ 0.72 → "match" (partial credit); score ≥ 0.88 → "strong match" (full credit).
+"""
 
 T2_SYSTEM = """You are a formal methods engineer specialising in Solidity smart contracts.
 
@@ -36,31 +45,6 @@ A good property covers:
 {"action": "get_function_code",     "params": {}}
 {"action": "get_function_natspec",  "params": {}}
 {"action": "get_file_natspec",      "params": {}}
-
-def _t3_user_msg(obs: Dict[str, Any]) -> str:
-    extra = obs.get("extra", {})
-    return (
-        f"Contract  : {obs['contract_name']}\n"
-        f"Property  : {extra.get('property_english', '(no property)')}\n"
-        f"Step: {obs['step_count']} | Reward: {obs['cumulative_reward']:.2f}\n\n"
-        f"Last action: {obs['last_action'] or 'None'}\n"
-        f"Result:\n{obs['last_action_result'] or 'Episode started.'}"
-    )
-
-
-def run_t3_episode(env: Task3Environment, seed: int, ep: int) -> Dict[str, Any]:
-    r   = env.reset(seed=seed)
-    obs = r.observation.model_dump()
-    prop_preview = obs['extra'].get('property_english', '')[:55]
-    print(f"    ep={ep} seed={seed}  {obs['contract_name']}  \"{prop_preview}...\"")
-
-    messages = [{"role": "system", "content": T3_SYSTEM}]
-    grader_score = 0.0
-    cum_reward   = 0.0
-
-    for step in range(15):
-        messages.append({"role": "user", "content": _t3_user_msg(obs)})
-
 {"action": "get_related_functions", "params": {}}
 {"action": "get_io",                "params": {}}
 {"action": "get_similar_rule",      "params": {}}
@@ -73,7 +57,14 @@ def run_t3_episode(env: Task3Environment, seed: int, ep: int) -> Dict[str, Any]:
 - Write 2–4 sentences. Be specific about variable names and amounts.
 - Do NOT guess — read the code first.
 
-Respond ONLY with valid JSON. No markdown, no explanation."""
+Respond ONLY with valid JSON. No markdown, no explanation.
+
+## Evaluation Strategy:
+Your output vulnerability_type will be compared to ground truth using a deterministic semantic matcher with three weighted components:
+- Lexical Jaccard (20%) - overlap of lemmatized, stopword‑removed tokens.
+- Synonym Jaccard (25%) - overlap after expanding each word with WordNet synonyms.
+- Semantic cosine (55%) - sentence‑embedding similarity (all‑MiniLM‑L6‑v2).
+Match Threshold: score ≥ 0.72 → "match" (partial credit); score ≥ 0.88 → "strong match" (full credit)."""
 
 
 T3_SYSTEM = """You are a smart contract security auditor checking rule compliance.

utils/propertyretriever.py

@@ -10,10 +10,8 @@ import pandas as pd
 import numpy as np
 from sentence_transformers import SentenceTransformer
 from sklearn.preprocessing import normalize
-import os
+from data.data_loader import DEFAULT_CSV_PATH
 
-DATA_DIR = os.path.join(os.path.dirname(__file__))
-DEFAULT_CSV_PATH = os.path.join(DATA_DIR, "properties.csv")
 SIMILARITY_THRESHOLD = 0.8  # Adjust as needed based on validation
 
 # -------------------------------------------------------------------

utils/semanticmatcher.py

@@ -200,6 +200,7 @@ class SemanticMatcher:
         """
         # Fast-path: normalized exact match
         if normalize(text_a) == normalize(text_b):
+            self.confidence_level = "strong"
             return True
 
         tokens_a = tokenize_and_lemmatize(text_a)

validate.py

@@ -77,7 +77,7 @@ def check_t2_env():
     assert r.observation.task_id == "task2_property_discovery"
     assert "target_function" in r.observation.extra
     for at in [ActionType.GET_FUNCTION_CODE, ActionType.GET_FUNCTION_NATSPEC,
-               ActionType.GET_FILE_NATSPEC, ActionType.GET_IO,
+               ActionType.GET_FILE_NATSPEC, ActionType.GET_SIGNATURE,
                ActionType.GET_RELATED_FUNCTIONS, ActionType.GET_SIMILAR_RULE]:
         env.step(Action(action_type=at))
 
@@ -180,7 +180,7 @@ def check_t2_grader():
     from data.data_loader import load_contracts, get_all_property_entries
     for c, fn in get_all_property_entries(load_contracts()):
         g = Task2Grader(fn["name"], fn["property"])
-        assert g.grade(fn["property"]["natural_language"]) >= 0.65
+        assert g.grade(fn["property"])[0] >= 0.65
         assert g.grade("") == 0.0
         s = g.grade("test"); assert s == g.grade("test")  # deterministic
 
@@ -299,4 +299,4 @@ def main():
         sys.exit(0)
 
 if __name__ == "__main__":
-    main()
+    main()