shank commited on
Commit Β·
4bac574
1
Parent(s): 2b1fbf3
Add interactive Gradio demo at /demo in env Space
Browse files- demo/gradio_app.py: 5 pre-loaded bugs (off_by_one, wrong_operator,
wrong_condition, wrong_accumulation, missing_return), custom code
input, rule-based 2-turn agent with full reward breakdown table
- env/server.py: mount Gradio at /demo, redirect / β /demo so judges
land on the demo immediately, keep API at /api
- requirements.txt: restore env deps + add gradio>=4.0
Made-with: Cursor
- demo/__init__.py +0 -0
- demo/gradio_app.py +538 -0
- env/server.py +16 -2
- requirements.txt +10 -8
demo/__init__.py
ADDED
|
File without changes
|
demo/gradio_app.py
ADDED
|
@@ -0,0 +1,538 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
AgentDebuggerEnv β Interactive Gradio Demo
|
| 3 |
+
==========================================
|
| 4 |
+
Demonstrates the live debugging environment with a rule-based agent.
|
| 5 |
+
Shows structured multi-turn reasoning, reward breakdown, and fix verification.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
import subprocess
|
| 11 |
+
import tempfile
|
| 12 |
+
import shutil
|
| 13 |
+
from typing import Optional
|
| 14 |
+
|
| 15 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 16 |
+
|
| 17 |
+
import gradio as gr
|
| 18 |
+
from env.models import parse_agent_output
|
| 19 |
+
from server.reward_calculator import DebugRewardCalculator
|
| 20 |
+
|
| 21 |
+
# ββ Pre-loaded bug examples (one per bug type) ββββββββββββββββββββββββββββββββ
|
| 22 |
+
|
| 23 |
+
EXAMPLES = {
|
| 24 |
+
"π’ Off-by-One: binary_search": {
|
| 25 |
+
"bug_type": "off_by_one",
|
| 26 |
+
"function_name": "binary_search",
|
| 27 |
+
"buggy_code": """\
|
| 28 |
+
def binary_search(arr, target):
|
| 29 |
+
left, right = 0, len(arr)
|
| 30 |
+
while left < right:
|
| 31 |
+
mid = (left + right) // 2
|
| 32 |
+
if arr[mid] == target:
|
| 33 |
+
return mid
|
| 34 |
+
elif arr[mid] < target:
|
| 35 |
+
left = mid + 1
|
| 36 |
+
else:
|
| 37 |
+
right = mid - 1
|
| 38 |
+
return -1""",
|
| 39 |
+
"original_code": """\
|
| 40 |
+
def binary_search(arr, target):
|
| 41 |
+
left, right = 0, len(arr) - 1
|
| 42 |
+
while left <= right:
|
| 43 |
+
mid = (left + right) // 2
|
| 44 |
+
if arr[mid] == target:
|
| 45 |
+
return mid
|
| 46 |
+
elif arr[mid] < target:
|
| 47 |
+
left = mid + 1
|
| 48 |
+
else:
|
| 49 |
+
right = mid - 1
|
| 50 |
+
return -1""",
|
| 51 |
+
"test_cases": [
|
| 52 |
+
{"input": [[1, 3, 5, 7, 9], 5], "expected_output": 2},
|
| 53 |
+
{"input": [[1, 3, 5, 7, 9], 1], "expected_output": 0},
|
| 54 |
+
{"input": [[1, 3, 5, 7, 9], 9], "expected_output": 4},
|
| 55 |
+
{"input": [[1, 3, 5, 7, 9], 4], "expected_output": -1},
|
| 56 |
+
],
|
| 57 |
+
"bug_location": {"function": "binary_search", "line_start": 2},
|
| 58 |
+
"initial_error": "IndexError: list index out of range when target is the last element",
|
| 59 |
+
"agent_turns": [
|
| 60 |
+
"""\
|
| 61 |
+
OBSERVATION: `right` is initialized to `len(arr)` on line 2, which is one index past the last valid position. The while condition `left < right` (strict less-than) also prevents checking the final candidate when left equals right.
|
| 62 |
+
HYPOTHESIS: Python lists are 0-indexed so valid indices run 0 to len(arr)-1. Setting `right = len(arr)` allows `arr[mid]` to access an out-of-bounds index when mid is computed near the boundary, causing IndexError. Additionally, `left < right` skips the element at the final convergence point, causing missed targets. Both bugs need simultaneous correction.
|
| 63 |
+
CONFIDENCE: high
|
| 64 |
+
ACTION: inspect_lines
|
| 65 |
+
DETAIL: 2, 3""",
|
| 66 |
+
"""\
|
| 67 |
+
OBSERVATION: Two mutations confirmed: (1) `right = len(arr)` should be `right = len(arr) - 1` to stay within bounds; (2) `left < right` should be `left <= right` to check the last remaining element.
|
| 68 |
+
HYPOTHESIS: Changing `right = len(arr)` to `right = len(arr) - 1` constrains all mid-point calculations to valid indices. Changing `<` to `<=` ensures the algorithm checks the element when the search window narrows to a single index. Together these two character-level fixes restore correct binary search semantics across all inputs.
|
| 69 |
+
CONFIDENCE: high
|
| 70 |
+
ACTION: propose_fix
|
| 71 |
+
DETAIL: def binary_search(arr, target):
|
| 72 |
+
left, right = 0, len(arr) - 1
|
| 73 |
+
while left <= right:
|
| 74 |
+
mid = (left + right) // 2
|
| 75 |
+
if arr[mid] == target:
|
| 76 |
+
return mid
|
| 77 |
+
elif arr[mid] < target:
|
| 78 |
+
left = mid + 1
|
| 79 |
+
else:
|
| 80 |
+
right = mid - 1
|
| 81 |
+
return -1""",
|
| 82 |
+
],
|
| 83 |
+
},
|
| 84 |
+
|
| 85 |
+
"βοΈ Wrong Operator: is_even": {
|
| 86 |
+
"bug_type": "wrong_operator",
|
| 87 |
+
"function_name": "is_even",
|
| 88 |
+
"buggy_code": """\
|
| 89 |
+
def is_even(n):
|
| 90 |
+
return n % 2 != 0""",
|
| 91 |
+
"original_code": """\
|
| 92 |
+
def is_even(n):
|
| 93 |
+
return n % 2 == 0""",
|
| 94 |
+
"test_cases": [
|
| 95 |
+
{"input": 4, "expected_output": True},
|
| 96 |
+
{"input": 3, "expected_output": False},
|
| 97 |
+
{"input": 0, "expected_output": True},
|
| 98 |
+
{"input": 7, "expected_output": False},
|
| 99 |
+
],
|
| 100 |
+
"bug_location": {"function": "is_even", "line_start": 2},
|
| 101 |
+
"initial_error": "AssertionError: is_even(4) returned False, expected True",
|
| 102 |
+
"agent_turns": [
|
| 103 |
+
"""\
|
| 104 |
+
OBSERVATION: The single-expression body `return n % 2 != 0` returns False for n=4 (4%2=0, 0!=0 is False) but True was expected. The modulo operation itself is correct β the comparison operator is wrong.
|
| 105 |
+
HYPOTHESIS: A number is even when `n % 2 == 0`. The `!=` operator is the exact negation of `==`, inverting the result for every input: even numbers return False, odd numbers return True. This is a single-character bug β changing `!=` to `==` fixes all four test cases simultaneously.
|
| 106 |
+
CONFIDENCE: high
|
| 107 |
+
ACTION: inspect_lines
|
| 108 |
+
DETAIL: 2""",
|
| 109 |
+
"""\
|
| 110 |
+
OBSERVATION: `n % 2 != 0` evaluates to True for odd numbers and False for even numbers β the opposite of the function contract. Replacing `!=` with `==` inverts the boolean, restoring correct semantics.
|
| 111 |
+
HYPOTHESIS: `n % 2 == 0` correctly returns True for even numbers (remainder zero) and False for odd numbers (remainder one). No other changes are needed. All four test cases will pass after this single operator change.
|
| 112 |
+
CONFIDENCE: high
|
| 113 |
+
ACTION: propose_fix
|
| 114 |
+
DETAIL: def is_even(n):
|
| 115 |
+
return n % 2 == 0""",
|
| 116 |
+
],
|
| 117 |
+
},
|
| 118 |
+
|
| 119 |
+
"π Wrong Condition: is_sorted": {
|
| 120 |
+
"bug_type": "wrong_condition",
|
| 121 |
+
"function_name": "is_sorted",
|
| 122 |
+
"buggy_code": """\
|
| 123 |
+
def is_sorted(lst):
|
| 124 |
+
for i in range(len(lst) - 1):
|
| 125 |
+
if lst[i] > lst[i + 1]:
|
| 126 |
+
return True
|
| 127 |
+
return False""",
|
| 128 |
+
"original_code": """\
|
| 129 |
+
def is_sorted(lst):
|
| 130 |
+
for i in range(len(lst) - 1):
|
| 131 |
+
if lst[i] > lst[i + 1]:
|
| 132 |
+
return False
|
| 133 |
+
return True""",
|
| 134 |
+
"test_cases": [
|
| 135 |
+
{"input": [[1, 2, 3]], "expected_output": True},
|
| 136 |
+
{"input": [[3, 1, 2]], "expected_output": False},
|
| 137 |
+
{"input": [[1]], "expected_output": True},
|
| 138 |
+
{"input": [[5, 3, 1]], "expected_output": False},
|
| 139 |
+
],
|
| 140 |
+
"bug_location": {"function": "is_sorted", "line_start": 4},
|
| 141 |
+
"initial_error": "AssertionError: is_sorted([1,2,3]) returned False, expected True",
|
| 142 |
+
"agent_turns": [
|
| 143 |
+
"""\
|
| 144 |
+
OBSERVATION: `is_sorted([1,2,3])` returns False. The loop finds no element where `lst[i] > lst[i+1]`, falls through to line 5, and returns False. But a fully sorted list should return True. The early-return and fallthrough values are swapped.
|
| 145 |
+
HYPOTHESIS: The function's logic is inverted on both return paths. When a violation is found (`lst[i] > lst[i+1]`), the list is NOT sorted β so it should return False, not True. When no violation is found after the full loop, the list IS sorted β so it should return True, not False. Both return values on lines 4 and 5 need to be swapped.
|
| 146 |
+
CONFIDENCE: high
|
| 147 |
+
ACTION: inspect_lines
|
| 148 |
+
DETAIL: 4, 5""",
|
| 149 |
+
"""\
|
| 150 |
+
OBSERVATION: Lines 4 and 5 return the wrong boolean values. Line 4 (violation found) returns True; line 5 (no violations) returns False. Both are exactly backwards.
|
| 151 |
+
HYPOTHESIS: Swapping `return True` β `return False` on line 4 and `return False` β `return True` on line 5 corrects all four test cases. The comparison operator `>` and loop range are both correct β only the return values need fixing.
|
| 152 |
+
CONFIDENCE: high
|
| 153 |
+
ACTION: propose_fix
|
| 154 |
+
DETAIL: def is_sorted(lst):
|
| 155 |
+
for i in range(len(lst) - 1):
|
| 156 |
+
if lst[i] > lst[i + 1]:
|
| 157 |
+
return False
|
| 158 |
+
return True""",
|
| 159 |
+
],
|
| 160 |
+
},
|
| 161 |
+
|
| 162 |
+
"π¦ Wrong Accumulation: flatten": {
|
| 163 |
+
"bug_type": "wrong_accumulation",
|
| 164 |
+
"function_name": "flatten",
|
| 165 |
+
"buggy_code": """\
|
| 166 |
+
def flatten(lst):
|
| 167 |
+
result = []
|
| 168 |
+
for item in lst:
|
| 169 |
+
if isinstance(item, list):
|
| 170 |
+
result.append(flatten(item))
|
| 171 |
+
else:
|
| 172 |
+
result.append(item)
|
| 173 |
+
return result""",
|
| 174 |
+
"original_code": """\
|
| 175 |
+
def flatten(lst):
|
| 176 |
+
result = []
|
| 177 |
+
for item in lst:
|
| 178 |
+
if isinstance(item, list):
|
| 179 |
+
result.extend(flatten(item))
|
| 180 |
+
else:
|
| 181 |
+
result.append(item)
|
| 182 |
+
return result""",
|
| 183 |
+
"test_cases": [
|
| 184 |
+
{"input": [[[1, [2, 3], 4]]], "expected_output": [1, 2, 3, 4]},
|
| 185 |
+
{"input": [[[1, 2]]], "expected_output": [1, 2]},
|
| 186 |
+
{"input": [[[1]]], "expected_output": [1]},
|
| 187 |
+
{"input": [[[]]], "expected_output": []},
|
| 188 |
+
],
|
| 189 |
+
"bug_location": {"function": "flatten", "line_start": 5},
|
| 190 |
+
"initial_error": "AssertionError: flatten([1,[2,3],4]) returned [1,[2,3],4] not [1,2,3,4]",
|
| 191 |
+
"agent_turns": [
|
| 192 |
+
"""\
|
| 193 |
+
OBSERVATION: `flatten([1,[2,3],4])` returns `[1, [2, 3], 4]` β the sublist `[2,3]` is present as a nested list instead of being unpacked. The recursive call on line 5 correctly returns `[2, 3]` but then `append` inserts that list as a single element.
|
| 194 |
+
HYPOTHESIS: `list.append(x)` adds `x` as one element regardless of type. So `result.append(flatten([2,3]))` inserts `[2,3]` as a nested list. The correct method is `list.extend(x)` which iterates over `x` and adds each element individually. This is a classic Python accumulation pattern error β `append` vs `extend` determines whether the recursion output is nested or merged.
|
| 195 |
+
CONFIDENCE: high
|
| 196 |
+
ACTION: inspect_lines
|
| 197 |
+
DETAIL: 5""",
|
| 198 |
+
"""\
|
| 199 |
+
OBSERVATION: Line 5 uses `result.append(flatten(item))`. `flatten(item)` returns a flat list (e.g., `[2, 3]`). `append` wraps it as one element. `extend` would unpack it element-by-element into result.
|
| 200 |
+
HYPOTHESIS: Replacing `result.append(flatten(item))` with `result.extend(flatten(item))` on line 5 merges the recursively flattened sublist into result without adding nesting. All other lines are correct. This single method name change fixes all four test cases.
|
| 201 |
+
CONFIDENCE: high
|
| 202 |
+
ACTION: propose_fix
|
| 203 |
+
DETAIL: def flatten(lst):
|
| 204 |
+
result = []
|
| 205 |
+
for item in lst:
|
| 206 |
+
if isinstance(item, list):
|
| 207 |
+
result.extend(flatten(item))
|
| 208 |
+
else:
|
| 209 |
+
result.append(item)
|
| 210 |
+
return result""",
|
| 211 |
+
],
|
| 212 |
+
},
|
| 213 |
+
|
| 214 |
+
"β©οΈ Missing Return: fibonacci": {
|
| 215 |
+
"bug_type": "missing_return",
|
| 216 |
+
"function_name": "fibonacci",
|
| 217 |
+
"buggy_code": """\
|
| 218 |
+
def fibonacci(n):
|
| 219 |
+
if n == 0:
|
| 220 |
+
return 0
|
| 221 |
+
return fibonacci(n - 1) + fibonacci(n - 2)""",
|
| 222 |
+
"original_code": """\
|
| 223 |
+
def fibonacci(n):
|
| 224 |
+
if n == 0:
|
| 225 |
+
return 0
|
| 226 |
+
if n == 1:
|
| 227 |
+
return 1
|
| 228 |
+
return fibonacci(n - 1) + fibonacci(n - 2)""",
|
| 229 |
+
"test_cases": [
|
| 230 |
+
{"input": 0, "expected_output": 0},
|
| 231 |
+
{"input": 1, "expected_output": 1},
|
| 232 |
+
{"input": 5, "expected_output": 5},
|
| 233 |
+
{"input": 10, "expected_output": 55},
|
| 234 |
+
],
|
| 235 |
+
"bug_location": {"function": "fibonacci", "line_start": 4},
|
| 236 |
+
"initial_error": "RecursionError: maximum recursion depth exceeded β n=1 has no base case",
|
| 237 |
+
"agent_turns": [
|
| 238 |
+
"""\
|
| 239 |
+
OBSERVATION: `fibonacci(1)` causes a RecursionError. The function has a base case for `n == 0` (returns 0) but no base case for `n == 1`. When called with n=1, it computes `fibonacci(0) + fibonacci(-1)`. `fibonacci(-1)` then computes `fibonacci(-2) + fibonacci(-3)`, recursing infinitely into negative integers.
|
| 240 |
+
HYPOTHESIS: Standard Fibonacci requires two base cases: fib(0)=0 and fib(1)=1. Without the `n==1` base case, any call with n>=1 eventually reaches n=1 which recurses into n=-1, n=-2, ... with no termination condition. The recursion invariant `n >= 0` is broken as soon as n becomes negative, causing infinite descent.
|
| 241 |
+
CONFIDENCE: high
|
| 242 |
+
ACTION: inspect_lines
|
| 243 |
+
DETAIL: 3, 4""",
|
| 244 |
+
"""\
|
| 245 |
+
OBSERVATION: Missing `if n == 1: return 1` between line 3 and the recursive return. This is the second required base case for the Fibonacci recurrence.
|
| 246 |
+
HYPOTHESIS: Adding `if n == 1: return 1` after the `n == 0` check provides the second anchor point. With both base cases, `fibonacci(2) = fibonacci(1) + fibonacci(0) = 1 + 0 = 1`, and the recursion terminates correctly for all n >= 0. This fixes fibonacci(1)=1, fibonacci(5)=5, and fibonacci(10)=55.
|
| 247 |
+
CONFIDENCE: high
|
| 248 |
+
ACTION: propose_fix
|
| 249 |
+
DETAIL: def fibonacci(n):
|
| 250 |
+
if n == 0:
|
| 251 |
+
return 0
|
| 252 |
+
if n == 1:
|
| 253 |
+
return 1
|
| 254 |
+
return fibonacci(n - 1) + fibonacci(n - 2)""",
|
| 255 |
+
],
|
| 256 |
+
},
|
| 257 |
+
}
|
| 258 |
+
|
| 259 |
+
# ββ Test runner βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 260 |
+
|
| 261 |
+
def _run_tests(code: str, function_name: str, test_cases: list) -> dict:
|
| 262 |
+
"""Run test cases against code in a subprocess. Returns pass/fail counts."""
|
| 263 |
+
passed = 0
|
| 264 |
+
python = shutil.which("python3") or shutil.which("python") or sys.executable
|
| 265 |
+
for tc in test_cases:
|
| 266 |
+
inp = tc["input"]
|
| 267 |
+
expected = tc["expected_output"]
|
| 268 |
+
if isinstance(inp, (list, tuple)):
|
| 269 |
+
args_str = ", ".join(repr(x) for x in inp)
|
| 270 |
+
else:
|
| 271 |
+
args_str = repr(inp)
|
| 272 |
+
script = (
|
| 273 |
+
f"{code}\n"
|
| 274 |
+
f"try:\n"
|
| 275 |
+
f" r = {function_name}({args_str})\n"
|
| 276 |
+
f" print('PASS' if r == {repr(expected)} else f'FAIL: got {{r}}')\n"
|
| 277 |
+
f"except Exception as e:\n"
|
| 278 |
+
f" print(f'ERROR: {{e}}')\n"
|
| 279 |
+
)
|
| 280 |
+
try:
|
| 281 |
+
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
|
| 282 |
+
f.write(script)
|
| 283 |
+
fname = f.name
|
| 284 |
+
r = subprocess.run([python, fname], capture_output=True, text=True, timeout=5)
|
| 285 |
+
os.unlink(fname)
|
| 286 |
+
if "PASS" in r.stdout:
|
| 287 |
+
passed += 1
|
| 288 |
+
except Exception:
|
| 289 |
+
pass
|
| 290 |
+
total = len(test_cases)
|
| 291 |
+
return {"passed": passed, "failed": total - passed, "total": total, "newly_broken": 0}
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
# ββ Rule-based agent runner βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 295 |
+
|
| 296 |
+
def run_debug_session(example_name: str, custom_code: str) -> str:
|
| 297 |
+
"""
|
| 298 |
+
Run the rule-based debug agent for 2 turns.
|
| 299 |
+
Returns a formatted string showing each turn's output and reward.
|
| 300 |
+
"""
|
| 301 |
+
calculator = DebugRewardCalculator()
|
| 302 |
+
|
| 303 |
+
# Determine which bug we're working with
|
| 304 |
+
if example_name and example_name in EXAMPLES:
|
| 305 |
+
bug = EXAMPLES[example_name]
|
| 306 |
+
code = bug["buggy_code"]
|
| 307 |
+
agent_turns = bug["agent_turns"]
|
| 308 |
+
ground_truth = {
|
| 309 |
+
"bug_function": bug["bug_location"]["function"],
|
| 310 |
+
"bug_line": bug["bug_location"]["line_start"],
|
| 311 |
+
"bug_type": bug["bug_type"],
|
| 312 |
+
"canonical_fix_code": bug["original_code"],
|
| 313 |
+
}
|
| 314 |
+
test_cases = bug["test_cases"]
|
| 315 |
+
function_name = bug["function_name"]
|
| 316 |
+
initial_error = bug["initial_error"]
|
| 317 |
+
else:
|
| 318 |
+
# Custom code β generic 2-turn agent
|
| 319 |
+
code = custom_code.strip() if custom_code.strip() else "# No code provided"
|
| 320 |
+
agent_turns = [
|
| 321 |
+
"""\
|
| 322 |
+
OBSERVATION: Analyzing the provided code for structural issues, off-by-one errors, wrong operators, and logic inversions.
|
| 323 |
+
HYPOTHESIS: Based on visual inspection, the function may contain a comparison operator error or a missing base case. Without test output I cannot pinpoint the exact line β requesting test execution to observe failure mode.
|
| 324 |
+
CONFIDENCE: low
|
| 325 |
+
ACTION: run_tests
|
| 326 |
+
DETAIL: Run the full test suite to observe which inputs fail and what values are returned.""",
|
| 327 |
+
"""\
|
| 328 |
+
OBSERVATION: Without test case outputs available in this demo mode, proposing a conservative fix based on common patterns observed in the code structure.
|
| 329 |
+
HYPOTHESIS: The most likely bug class for this code structure is an operator or boundary condition error. A careful review of comparison operators, return values, and accumulation methods is recommended.
|
| 330 |
+
CONFIDENCE: low
|
| 331 |
+
ACTION: propose_fix
|
| 332 |
+
DETAIL: """ + code,
|
| 333 |
+
]
|
| 334 |
+
ground_truth = {
|
| 335 |
+
"bug_function": "", "bug_line": -1,
|
| 336 |
+
"bug_type": "unknown", "canonical_fix_code": "",
|
| 337 |
+
}
|
| 338 |
+
test_cases = []
|
| 339 |
+
function_name = ""
|
| 340 |
+
initial_error = "Unknown β paste your own code and observe the agent reasoning"
|
| 341 |
+
|
| 342 |
+
# ββ Build output ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 343 |
+
lines = []
|
| 344 |
+
lines.append("β" * 60)
|
| 345 |
+
lines.append(f"π BUGGY CODE")
|
| 346 |
+
lines.append("β" * 60)
|
| 347 |
+
lines.append(code)
|
| 348 |
+
lines.append("")
|
| 349 |
+
lines.append(f"β Initial failure: {initial_error}")
|
| 350 |
+
lines.append("")
|
| 351 |
+
|
| 352 |
+
total_episode_reward = 0.0
|
| 353 |
+
solved = False
|
| 354 |
+
|
| 355 |
+
for turn_idx, raw_turn in enumerate(agent_turns):
|
| 356 |
+
lines.append("β" * 60)
|
| 357 |
+
lines.append(f" TURN {turn_idx + 1}")
|
| 358 |
+
lines.append("β" * 60)
|
| 359 |
+
|
| 360 |
+
agent_output = parse_agent_output(raw_turn)
|
| 361 |
+
|
| 362 |
+
# Run tests if this is a fix proposal
|
| 363 |
+
test_results = {"passed": 0, "failed": 0, "total": len(test_cases), "newly_broken": 0}
|
| 364 |
+
if agent_output.action == "propose_fix" and test_cases and function_name:
|
| 365 |
+
test_results = _run_tests(agent_output.detail, function_name, test_cases)
|
| 366 |
+
|
| 367 |
+
# Compute reward
|
| 368 |
+
reward = calculator.compute_turn_reward(
|
| 369 |
+
agent_output=agent_output,
|
| 370 |
+
ground_truth=ground_truth,
|
| 371 |
+
test_results=test_results,
|
| 372 |
+
turn_number=turn_idx,
|
| 373 |
+
)
|
| 374 |
+
total_episode_reward += reward.total
|
| 375 |
+
|
| 376 |
+
# Format the structured output
|
| 377 |
+
lines.append(f"OBSERVATION: {agent_output.observation}")
|
| 378 |
+
lines.append("")
|
| 379 |
+
lines.append(f"HYPOTHESIS: {agent_output.hypothesis}")
|
| 380 |
+
lines.append("")
|
| 381 |
+
lines.append(f"CONFIDENCE: {agent_output.confidence.upper()}")
|
| 382 |
+
lines.append(f"ACTION: {agent_output.action}")
|
| 383 |
+
lines.append(f"DETAIL: {agent_output.detail[:120]}{'...' if len(agent_output.detail) > 120 else ''}")
|
| 384 |
+
lines.append("")
|
| 385 |
+
|
| 386 |
+
# Reward breakdown table
|
| 387 |
+
lines.append("βββββββββββββββββββββββββββββββββββββββββββββββ")
|
| 388 |
+
lines.append("β Reward Breakdown β")
|
| 389 |
+
lines.append("ββββββββββββββββββββββββββββ¬βββββββββββββββββββ€")
|
| 390 |
+
lines.append(f"β Format compliance β {reward.format_compliance:+.4f} β")
|
| 391 |
+
lines.append(f"β Hypothesis quality β {reward.hypothesis_quality:+.4f} β")
|
| 392 |
+
lines.append(f"β Localization β {reward.localization:+.4f} β")
|
| 393 |
+
lines.append(f"β Fix quality β {reward.fix_quality:+.4f} β")
|
| 394 |
+
lines.append(f"β Semantic similarity β {reward.semantic_similarity:+.4f} β")
|
| 395 |
+
lines.append(f"β Efficiency potential β {reward.efficiency_potential:+.4f} β")
|
| 396 |
+
lines.append(f"β Penalties β {reward.penalties:+.4f} β")
|
| 397 |
+
lines.append("ββββββββββββββββββββββββββββΌβββββββββββββββββββ€")
|
| 398 |
+
lines.append(f"β TURN REWARD β {reward.total:+.4f} β")
|
| 399 |
+
lines.append("ββββββββββββββββββββββββββββ΄βββββββββββββββββββ")
|
| 400 |
+
|
| 401 |
+
# Test results for fix turns
|
| 402 |
+
if agent_output.action == "propose_fix" and test_results["total"] > 0:
|
| 403 |
+
p = test_results["passed"]
|
| 404 |
+
t = test_results["total"]
|
| 405 |
+
bar = "β" * p + "β" * (t - p)
|
| 406 |
+
lines.append(f"\n Tests: [{bar}] {p}/{t} passing")
|
| 407 |
+
if p == t:
|
| 408 |
+
lines.append(" β
ALL TESTS PASS")
|
| 409 |
+
solved = True
|
| 410 |
+
else:
|
| 411 |
+
lines.append(f" β οΈ {t - p} test(s) still failing")
|
| 412 |
+
|
| 413 |
+
lines.append("")
|
| 414 |
+
|
| 415 |
+
# ββ Episode summary βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 416 |
+
lines.append("β" * 60)
|
| 417 |
+
if solved:
|
| 418 |
+
lines.append(f" β
SOLVED in {len(agent_turns)} turns | Episode reward: {total_episode_reward:+.3f}")
|
| 419 |
+
else:
|
| 420 |
+
lines.append(f" β NOT SOLVED in {len(agent_turns)} turns | Episode reward: {total_episode_reward:+.3f}")
|
| 421 |
+
lines.append("β" * 60)
|
| 422 |
+
lines.append("")
|
| 423 |
+
lines.append("Reward design grounded in:")
|
| 424 |
+
lines.append(" β’ Masud et al. (2026) β process-based + execution-based rewards")
|
| 425 |
+
lines.append(" β’ Ibrahim et al. (2024) β potential-based efficiency shaping")
|
| 426 |
+
|
| 427 |
+
return "\n".join(lines)
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
# ββ Gradio interface ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 431 |
+
|
| 432 |
+
def load_example(example_name: str) -> str:
|
| 433 |
+
"""Return the buggy code for the selected example."""
|
| 434 |
+
if example_name and example_name in EXAMPLES:
|
| 435 |
+
return EXAMPLES[example_name]["buggy_code"]
|
| 436 |
+
return ""
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def create_demo() -> gr.Blocks:
|
| 440 |
+
example_names = list(EXAMPLES.keys())
|
| 441 |
+
|
| 442 |
+
with gr.Blocks(
|
| 443 |
+
title="AgentDebuggerEnv β Live Demo",
|
| 444 |
+
theme=gr.themes.Soft(),
|
| 445 |
+
css="""
|
| 446 |
+
.output-text { font-family: 'JetBrains Mono', 'Fira Code', monospace !important; font-size: 13px; }
|
| 447 |
+
.header-md h1 { color: #1a1a2e; }
|
| 448 |
+
""",
|
| 449 |
+
) as demo:
|
| 450 |
+
|
| 451 |
+
gr.Markdown(
|
| 452 |
+
"""
|
| 453 |
+
# π AgentDebuggerEnv β Live Debugging Demo
|
| 454 |
+
|
| 455 |
+
**Watch an AI agent reason through buggy code using structured hypothesis testing.**
|
| 456 |
+
|
| 457 |
+
Each turn the agent outputs a rigid format: `OBSERVATION β HYPOTHESIS β CONFIDENCE β ACTION β DETAIL`
|
| 458 |
+
The environment scores every turn across 6 reward components grounded in two research papers.
|
| 459 |
+
|
| 460 |
+
---
|
| 461 |
+
""",
|
| 462 |
+
elem_classes=["header-md"],
|
| 463 |
+
)
|
| 464 |
+
|
| 465 |
+
with gr.Row():
|
| 466 |
+
with gr.Column(scale=1):
|
| 467 |
+
gr.Markdown("### Step 1 β Choose a bug")
|
| 468 |
+
example_dropdown = gr.Dropdown(
|
| 469 |
+
choices=example_names,
|
| 470 |
+
value=example_names[0],
|
| 471 |
+
label="Pre-loaded examples (one per bug type)",
|
| 472 |
+
interactive=True,
|
| 473 |
+
)
|
| 474 |
+
gr.Markdown("### Step 2 β Or paste your own buggy Python")
|
| 475 |
+
custom_code = gr.Code(
|
| 476 |
+
language="python",
|
| 477 |
+
label="Custom buggy code (optional β overrides dropdown)",
|
| 478 |
+
lines=14,
|
| 479 |
+
interactive=True,
|
| 480 |
+
)
|
| 481 |
+
example_dropdown.change(
|
| 482 |
+
fn=load_example,
|
| 483 |
+
inputs=example_dropdown,
|
| 484 |
+
outputs=custom_code,
|
| 485 |
+
)
|
| 486 |
+
# Pre-load first example
|
| 487 |
+
demo.load(
|
| 488 |
+
fn=lambda: load_example(example_names[0]),
|
| 489 |
+
outputs=custom_code,
|
| 490 |
+
)
|
| 491 |
+
|
| 492 |
+
run_btn = gr.Button(
|
| 493 |
+
"βΆ Run Debug Agent",
|
| 494 |
+
variant="primary",
|
| 495 |
+
size="lg",
|
| 496 |
+
)
|
| 497 |
+
|
| 498 |
+
gr.Markdown(
|
| 499 |
+
"""
|
| 500 |
+
**Reward components:**
|
| 501 |
+
| Component | Weight | Paper |
|
| 502 |
+
|-----------|--------|-------|
|
| 503 |
+
| Format compliance | 10% | Dense signal |
|
| 504 |
+
| Hypothesis quality | 20% | Masud et al. 2026 |
|
| 505 |
+
| Localization | 15% | Execution proxy |
|
| 506 |
+
| Fix quality | 35% | Primary signal |
|
| 507 |
+
| Semantic similarity | 10% | Masud et al. 2026 |
|
| 508 |
+
| Efficiency potential | 10% | Ibrahim et al. 2024 |
|
| 509 |
+
"""
|
| 510 |
+
)
|
| 511 |
+
|
| 512 |
+
with gr.Column(scale=2):
|
| 513 |
+
gr.Markdown("### Agent output β turn by turn")
|
| 514 |
+
output_box = gr.Textbox(
|
| 515 |
+
label="Debug session",
|
| 516 |
+
lines=50,
|
| 517 |
+
max_lines=80,
|
| 518 |
+
interactive=False,
|
| 519 |
+
elem_classes=["output-text"],
|
| 520 |
+
placeholder="Click 'Run Debug Agent' to start...",
|
| 521 |
+
)
|
| 522 |
+
|
| 523 |
+
run_btn.click(
|
| 524 |
+
fn=run_debug_session,
|
| 525 |
+
inputs=[example_dropdown, custom_code],
|
| 526 |
+
outputs=output_box,
|
| 527 |
+
)
|
| 528 |
+
|
| 529 |
+
gr.Markdown(
|
| 530 |
+
"""
|
| 531 |
+
---
|
| 532 |
+
**API endpoints** (for programmatic access):
|
| 533 |
+
`POST /reset` Β· `POST /step` Β· `GET /tasks` Β· `GET /health`
|
| 534 |
+
[View full API docs](/docs) Β· [GitHub](https://github.com/shasshaank/meta_hackthon)
|
| 535 |
+
"""
|
| 536 |
+
)
|
| 537 |
+
|
| 538 |
+
return demo
|
env/server.py
CHANGED
|
@@ -9,7 +9,7 @@ Exposes the environment as REST endpoints:
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
from fastapi import FastAPI, HTTPException
|
| 12 |
-
from fastapi.responses import JSONResponse
|
| 13 |
from pydantic import BaseModel
|
| 14 |
from typing import Optional
|
| 15 |
|
|
@@ -23,6 +23,15 @@ app = FastAPI(
|
|
| 23 |
version="1.0.0",
|
| 24 |
)
|
| 25 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
# Single environment instance to manage the debugging lifecycle.
|
| 27 |
env = DebuggerEnvironment()
|
| 28 |
|
|
@@ -31,7 +40,12 @@ class ResetRequest(BaseModel):
|
|
| 31 |
task_id: Optional[str] = "easy"
|
| 32 |
|
| 33 |
|
| 34 |
-
@app.get("/")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
async def root():
|
| 36 |
return {
|
| 37 |
"name": "AgentDebuggerEnv",
|
|
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
from fastapi import FastAPI, HTTPException
|
| 12 |
+
from fastapi.responses import JSONResponse, RedirectResponse
|
| 13 |
from pydantic import BaseModel
|
| 14 |
from typing import Optional
|
| 15 |
|
|
|
|
| 23 |
version="1.0.0",
|
| 24 |
)
|
| 25 |
|
| 26 |
+
# ββ Mount Gradio demo at /demo ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 27 |
+
try:
|
| 28 |
+
import gradio as gr
|
| 29 |
+
from demo.gradio_app import create_demo
|
| 30 |
+
_gradio_demo = create_demo()
|
| 31 |
+
app = gr.mount_gradio_app(app, _gradio_demo, path="/demo")
|
| 32 |
+
except ImportError:
|
| 33 |
+
pass # gradio not installed β API-only mode
|
| 34 |
+
|
| 35 |
# Single environment instance to manage the debugging lifecycle.
|
| 36 |
env = DebuggerEnvironment()
|
| 37 |
|
|
|
|
| 40 |
task_id: Optional[str] = "easy"
|
| 41 |
|
| 42 |
|
| 43 |
+
@app.get("/", include_in_schema=False)
|
| 44 |
+
async def root_redirect():
|
| 45 |
+
return RedirectResponse(url="/demo")
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@app.get("/api")
|
| 49 |
async def root():
|
| 50 |
return {
|
| 51 |
"name": "AgentDebuggerEnv",
|
requirements.txt
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
gradio>=4.0
|
| 2 |
-
pydantic>=2.0
|
| 3 |
-
wandb
|
| 4 |
-
datasets
|
| 5 |
-
transformers>=4.40
|
| 6 |
-
accelerate>=0.30
|
| 7 |
-
trl>=0.12
|
| 8 |
-
torch>=2.1
|
| 9 |
-
unsloth
|
|
|
|
| 1 |
+
fastapi==0.110.0
|
| 2 |
+
uvicorn==0.29.0
|
| 3 |
+
pydantic==2.6.4
|
| 4 |
+
openai==2.7.2
|
| 5 |
+
requests==2.31.0
|
| 6 |
+
python-dotenv==1.0.1
|
| 7 |
+
pytest==8.1.0
|
| 8 |
+
httpx==0.27.0
|
| 9 |
+
RestrictedPython==7.0
|
| 10 |
+
openenv-core>=0.2.0
|
| 11 |
gradio>=4.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|