update endpoint helper files

evaluate_model.py

@@ -1,7 +1,8 @@
-import argparse
-import json
-import subprocess
-import sys
+import argparse
+import ast
+import json
+import subprocess
+import sys
 
 
 DEFAULT_TEST_PROMPTS = [
@@ -50,14 +51,63 @@ def run_inference(python_exec, model_path, base_model, prompt, max_new_tokens, a
         return None, f"invalid json output: {exc}: {stdout[:300]}"
 
 
-def safe_float(value):
+def safe_float(value):
     try:
         return float(value)
     except (TypeError, ValueError):
-        return 0.0
-
-
-def score_payload(payload):
+        return 0.0
+
+
+def prompt_expects_code(prompt):
+    prompt_l = prompt.lower()
+    markers = (
+        "fix",
+        "debug",
+        "repair",
+        "write",
+        "create",
+        "generate",
+        "implement",
+        "function",
+        "code",
+        "snippet",
+        "python",
+        "multiply",
+        "multiplication",
+        "product",
+        "add",
+        "addition",
+        "sum",
+        "subtract",
+        "subtraction",
+        "difference",
+        "divide",
+        "division",
+        "quotient",
+    )
+    return any(marker in prompt_l for marker in markers)
+
+
+def code_is_valid_for_prompt(prompt, code):
+    code = str(code or "").strip()
+    if not code:
+        return False
+    if not prompt_expects_code(prompt):
+        return True
+    python_like = any(
+        marker in code
+        for marker in ("def ", "import ", "class ", "print(", "return ", "for ", "if ")
+    )
+    if not python_like:
+        return False
+    try:
+        ast.parse(code)
+        return True
+    except SyntaxError:
+        return False
+
+
+def score_payload(prompt, payload):
     required_keys = {
         "code",
         "explanation",
@@ -69,7 +119,7 @@ def score_payload(payload):
         "latency_ms",
     }
     has_all_keys = required_keys.issubset(payload.keys())
-    code_ok = bool(str(payload.get("code", "")).strip())
+    code_ok = code_is_valid_for_prompt(prompt, payload.get("code", ""))
     explanation_ok = bool(str(payload.get("explanation", "")).strip())
     confidence = safe_float(payload.get("confidence", 0.0))
     relevancy = safe_float(payload.get("relevancy_score", 0.0))
@@ -116,7 +166,7 @@ def main():
             results.append({"prompt": prompt, "error": error, "pass": False})
             continue
 
-        metrics = score_payload(payload)
+        metrics = score_payload(prompt, payload)
         is_pass = (
             metrics["schema_ok"]
             and metrics["content_ok"]