submit.py

from __future__ import annotations

import argparse
import importlib.util
import json
import os
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from typing import Any, Dict, List, Tuple

from dotenv import load_dotenv


load_dotenv()


CORE_TASKS = [
    "bug_detection_easy_1",
    "memory_leak_medium_1",
    "security_hard_1",
]


def _run_cmd(command: List[str]) -> Tuple[int, str, str]:
    """Run a command and return (returncode, stdout, stderr)."""
    result = subprocess.run(
        command,
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
    )
    return result.returncode, result.stdout, result.stderr


def _require_command(binary: str) -> bool:
    """Return True if a command exists on PATH."""
    return shutil.which(binary) is not None


def run_validation() -> Tuple[bool, Dict[str, Any]]:
    print("Running OpenEnv validation...")
    if not _require_command("openenv"):
        return False, {
            "ok": False,
            "reason": "openenv command not found on PATH",
            "stdout": "",
            "stderr": "",
        }

    code, out, err = _run_cmd(["openenv", "validate"])
    if code != 0:
        print("Validation failed")
        return False, {"ok": False, "stdout": out, "stderr": err}

    print("Validation passed")
    return True, {"ok": True, "stdout": out, "stderr": err}


def run_tests(with_coverage: bool) -> Tuple[bool, Dict[str, Any]]:
    print("Running unit tests...")
    cmd = ["pytest", "tests/", "-v"]
    coverage_enabled = False
    coverage_reason = ""
    if with_coverage and importlib.util.find_spec("pytest_cov") is not None:
        cmd.extend(["--cov=environment", "--cov-report=html"])
        coverage_enabled = True
    elif with_coverage:
        coverage_reason = "pytest-cov not installed; ran tests without coverage"

    code, out, err = _run_cmd(cmd)
    if code != 0:
        print("Tests failed")
        return False, {
            "ok": False,
            "stdout": out,
            "stderr": err,
            "coverage_enabled": coverage_enabled,
            "coverage_reason": coverage_reason,
        }

    print("Tests passed")
    return True, {
        "ok": True,
        "stdout": out,
        "stderr": err,
        "coverage_enabled": coverage_enabled,
        "coverage_reason": coverage_reason,
    }


def check_docker(image_name: str) -> Tuple[bool, Dict[str, Any]]:
    print("Checking Docker build...")
    if not _require_command("docker"):
        return False, {
            "ok": False,
            "reason": "docker command not found on PATH",
            "stdout": "",
            "stderr": "",
        }

    info_code, info_out, info_err = _run_cmd(["docker", "info"])
    if info_code != 0:
        return False, {
            "ok": False,
            "reason": "docker daemon not reachable. Start Docker Desktop and retry.",
            "stdout": info_out,
            "stderr": info_err,
        }

    code, out, err = _run_cmd(["docker", "build", "-t", image_name, "."])
    if code != 0:
        print("Docker build failed")
        return False, {"ok": False, "stdout": out, "stderr": err}

    print("Docker build successful")
    return True, {"ok": True, "stdout": out, "stderr": err}


def _inference_env_ready() -> Tuple[bool, str]:
    if not (os.getenv("API_BASE_URL") or "").strip():
        return False, "API_BASE_URL is not set"
    if not (os.getenv("MODEL_NAME") or "").strip():
        return False, "MODEL_NAME is not set"
    token = (os.getenv("API_KEY") or os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY") or "").strip()
    if not token:
        return False, "API_KEY is not set (supported aliases: HF_TOKEN, OPENAI_API_KEY)"
    return True, ""


def run_baseline(tasks: List[str], max_steps: int) -> Tuple[bool, Dict[str, Any], Dict[str, float]]:
    """Run inference for each task and collect task_score from result JSONs."""
    print("Running baseline inference for core tasks...")

    ready, reason = _inference_env_ready()
    if not ready:
        print(f"Skipping baseline inference: {reason}")
        return False, {"ok": False, "reason": reason}, {}

    baseline_scores: Dict[str, float] = {}
    details: Dict[str, Any] = {"ok": True, "runs": []}

    for task_id in tasks:
        output_file = f"baseline_{task_id}.json"
        cmd = [
            sys.executable,
            "inference.py",
            "--task-id",
            task_id,
            "--max-steps",
            str(max_steps),
            "--output",
            output_file,
        ]
        code, out, err = _run_cmd(cmd)
        run_info: Dict[str, Any] = {
            "task_id": task_id,
            "ok": code == 0,
            "stdout": out,
            "stderr": err,
            "output_file": output_file,
        }

        if code != 0:
            details["ok"] = False
            details["runs"].append(run_info)
            continue

        # Inference currently catches model-call exceptions and can still exit 0
        # after using fallback actions. Treat this as a baseline failure signal.
        combined_logs = f"{out}\n{err}".lower()
        if "error getting action from llm" in combined_logs or "insufficient balance" in combined_logs:
            details["ok"] = False
            run_info["ok"] = False
            run_info["reason"] = "Model API call failed; fallback action used"

        try:
            with open(output_file, "r", encoding="utf-8") as fh:
                payload = json.load(fh)
            score = float(payload.get("task_score", 0.0))
            baseline_scores[task_id] = score
            run_info["task_score"] = score
        except (OSError, json.JSONDecodeError, ValueError) as exc:
            details["ok"] = False
            run_info["ok"] = False
            run_info["parse_error"] = str(exc)

        details["runs"].append(run_info)

    if details["ok"]:
        print("Baseline inference passed for all selected tasks")
    else:
        print("Baseline inference had failures")

    return bool(details["ok"]), details, baseline_scores


def generate_report(
    checks: Dict[str, Dict[str, Any]],
    baseline_scores: Dict[str, float],
    report_path: str,
) -> None:
    openenv_ok = checks["validation"]["ok"]
    tests_ok = checks["tests"]["ok"]
    docker_ok = checks["docker"]["ok"]
    baseline_ok = checks["baseline"]["ok"]

    report = {
        "project": "code-review-agent-env",
        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
        "tasks": CORE_TASKS,
        "difficulties": ["easy", "medium", "hard"],
        "openenv_compliant": openenv_ok,
        "docker_supported": docker_ok,
        "tests_passed": tests_ok,
        "baseline_passed": baseline_ok,
        "baseline_scores": baseline_scores,
        "checks": checks,
    }

    with open(report_path, "w", encoding="utf-8") as fh:
        json.dump(report, fh, indent=2)

    print(f"Submission report generated: {report_path}")


def main() -> int:
    parser = argparse.ArgumentParser(description="Pre-submission checklist for OpenEnv hackathon")
    parser.add_argument(
        "--skip-baseline",
        action="store_true",
        help="Skip inference baseline runs",
    )
    parser.add_argument(
        "--max-steps",
        type=int,
        default=50,
        help="Max steps for each baseline inference run",
    )
    parser.add_argument(
        "--no-coverage",
        action="store_true",
        help="Run tests without coverage output",
    )
    parser.add_argument(
        "--image-name",
        default="code-review-env",
        help="Docker image name for validation build",
    )
    parser.add_argument(
        "--report-path",
        default="submission_report.json",
        help="Where to write the JSON report",
    )
    args = parser.parse_args()

    print("=" * 50)
    print("Pre-submission Checklist")
    print("=" * 50)

    checks: Dict[str, Dict[str, Any]] = {}

    ok, detail = run_validation()
    checks["validation"] = detail

    ok, detail = run_tests(with_coverage=not args.no_coverage)
    checks["tests"] = detail

    ok, detail = check_docker(args.image_name)
    checks["docker"] = detail

    baseline_scores: Dict[str, float] = {}
    if args.skip_baseline:
        checks["baseline"] = {"ok": False, "skipped": True, "reason": "Skipped by --skip-baseline"}
    else:
        ok, detail, baseline_scores = run_baseline(CORE_TASKS, max_steps=args.max_steps)
        checks["baseline"] = detail

    generate_report(checks, baseline_scores, args.report_path)

    required_checks_ok = (
        checks["validation"]["ok"]
        and checks["tests"]["ok"]
        and checks["docker"]["ok"]
    )

    if required_checks_ok:
        print("\nRequired checks passed. Ready for submission.")
        return 0

    print("\nSome required checks failed. Please fix before submitting.")
    return 1


if __name__ == "__main__":
    raise SystemExit(main())