src/execute.py

import gradio as gr
import subprocess
import sys
import os
import threading
import time
import uuid
import glob
import shutil
from pathlib import Path

default_command = "bigcodebench.evaluate"
is_running = False

def generate_command(
    jsonl_file, split, subset, parallel,
    min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
    check_gt_only, no_gt
):
    command = [default_command]
    
    if jsonl_file is not None:
        # Copy the uploaded file to the current directory
        local_filename = os.path.basename(jsonl_file.name)
        shutil.copy(jsonl_file.name, local_filename)
        command.extend(["--samples", local_filename])
    
    command.extend(["--split", split, "--subset", subset])
    
    if parallel is not None and parallel != 0:
        command.extend(["--parallel", str(int(parallel))])
    
    command.extend([
        "--min-time-limit", str(min_time_limit),
        "--max-as-limit", str(int(max_as_limit)),
        "--max-data-limit", str(int(max_data_limit)),
        "--max-stack-limit", str(int(max_stack_limit))
    ])
    
    if check_gt_only:
        command.append("--check-gt-only")
    
    if no_gt:
        command.append("--no-gt")
    
    return " ".join(command)


def cleanup_previous_files(jsonl_file):
    if jsonl_file is not None:
        file_list = ['Dockerfile', 'app.py', 'README.md', os.path.basename(jsonl_file.name), "__pycache__"]
    else:
        file_list = ['Dockerfile', 'app.py', 'README.md', "__pycache__"]
    for file in glob.glob("*"):
        try:
            if file not in file_list:
                os.remove(file)
        except Exception as e:
            print(f"Error during cleanup of {file}: {e}")

def find_result_file():
    json_files = glob.glob("*.json")
    if json_files:
        return max(json_files, key=os.path.getmtime)
    return None

def run_bigcodebench(command):
    global is_running
    if is_running:
        yield "A command is already running. Please wait for it to finish.\n"
        return
    is_running = True

    try:
        yield f"Executing command: {command}\n"
        
        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
        
        def kill_process():
            if process.poll() is None:  # If the process is still running
                process.terminate()
                is_running = False
                yield "Process terminated after 12 minutes timeout.\n"

        # Start a timer to kill the process after 12 minutes
        timer = threading.Timer(720, kill_process)
        timer.start()
        
        for line in process.stdout:
            yield line
        
        # process.wait()
        
        timer.cancel()
        
        if process.returncode != 0:
            yield f"Error: Command exited with status {process.returncode}\n"
        
        yield "Evaluation completed.\n"
        
        result_file = find_result_file()
        if result_file:
            yield f"Result file found: {result_file}\n"
        else:
            yield "No result file found.\n"
    finally:
        is_running = False

def stream_logs(command, jsonl_file=None):
    global is_running
        
    if is_running:
        yield "A command is already running. Please wait for it to finish.\n"
        return
    
    cleanup_previous_files(jsonl_file)
    yield "Cleaned up previous files.\n"

    log_content = []
    for log_line in run_bigcodebench(command):
        log_content.append(log_line)
        yield "".join(log_content)
        
# with gr.Blocks() as demo:
#     gr.Markdown("# BigCodeBench Evaluator")
    
#     with gr.Row():
#         jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
#         split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
#         subset = gr.Dropdown(choices=["hard", "full"], label="Subset", value="hard")
    
#     with gr.Row():
#         parallel = gr.Number(label="Parallel (optional)", precision=0)
#         min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
#         max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
    
#     with gr.Row():
#         max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
#         max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
#         check_gt_only = gr.Checkbox(label="Check GT Only")
#         no_gt = gr.Checkbox(label="No GT")
    
#     command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
#     with gr.Row():
#         submit_btn = gr.Button("Run Evaluation")
#         download_btn = gr.DownloadButton(label="Download Result")
#     log_output = gr.Textbox(label="Execution Logs", lines=20)
    
#     input_components = [
#         jsonl_file, split, subset, parallel,
#         min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
#         check_gt_only, no_gt
#     ]
    
#     for component in input_components:
#         component.change(generate_command, inputs=input_components, outputs=command_output)
        
    
#     def start_evaluation(command, jsonl_file, subset, split):
#         extra = subset + "_" if subset != "full" else ""
#         if jsonl_file is not None:
#             result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
#         else:
#             result_path = None

#         for log in stream_logs(command, jsonl_file):
#             if jsonl_file is not None:
#                 yield log, gr.update(value=result_path, label=result_path), gr.update()
#             else:
#                 yield log, gr.update(), gr.update()
#         result_file = find_result_file()
#         if result_file:
#             return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
#                     # gr.Button(visible=False)#,
#                     # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
#         else:
#             return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
#                     # gr.Button("Run Evaluation", visible=True),
#                     # gr.DownloadButton(visible=False))
#     submit_btn.click(start_evaluation,
#                  inputs=[command_output, jsonl_file, subset, split],
#                  outputs=[log_output, download_btn])

# REPO_ID = "bigcode/bigcodebench-evaluator"
# HF_TOKEN = os.environ.get("HF_TOKEN", None)
# API = HfApi(token=HF_TOKEN)

# def restart_space():
#     API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)

# demo.queue(max_size=300).launch(share=True, server_name="0.0.0.0", server_port=7860)
# scheduler = BackgroundScheduler()
# scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h as backup in case automatic updates are not working
# scheduler.start()