Repository: UKGovernmentBEIS/inspect_ai
Issue: [bug] `eval_set` raises PrerequisiteError when rerun after KeyboardInterrupt
Root cause: Validation Gap

When an eval_set execution is interrupted (e.g., by KeyboardInterrupt) and then rerun to completion, on any subsequent reruns it fails with a PrerequisiteError.

This is quite a painful one because our tasks often hang and need to be manually cancelled, and then when they are rerun we often lose samples that were already completed.

The following script documents and reproduces the bug (running on OS X with inspect 0.3.42)

```python
import asyncio
import contextlib
import re
import signal
import uuid
from collections.abc import Generator
from types import FrameType

from inspect_ai import Task, eval_set, task
from inspect_ai._util.error import PrerequisiteError
from inspect_ai.dataset import MemoryDataset, Sample
from inspect_ai.log import read_eval_log
from inspect_ai.solver import Generate, TaskState, solver


def main():
    run_failing_scenario()
    print("\n\n========================================\n\n")
    run_successful_scenario()


def run_failing_scenario():
    log_dir = f"tmp/scratch_{str(uuid.uuid4())[:4]}"
    print(f"Using log directory: {log_dir}")

    def run_eval_set():
        eval_set(
            tasks=[sleep_for_5_task("foo"), sleep_for_1_task("bar")],
            log_dir=log_dir,
            max_tasks=2,
        )

    print("===First we run the evalset and simulate a KeyboardInterrupt after 3 seconds===")
    try:
        with keyboard_interrupt(3):
            run_eval_set()
    except KeyboardInterrupt:
        pass

    print("===Second we run the evalset to completion===")
    run_eval_set()

    print(
        "===Now we try to rerun the evalset. We expect it to succeed instantly because both tasks are",
        "already completed. Instead, it raises PrerequisiteError.===",
    )
    try:
        run_eval_set()
    except PrerequisiteError as error:
        match = re.search(r"Existing log file '([^']+)'", str(error))
        assert match
        filename: str = match.group(1)

        print(
            "===If we check the problematic log file we see that the task args are empty, even though we",
            "passed task args 'foo' and 'bar' to each task respectively. That's why it thinks that the",
            "file is not associated with a task we are trying to run===",
        )
        log = read_eval_log(f"{log_dir}/{filename}")
        assert log.eval.task_args == {}

        print(str(error))


def run_successful_scenario():
    print(
        "===In the successful scenario we run the evalset twice in a row without cancelling. This time"
        "everything works as expected.==="
    )

    log_dir = f"tmp/scratch_{str(uuid.uuid4())[:4]}"
    print(f"Using log directory: {log_dir}")

    def run_eval_set():
        eval_set(
            tasks=[sleep_for_5_task("foo"), sleep_for_1_task("bar")],
            log_dir=log_dir,
            max_tasks=2,
        )

    run_eval_set()
    run_eval_set()


@solver
def sleep_for_5_solver():
    async def solve(state: TaskState, generate: Generate):
        await asyncio.sleep(5)
        return state

    return solve


@task
def sleep_for_5_task(task_arg: str):
    return Task(
        dataset=MemoryDataset([Sample(input="")]),
        solver=[sleep_for_5_solver()],
    )


@solver
def sleep_for_1_solver():
    async def solve(state: TaskState, generate: Generate):
        await asyncio.sleep(1)
        return state

    return solve


@task
def sleep_for_1_task(task_arg: str):
    return Task(
        dataset=MemoryDataset([Sample(input="")]),
        solver=[sleep_for_1_solver()],
    )


@contextlib.contextmanager
def keyboard_interrupt(seconds: int) -> Generator[None, None, None]:
    def handler(signum: int, frame: FrameType | None) -> None:
        raise KeyboardInterrupt

    original_handler = signal.signal(signal.SIGALRM, handler)
    signal.alarm(seconds)

    try:
        yield
    finally:
        signal.alarm(0)
        signal.signal(signal.SIGALRM, original_handler)


if __name__ == "__main__":
    main()
```