Repository: UKGovernmentBEIS/inspect_ai
Issue: Adding a Task setup method leads to RuntimeError "Log not properly initialised"
Root cause: Documentation Deficiency

The `setup` argument for a Task is supposed to let a solver run during task setup, but on the current head of main (cd6d9488ac383d38391da6ae807620a7644dddd7) it seems like adding any setup argument to a Task leads to an error.

I copied the example setup solver from [the docs](https://inspect.aisi.org.uk/tasks.html#setup-parameter) and made minimal task and it reproduces the error:

```py
from inspect_ai import Task, task
from inspect_ai.dataset import Sample
from inspect_ai.solver import generate, solver

# Solver from docs
@solver
def ctf_prompt():
    async def solve(state, generate):
        # TODO: dynamic prompt engineering
        return state

# Minimal task
@task
def ctf():
    task = Task(
        setup=ctf_prompt,
        dataset=[Sample(input="Submit the flag 'hello'.")],
        solver=generate(),
        sandbox="docker",
        max_messages=5,
    )
    return task
```

Running `inspect eval <scriptname.py>` produces the following traceback:

```
  + Exception Group Traceback (most recent call last):
  |   File ".venv/bin/inspect", line 8, in <module>
  |     sys.exit(main())
  |              ^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_cli/main.py", line 56, in main
  |     inspect(auto_envvar_prefix="INSPECT")  # pylint: disable=no-value-for-parameter
  |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File ".venv/lib/python3.12/site-packages/click/core.py", line 1161, in __call__
  |     return self.main(*args, **kwargs)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "venv/lib/python3.12/site-packages/click/core.py", line 1082, in main
  |     rv = self.invoke(ctx)
  |          ^^^^^^^^^^^^^^^^
  |   File "venv/lib/python3.12/site-packages/click/core.py", line 1697, in invoke
  |     return _process_result(sub_ctx.command.invoke(sub_ctx))
  |                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File ".venv/lib/python3.12/site-packages/click/core.py", line 1443, in invoke
  |     return ctx.invoke(self.callback, **ctx.params)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "venv/lib/python3.12/site-packages/click/core.py", line 788, in invoke
  |     return __callback(*args, **kwargs)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_cli/common.py", line 43, in wrapper
  |     return cast(click.Context, func(*args, **kwargs))
  |                                ^^^^^^^^^^^^^^^^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_cli/common.py", line 100, in wrapper
  |     return cast(click.Context, func(*args, **kwargs))
  |                                ^^^^^^^^^^^^^^^^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_cli/eval.py", line 462, in wrapper
  |     return cast(click.Context, func(*args, **kwargs))
  |                                ^^^^^^^^^^^^^^^^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_cli/eval.py", line 545, in eval_command
  |     eval_exec(
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_cli/eval.py", line 923, in eval_exec
  |     eval(**params)
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/eval.py", line 241, in eval
  |     return task_display().run_task_app(run_task_app)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_display/textual/display.py", line 51, in run_task_app
  |     raise result.value
  |   File "venv/lib/python3.12/site-packages/textual/worker.py", line 368, in _run
  |     self._result = await self.run()
  |                    ^^^^^^^^^^^^^^^^
  |   File ".venv/lib/python3.12/site-packages/textual/worker.py", line 352, in run
  |     return await (
  |            ^^^^^^^
  |   File ".venv/lib/python3.12/site-packages/textual/worker.py", line 339, in _run_async
  |     return await self._work
  |            ^^^^^^^^^^^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/eval.py", line 196, in run_task_app
  |     return await eval_async(
  |            ^^^^^^^^^^^^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/eval.py", line 490, in eval_async
  |     await eval_run(
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/run.py", line 256, in eval_run
  |     return await run_single(task_run_options, debug_errors)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/run.py", line 278, in run_single
  |     async with anyio.create_task_group() as tg:
  |                ^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "venv/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 772, in __aexit__
  |     raise BaseExceptionGroup(
  | ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/task/run.py", line 242, in task_run
    |     await log_start(logger, plan, generate_config)
    |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/task/log.py", line 260, in log_start
    |     steps=[eval_plan_step(solver) for solver in plan.steps],
    |            ^^^^^^^^^^^^^^^^^^^^^^
    |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/task/log.py", line 255, in eval_plan_step
    |     solver=registry_log_name(solver), params=registry_params(solver)
    |                                              ^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/home/dev/git/inspect_ai/src/inspect_ai/_util/registry.py", line 264, in registry_params
    |     raise ValueError("Object does not have registry info")
    | ValueError: Object does not have registry info
    | 
    | During handling of the above exception, another exception occurred:
    | 
    | Traceback (most recent call last):
    |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/run.py", line 281, in run_task
    |     result = await task_run(tasks[index])
    |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/task/run.py", line 407, in task_run
    |     eval_log = await logger.log_finish(
    |                ^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/home/dev/git/inspect_ai/src/inspect_ai/_eval/task/log.py", line 237, in log_finish
    |     log = await self.recorder.log_finish(
    |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/home/dev/git/inspect_ai/src/inspect_ai/log/_recorders/eval.py", line 167, in log_finish
    |     raise RuntimeError("Log not properly initialised")
    | RuntimeError: Log not properly initialised
```

I found a way to make this work, but it doesn't match what's in the docs. This is actually how I had set up solvers in the past so I think the docs for task setup might just be wrong?

There are two changes to the above script:
1) Update the `ctf_prompt` method to return the inner method
2) Actually call the `ctf_prompt` method when creating the setup method, i.e., `setup=ctf_prompt()`,

```py
from inspect_ai import Task, task
from inspect_ai.dataset import Sample
from inspect_ai.solver import generate, solver

# prompt solver which should always be run
@solver
def ctf_prompt():
    async def solve(state, generate):
        # TODO: dynamic prompt engineering
        return state
    return solve
@task
def ctf():
    task = Task(
        setup=ctf_prompt(),
        dataset=[Sample(input="Submit the flag 'hello'.")],
        solver=generate(),
        sandbox="docker",
        max_messages=5,
    )
    return task
```

Good catch! The `return solve` does indeed make this work. Docs updated here: https://github.com/UKGovernmentBEIS/inspect_ai/commit/865f17926281cc785a7c26b486938cc8533fb2e8 (note it looks like we were already calling `ctf_prompt()` in the task constructor).

I encountered this error while running using chain. But if I remove chain and only return ```solve``` in ```ctf_prompt``` then the code works fine.

```python
from inspect_ai import Task, eval
from inspect_ai.dataset import Sample
from inspect_ai.solver import generate, solver, chain

# Prompt solver which should always be run
@solver
def ctf_prompt():
    async def solve(state, generate):
        # TODO: dynamic prompt engineering
        print("[DEBUG - ctf_prompt] Running setup solver")
        print(f"[DEBUG - ctf_prompt] Current state input: {state.input_text}")
        return state
    return chain(
        [solve, solve]
    )


def main():
    """Run the CTF task"""
    
    print("\n" + "="*60)
    print("TESTING CTF TASK")
    print("="*60)
    
    # Create the task inline (not using @task decorator for direct execution)
    task = Task(
        setup=ctf_prompt(),  # Runs before the main solver
        dataset=[Sample(input="Submit the flag 'hello'.")],
        solver=generate(),  # Just generate a response
        # Note: sandbox="docker" requires Docker setup, commented out for now
        # sandbox="docker",
        max_messages=5,
    )
    
    # Run evaluation
    result = eval(
        task,
        model="google/gemini-2.0-flash-lite",
    )
    
    # Display results
    if result and len(result) > 0:
        sample_result = result[0].samples[0]
        
        print("\n" + "="*60)
        print("FINAL RESULTS")
        print("="*60)
        print(f"\nInput:\n{sample_result.input}")
        print(f"\nModel Output:\n{sample_result.output.completion}")
        print("\n" + "="*60)
        
        # Show message history
        print("\n[DEBUG] Message history:")
        for i, msg in enumerate(sample_result.messages):
            print(f"\n  Message {i} ({type(msg).__name__}):")
            if hasattr(msg, 'content'):
                content_preview = str(msg.content)[:100]
                print(f"    {content_preview}...")


if __name__ == "__main__":
    main()
```

@phuvinhnguyen thanks for reporting this! The error message is pretty bad here, we've fixed that on `main` but the problem you reported remains (you can't return a `chain(...)` from an `@solver` decorated function). I am going to write that up as a separate bug and hopefully resolve that very soon.

New issue here: https://github.com/UKGovernmentBEIS/inspect_ai/issues/2554