Repository: UKGovernmentBEIS/inspect_ai
Issue: Multi-Turn LLM to LLM conversations toy example evaluations
Root cause: Documentation Deficiency

I want to evaluate a conversation between two LLMs.  
An example will be that, starting from a user prompt "tell me a joke", LLM1 answers, and then LLM2 answers. The conversation between the two LLMs keeps up for N turns. Finally a third LLM evaluates the conversation

I've started from the example of https://github.com/UKGovernmentBEIS/inspect_ai/issues/25, which evaluates an LLM answering a sequenced set of fixed prompts. Below a simplified version.

Are there examples in the documentation that allow to create an LLM-LLM eval? (I've not been able to find it)
If not, is it possible to adapt the following example?

```python
import re

from inspect_ai.model import (
    ChatMessageUser,
    ChatMessageSystem,
    Model,
    get_model,
    ChatMessageAssistant,
)
from inspect_ai.dataset import Sample, json_dataset, MemoryDataset
from inspect_ai.solver import Generate, Solver, TaskState, solver
from inspect_ai.scorer import (
    Scorer,
    Score,
    scorer,
    Target,
    mean,
    bootstrap_std,
)
from inspect_ai import Task, eval, task


# Vanilla dataset
from inspect_ai.dataset import MemoryDataset, Sample
from inspect_ai.model import ChatMessageUser

dataset = MemoryDataset(
    samples=[
        Sample(
            input=[
                ChatMessageUser(content=("Tell me a short joke")),
                ChatMessageUser(content=("Traducela a espanol")),
                ChatMessageUser(content="Back to english"),
            ],
        )
    ]
)


# %%
@solver
def multi_dialogue_solver() -> Solver:
    """Solver for the multi-dialogue task.

    This solver simulates a dialogue between a user and the LLM.

    It assumes that `state.messages` is a list of ChatMessageUser objects.
    It will write to `state.messages` a list of ChatMessageUser and ChatMessageSystem
    objects corresponding to the dialogue between the user and the LLM.
    """

    async def solve(state: TaskState, generate: Generate) -> TaskState:
        # copy the messages from the state
        input = state.messages.copy()

        # for this solver, we expect input to be a list of ChatMessageUser
        if not isinstance(input, list):
            raise TypeError(
                f"Inputs in samples of the dataset should be list of ChatMessageUser. Found {input}"
            )
        if not all(isinstance(turn, ChatMessageUser) for turn in input):
            raise TypeError(
                f"Inputs in samples of the dataset should be list of ChatMessageUser. Found {input}"
            )

        # generate the output after each turn in the input and write to state.messages
        state.messages = []
        for turn in input:
            state.messages.append(turn)
            state = await generate(state)

        return state

    return solve


# %%
# prompt based on 'single-v1-multi-turn' prompt from https://github.com/lm-sys/FastChat/blob/main/fastchat/llm_judge/data/judge_prompts.jsonl
# Changed system prompt so it judges all answers to user questions, not just answer to the second question.
# Changed prompt template to remove reference to assistant A and to allow for conversations of arbitrary length.

SYSTEM_PROMPT = "Please act as an impartial judge and evaluate the quality of the translations provided by an AI assistant to the user question displayed below. You must rate the response on a scale of 1 to 10 by strictly following this format: Rating: [[rating]].\n\n"
PROMPT_TEMPLATE_START = "<|The Start of the Assistant's Conversation with User|>\n\n"
PROMPT_TEMPLATE_USER = "### User:\n{message}\n\n"
PROMPT_TEMPLATE_ASSISTANT = "### Assistant:\n{message}\n\n"
PROMPT_TEMPLATE_END = "<|The End of the Assistant's Conversation with User|>"

# scorer adapted from example of model_graded_qa from https://ukgovernmentbeis.github.io/inspect_ai/scorers.html


@scorer(metrics=[mean(), bootstrap_std()])
def model_graded_dialogue(
    system_prompt: str = SYSTEM_PROMPT,
    prompt_template_start: str = PROMPT_TEMPLATE_START,
    prompt_template_user: str = PROMPT_TEMPLATE_USER,
    prompt_template_assistant: str = PROMPT_TEMPLATE_ASSISTANT,
    prompt_template_end: str = PROMPT_TEMPLATE_END,
    model: str | Model | None = None,
) -> Scorer:
    # resolve model
    grader_model = get_model(model)

    async def score(state: TaskState, target: Target) -> Score:
        # create system message
        system_message = ChatMessageSystem(content=system_prompt)

        # create user message, by looping through TaskState.messages
        user_message = prompt_template_start
        for message in state.messages:
            if isinstance(message, ChatMessageUser):
                user_message += prompt_template_user.format(message=message.content)
            elif isinstance(message, ChatMessageAssistant):
                user_message += prompt_template_assistant.format(
                    message=message.content
                )
            else:
                continue
        user_message += prompt_template_end

        user_message = ChatMessageUser(content=user_message)

        # query the model for the score
        result = await grader_model.generate([system_message, user_message])

        # extract the grade
        grade_pattern = r"Rating\s*:?\s*\[\[(\d+)\]\]"
        match = re.search(grade_pattern, result.completion, flags=re.IGNORECASE)
        if match:
            return Score(
                value=int(match.group(1)),
                explanation=result.completion,
            )
        else:
            return Score(
                value=0,
                explanation="Grade not found in model output: "
                + f"{result.completion}",
            )

    return score


# %%
@task
def multi_dialogue_task():
    return Task(
        dataset=dataset,
        plan=[
            multi_dialogue_solver(),
        ],
        scorer=model_graded_dialogue(model="bedrock/openai.gpt-oss-20b-1:0"),
    )


# %%
## Execute
MODEL = "bedrock/openai.gpt-oss-20b-1:0"
LIMIT = 1
EPOCHS = 1
MAX_RETRIES = 5
MAX_TOKENS = 1000
LOG_DIR = ""
# %%
eval(
    multi_dialogue_task,
    model=MODEL,
    max_tokens=MAX_TOKENS,
    limit=LIMIT,
    epochs=EPOCHS,
    max_retries=MAX_RETRIES,
    log_dir=LOG_DIR,
)
```

Perhaps @Lovkush has an answer


So....
Basically I asked an LLM and here is the result. At a first glance looks okey
```python

from inspect_ai.dataset import MemoryDataset, Sample
from inspect_ai.model import ChatMessageUser

dataset = MemoryDataset(
    samples=[
        Sample(
            input=[ChatMessageUser(content="Tell me a joke. If there is a joke in this conversation, then tell the joke in another language.")],
        )
    ]
)
from inspect_ai.model import (
    get_model,
    ChatMessageUser,
    ChatMessageAssistant,
)
from inspect_ai.solver import Solver, TaskState, solver, Generate

@solver
def llm_llm_conversation_solver(
    model1_name: str = "bedrock/openai.gpt-oss-20b-1:0",
    model2_name: str = "bedrock/openai.gpt-oss-20b-1:0",
    num_rounds: int = 3,  # number of "back-and-forth" rounds (each round = 2 assistant turns)
) -> Solver:
    """Have LLM1 and LLM2 converse with each other for `num_rounds` rounds.

    Structure:
      User: initial prompt (from dataset)
      Assistant 1 (LLM1): turn 1
      Assistant 2 (LLM2): turn 1
      Assistant 1 (LLM1): turn 2
      Assistant 2 (LLM2): turn 2
      ...
    """

    model1 = get_model(model1_name)
    model2 = get_model(model2_name)

    async def solve(state: TaskState, generate: Generate) -> TaskState:
        # Start from the sample's messages (initial user prompt)
        messages = list(state.messages)

        if not messages or not isinstance(messages[0], ChatMessageUser):
            raise TypeError("First message in the sample must be a ChatMessageUser with the initial prompt.")

        # Alternate turns: LLM1, LLM2, LLM1, LLM2, ...
        for round_idx in range(num_rounds):
            # --- LLM1's turn ---
            result1 = await model1.generate(messages)
            messages.append(ChatMessageAssistant(content=result1.completion))

            # --- LLM2's turn (sees everything so far, including LLM1's last reply) ---
            result2 = await model2.generate(messages)
            messages.append(ChatMessageAssistant(content=result2.completion))

        # Store the whole conversation for the scorer
        state.messages = messages
        return state

    return solve

import re
from inspect_ai.model import ChatMessageSystem, ChatMessageUser
from inspect_ai.model import Model, get_model
from inspect_ai.scorer import scorer, Scorer, Score, mean, bootstrap_std
from inspect_ai.solver import TaskState

SYSTEM_PROMPT = (
    "You are an impartial judge evaluating a conversation between two AI assistants.\n"
    "You will see the full conversation starting from the user's initial prompt.\n"
    "Evaluate if the jokes change language one afer the other and rate it from 1 to 10.\n"
    "Respond strictly in this format:\n"
    "Rating: [[x]]\n"
    "where x is an integer between 1 and 10.\n"
)

PROMPT_HEADER = "<|Start of Conversation|>\n\n"
PROMPT_USER = "User:\n{message}\n\n"
PROMPT_ASSISTANT_1 = "Assistant 1:\n{message}\n\n"
PROMPT_ASSISTANT_2 = "Assistant 2:\n{message}\n\n"
PROMPT_FOOTER = "<|End of Conversation|>"

@scorer(metrics=[mean(), bootstrap_std()])
def conversation_judge(
    judge_model_name: str | Model = "bedrock/openai.gpt-oss-20b-1:0",
) -> Scorer:
    """Use a judge model (LLM3) to rate the LLM1–LLM2 conversation."""

    judge_model = get_model(judge_model_name)

    async def score(state: TaskState, target) -> Score:
        messages = state.messages

        if not messages or not isinstance(messages[0], ChatMessageUser):
            return Score(
                value=0,
                explanation="Invalid conversation: first message is not a user prompt.",
            )

        transcript = PROMPT_HEADER
        transcript += PROMPT_USER.format(message=messages[0].content)

        assistant_turn_idx = 0
        for msg in messages[1:]:
            if isinstance(msg, ChatMessageAssistant):
                assistant_turn_idx += 1
                is_assistant1 = (assistant_turn_idx % 2 == 1)
                if is_assistant1:
                    transcript += PROMPT_ASSISTANT_1.format(message=msg.content)
                else:
                    transcript += PROMPT_ASSISTANT_2.format(message=msg.content)
            else:
                # skip any unexpected non-assistant messages
                continue

        transcript += PROMPT_FOOTER

        system_msg = ChatMessageSystem(content=SYSTEM_PROMPT)
        user_msg = ChatMessageUser(content=transcript)

        result = await judge_model.generate([system_msg, user_msg])
        text = result.completion

        pattern = r"Rating\s*:?\s*\[\[(\d+)\]\]"
        match = re.search(pattern, text, flags=re.IGNORECASE)

        if match:
            value = int(match.group(1))
            return Score(
                value=value,
                explanation=f"Judge output:\n{text}\n\nParsed rating: {value}",
            )
        else:
            return Score(
                value=0,
                explanation="Could not parse rating from judge output:\n" + text,
            )

    return score


from inspect_ai import Task, eval, task

@task
def llm_llm_joke_task():
    return Task(
        dataset=dataset,
        plan=[
            llm_llm_conversation_solver(
                model1_name="bedrock/openai.gpt-oss-20b-1:0",  # LLM1
                model2_name="bedrock/openai.gpt-oss-20b-1:0",  # LLM2 (change as needed)
                num_rounds=3,  # 3 rounds = 6 assistant turns
            ),
        ],
        scorer=conversation_judge(
            judge_model_name="bedrock/openai.gpt-oss-20b-1:0",  # LLM3
        ),
    )

DUMMY_MODEL = "bedrock/openai.gpt-oss-20b-1:0"  # not actually used in solver

eval(
    llm_llm_joke_task,
    model=DUMMY_MODEL,
    max_tokens=1000,
    limit=1,
    epochs=1,
    max_retries=5,
    log_dir="",
)
```