Repository: oumi-ai/oumi
File: src/oumi/evaluation/registry/count_letters_task.py (Python)

Code before:
    # dataset = build_dataset("oumi-ai/oumi-letter-count", tokenizer=None, sample_count=10)  # noqa: E501
    # dataset = build_dataset("oumi-ai/berrybench-v0.1.0", tokenizer=None, sample_count=10)  # noqa: E501
    num_samples = task_params.num_samples
    if num_samples is None:
        num_samples = len(dataset)
    input_conversations = [dataset.conversation(i) for i in range(num_samples)]
    conversations = inference_engine.infer(input_conversations)
    logger.info(f"Finished inference on {len(conversations)} conversations!")
    if len(conversations) > 0:
        logger.info(f"Sample conversation: {conversations[0]}")

    count = 0
    total = 0
    for i, conversation in enumerate(conversations):
        total += 1
        response = conversation.last_message()
        if not response or not isinstance(response.content, str):
            continue
        prediction = _extract_prediction(response.content)
        if (
            prediction is not None
            and prediction == conversation.metadata["letter_count_integer"]
        ):
            count += 1

    return {"accuracy": count / total}


Review comment (bug):
Very debatable but just FYI: 
if your letter counter fails to produce a prediction (prediction is None), then you count this as a model failure, while it could be a prompt failure (your prompt may work better for some models vs others).
An alternative (not sure which solution is more fair, maybe check with Jeremy too): If prediction is None ignore it (`count` does not increase but also `total`--)

Code after:
    # dataset = build_dataset("oumi-ai/oumi-letter-count", tokenizer=None, sample_count=10)  # noqa: E501
    # dataset = build_dataset("oumi-ai/berrybench-v0.1.0", tokenizer=None, sample_count=10)  # noqa: E501
    num_samples = task_params.num_samples
    if num_samples is None:
        num_samples = len(dataset)
    input_conversations = [dataset.conversation(i) for i in range(num_samples)]
    conversations = inference_engine.infer(input_conversations)
    logger.info(f"Finished inference on {len(conversations)} conversations!")
    if len(conversations) > 0:
        logger.info(f"Sample conversation: {conversations[0]}")

    count = 0  # The number of examples with correct answers extracted.
    total = 0  # All examples.
    valid_count = 0  # The number of examples with valid answers extracted.
    for i, conversation in enumerate(conversations):
        total += 1
        # Grab the model's response
        response = conversation.last_message()
        # Ignore cases where model didn't respond or it's a multimodal response.
        # For now, we focus on text-only responses.
        if not response or not isinstance(response.content, str):
            continue
        # Count the example as correct if the extracted prediction is correct.
        prediction = _extract_prediction(response.content)
        if prediction is None:
            continue
        valid_count += 1
        if prediction == conversation.metadata["letter_count_integer"]:
            count += 1

    return {
        # Accuracy across all examples.
        "accuracy": count / total,
        # Accuracy when only counting examples with properly extracted answers.
        "properly_extracted_accuracy": count / valid_count,
        "num_samples": num_samples,
        # These three values sum up to num_samples.
        "num_correct_answers": count,
        "num_incorrect_answers": valid_count - count,
        "num_invalid_answers": total - valid_count,
    }