Repository: oumi-ai/oumi
File: src/oumi/core/evaluation/backends/alpaca_eval.py (Python)

Code before:
    alpaca_dataset = AlpacaEvalDataset(
        dataset_name="tatsu-lab/alpaca_eval"
    ).conversations()

    if task_params.num_samples:
        alpaca_dataset = alpaca_dataset[: task_params.num_samples]

    # Run inference for the alpaca_dataset.
    logger.info(
        "\tAlpacaEval inference `model_params`:\n"
        f"{pformat(inference_engine._model_params)}\n"
        "\tAlpacaEval inference `generation_params`:\n"
        f"{pformat(inference_engine._generation_params)}"
    )
    responses = inference_engine.infer(input=alpaca_dataset)

    # Convert the model responses from Oumi format to Alpaca format.
    generator_display_name = config.run_name or start_time_str  # No run name? use time.
    responses_json = utils.conversations_to_alpaca_format(responses)
    responses_df = pd.DataFrame(responses_json)
    responses_df["generator"] = generator_display_name

    # Run AlpacaEval evaluation, i.e. annotate the model's responses.
    logger.info("Running AlpacaEval annotation.")
    logger.info(f"\tAlpacaEval `task_params`:\n{pformat(task_params)}")
    # Editted for type safety
    result = alpaca_eval.evaluate(
        model_outputs=responses_df,
        annotators_config=annotators_config,
        fn_metric=fn_metric,
        is_return_instead_of_print=True,
        is_overwrite_leaderboard=True,
        max_instances=task_params.num_samples,
        sort_by=sort_by_metric,
        **task_params.eval_kwargs,
    )
    if isinstance(result, tuple):
        df_leaderboard = result[0]
    else:
        df_leaderboard = result

    # Metrics are only available on the main process, and `None` on others.
    if not is_world_process_zero():
        return EvaluationResult()

    metric_dict = {}
    if df_leaderboard is not None:
        if generator_display_name in df_leaderboard.index:
            metrics = df_leaderboard.loc[generator_display_name]
            metric_dict: dict[str, Any] = {
                str(metric): value for metric, value in metrics.items()

Review comment (suggestion):
drop the comment "Editted for type safety"
This wont make sense for a user reading this for the first time (they wont know the history; ie that this used to be implemented differently etc)

Code after:
    alpaca_dataset = AlpacaEvalDataset(
        dataset_name="tatsu-lab/alpaca_eval"
    ).conversations()

    if task_params.num_samples:
        alpaca_dataset = alpaca_dataset[: task_params.num_samples]

    # Run inference for the alpaca_dataset.
    logger.info(
        "\tAlpacaEval inference `model_params`:\n"
        f"{pformat(inference_engine._model_params)}\n"
        "\tAlpacaEval inference `generation_params`:\n"
        f"{pformat(inference_engine._generation_params)}"
    )
    responses = inference_engine.infer(input=alpaca_dataset)

    # Convert the model responses from Oumi format to Alpaca format.
    generator_display_name = config.run_name or start_time_str  # No run name? use time.
    responses_json = utils.conversations_to_alpaca_format(responses)
    responses_df = pd.DataFrame(responses_json)
    responses_df["generator"] = generator_display_name

    # Run AlpacaEval evaluation, i.e. annotate the model's responses.
    logger.info("Running AlpacaEval annotation.")
    logger.info(f"\tAlpacaEval `task_params`:\n{pformat(task_params)}")
    result = alpaca_eval.evaluate(
        model_outputs=responses_df,
        annotators_config=annotators_config,
        fn_metric=fn_metric,
        is_return_instead_of_print=True,
        is_overwrite_leaderboard=True,
        max_instances=task_params.num_samples,
        sort_by=sort_by_metric,
        **task_params.eval_kwargs,
    )
    if isinstance(result, tuple):
        df_leaderboard = result[0]
    else:
        df_leaderboard = result

    # Metrics are only available on the main process, and `None` on others.
    if not is_world_process_zero():
        return EvaluationResult()

    metric_dict = {}
    if df_leaderboard is not None:
        if generator_display_name in df_leaderboard.index:
            metrics = df_leaderboard.loc[generator_display_name]
            metric_dict: dict[str, Any] = {
                str(metric): value for metric, value in metrics.items()
            }