Repository: oumi-ai/oumi
File: src/oumi/core/evaluation/backends/lm_harness.py (Python)

Code before:
            "the `NATIVE`, `VLLM` and `REMOTE` inference_engine types."
        )

    # Instantiate an LM Harness task dictionary.
    task_dict = _get_task_dict(task_params)
    logger.info(f"\tLM Harness `task_params`:\n{pformat(task_params)}")
    logger.info(f"\tLM Harness `task_dict`:\n{pformat(task_dict)}")

    # Instantiate an LM Harness language model (lm).
    lm_harness_model_params = _generate_lm_harness_model_args(
        lm_harness_model=lm_harness_model,
        is_multimodal=is_multimodal,
        device=device,
        model_params=config.model,
        generation_params=config.generation,
        inference_engine_type=config.inference_engine,
        inference_remote_params=config.inference_remote_params,
    )
    logger.info(f"\tLM Harness `model_params`:\n{pformat(lm_harness_model_params)}")
    lm_class = lm_harness_get_model_class(lm_harness_model)
    lm = lm_class(**lm_harness_model_params)

    logger.info("Starting evaluation...")
    log_samples = False
    if _LOG_SAMPLES_KEY in task_params.eval_kwargs:
        log_samples = task_params.eval_kwargs.pop(_LOG_SAMPLES_KEY)
    lm_eval_output = lm_harness_evaluate(
        lm=lm,
        task_dict=task_dict,
        limit=task_params.num_samples,
        log_samples=log_samples,  # Set to `True` to log all responses or logits.
        apply_chat_template=is_multimodal,
        **task_params.eval_kwargs,  # type: ignore
    )

    # Metrics are only available on the main process, and `None` on others.
    if not is_world_process_zero():
        return EvaluationResult()

    assert lm_eval_output is not None
    task_name = task_params.task_name
    metric_dict = lm_eval_output["results"][task_name]  # type: ignore
    logger.info(f"{task_name}'s metric dict is {pformat(metric_dict)}")

    if config.enable_wandb:
        project_name = os.environ.get("WANDB_PROJECT", "oumi")
        logger.info(f"Logging to Weights and Biases project: '{project_name}'")
        wandb_logger = WandbLogger(
            project=project_name, name=config.run_name, job_type="eval"
        )
        wandb_logger.post_init(lm_eval_output)

Review comment (suggestion):
Avoid poping, because we store the full input config and all input params for reproducibility.
Maybe instead just do `log_samples = task_params.eval_kwargs[_LOG_SAMPLES_KEY]`

Code after:
            "the `NATIVE`, `VLLM` and `REMOTE` inference_engine types."
        )

    # Instantiate an LM Harness task dictionary.
    task_dict = _get_task_dict(task_params)
    logger.info(f"\tLM Harness `task_params`:\n{pformat(task_params)}")
    logger.info(f"\tLM Harness `task_dict`:\n{pformat(task_dict)}")

    # Instantiate an LM Harness language model (lm).
    lm_harness_model_params = _generate_lm_harness_model_args(
        lm_harness_model=lm_harness_model,
        is_multimodal=is_multimodal,
        device=device,
        model_params=config.model,
        generation_params=config.generation,
        inference_engine_type=config.inference_engine,
        inference_remote_params=config.inference_remote_params,
    )
    logger.info(f"\tLM Harness `model_params`:\n{pformat(lm_harness_model_params)}")
    lm_class = lm_harness_get_model_class(lm_harness_model)
    lm = lm_class(**lm_harness_model_params)

    logger.info("Starting evaluation...")
    if _LOG_SAMPLES_KEY in task_params.eval_kwargs:
        # Avoid popping kwargs due to the need to write the kwargs to log files.
        lm_eval_output = lm_harness_evaluate(
            lm=lm,
            task_dict=task_dict,
            limit=task_params.num_samples,
            apply_chat_template=is_multimodal,
            **task_params.eval_kwargs,  # type: ignore
        )
    else:
        lm_eval_output = lm_harness_evaluate(
            lm=lm,
            task_dict=task_dict,
            log_samples=False,
            limit=task_params.num_samples,
            apply_chat_template=is_multimodal,
            **task_params.eval_kwargs,  # type: ignore
        )

    # Metrics are only available on the main process, and `None` on others.
    if not is_world_process_zero():
        return EvaluationResult()

    assert lm_eval_output is not None
    task_name = task_params.task_name
    metric_dict = lm_eval_output["results"][task_name]  # type: ignore
    logger.info(f"{task_name}'s metric dict is {pformat(metric_dict)}")