Repository: oumi-ai/oumi
File: src/oumi/core/evaluation/backends/lm_harness.py (Python)

Code before:
def _add_remote_engine_args(
    model_args: dict[str, Any],
    inference_remote_params: RemoteParams | None,
) -> None:
    """Add REMOTE engine-specific arguments."""
    if not inference_remote_params:
        raise ValueError(
            "The `REMOTE` inference engine requires `inference_remote_params`."
        )

    model_args["base_url"] = inference_remote_params.api_url
    if inference_remote_params.num_workers > 0:
        model_args["num_concurrent"] = inference_remote_params.num_workers
    if inference_remote_params.max_retries > 0:
        model_args["max_retries"] = inference_remote_params.max_retries
    if inference_remote_params.connection_timeout > 0:
        model_args["timeout"] = int(inference_remote_params.connection_timeout)


def _add_multimodal_args(
    model_args: dict[str, Any],
    lm_harness_model: str,
    model_params: ModelParams,
) -> None:
    """Add multimodal (vision) model arguments."""
    # FIXME OPE-355 To remove `max_images=1` limit
    model_args["max_images"] = 1
    model_args["interleave"] = True

    # HuggingFace multimodal needs additional processor setup
    if lm_harness_model == "hf-multimodal":
        model_args["convert_img_format"] = True

        tokenizer = build_tokenizer(model_params)
        processor = build_processor(
            model_params.model_name,
            tokenizer,
            trust_remote_code=model_params.trust_remote_code,
            processor_kwargs=model_params.processor_kwargs,
        )
        if image_token := processor.image_token:
            model_args["image_string"] = image_token
        if image_token_id := processor.image_token_id:
            model_args["image_token_id"] = image_token_id

    if model_args.get("batch_size") == "auto":
        logger.info("Setting batch_size to 1 for multimodal evaluation.")
        model_args["batch_size"] = 1


Review comment (suggestion):
```suggestion
    # TODO: OPE-355 - To remove `max_images=1` limit
```

Code after:
def _add_remote_engine_args(
    model_args: dict[str, Any],
    inference_remote_params: RemoteParams | None,
) -> None:
    """Add REMOTE engine-specific arguments."""
    if not inference_remote_params:
        raise ValueError(
            "The `REMOTE` inference engine requires `inference_remote_params`."
        )

    model_args["base_url"] = inference_remote_params.api_url
    if inference_remote_params.num_workers > 0:
        model_args["num_concurrent"] = inference_remote_params.num_workers
    if inference_remote_params.max_retries > 0:
        model_args["max_retries"] = inference_remote_params.max_retries
    if inference_remote_params.connection_timeout > 0:
        model_args["timeout"] = int(inference_remote_params.connection_timeout)


def _add_multimodal_args(
    model_args: dict[str, Any],
    lm_harness_model: str,
    model_params: ModelParams,
) -> None:
    """Add multimodal (vision) model arguments."""
    # TODO: OPE-355 - To remove `max_images=1` limit
    model_args["max_images"] = 1
    model_args["interleave"] = True

    # HuggingFace multimodal needs additional processor setup
    if lm_harness_model == "hf-multimodal":
        model_args["convert_img_format"] = True

        tokenizer = build_tokenizer(model_params)
        processor = build_processor(
            model_params.model_name,
            tokenizer,
            trust_remote_code=model_params.trust_remote_code,
            processor_kwargs=model_params.processor_kwargs,
        )
        if image_token := processor.image_token:
            model_args["image_string"] = image_token
        if image_token_id := processor.image_token_id:
            model_args["image_token_id"] = image_token_id

    if model_args.get("batch_size") == "auto":
        logger.info("Setting batch_size to 1 for multimodal evaluation.")
        model_args["batch_size"] = 1