Repository: oumi-ai/oumi
File: tests/unit/cli/test_cli_evaluate.py (Python)

Code before:
#
@pytest.fixture
def app():
    fake_app = typer.Typer()
    fake_app.command(context_settings=CONTEXT_ALLOW_EXTRA_ARGS)(evaluate)
    yield fake_app


@pytest.fixture
def mock_evaluate():
    with patch.object(oumi, "evaluate", autospec=True) as m_evaluate:
        yield m_evaluate


def test_evaluate_runs(app, mock_evaluate):
    with tempfile.TemporaryDirectory() as output_temp_dir:
        yaml_path = str(Path(output_temp_dir) / "eval.yaml")
        config: EvaluationConfig = _create_eval_config()
        config.to_yaml(yaml_path)
        _ = runner.invoke(app, ["--config", yaml_path])
        mock_evaluate.assert_has_calls([call(config)])


def test_evaluate_unparsable_metrics(app, mock_evaluate):
    with tempfile.TemporaryDirectory() as output_temp_dir:
        mock_evaluate.return_value = [
            {
                "results": {
                    "mmlu_college_computer_science": {
                        "alias": "college_computer_science",
                        "acc,none": 0.24,
                        "acc_stderr,none": 0.042923469599092816,
                    }
                }
            },
            {"results": {"mmlu": {"parsable_metric": 1.0}}},
            {
                "results": {
                    "q": {
                        "name": "our name",
                        "second_metric": 0.5,
                        "parsable_metric": 0.77,
                    },
                    "unparsable": "real name",
                }
            },
            {"results": []},
        ]
        yaml_path = str(Path(output_temp_dir) / "eval.yaml")
        config: EvaluationConfig = _create_eval_config()
        config.to_yaml(yaml_path)

Review comment (suggestion):
Maybe add another one, to make sure that tasks with subgroups display ok:

```
  "results": {
    "leaderboard_musr": {
      " ": " ",
      "alias": "leaderboard_musr"
    },
    "leaderboard_musr_murder_mysteries": {
      "alias": " - leaderboard_musr_murder_mysteries",
      "acc_norm,none": 0.516,
      "acc_norm_stderr,none": 0.03166998503010742
    },
    "leaderboard_musr_object_placements": {
      "alias": " - leaderboard_musr_object_placements",
      "acc_norm,none": 0.265625,
      "acc_norm_stderr,none": 0.027658162598649488
    },
    "leaderboard_musr_team_allocation": {
      "alias": " - leaderboard_musr_team_allocation",
      "acc_norm,none": 0.36,
      "acc_norm_stderr,none": 0.03041876402517498
    }
  }
 ```


Code after:
#
@pytest.fixture
def app():
    fake_app = typer.Typer()
    fake_app.command(context_settings=CONTEXT_ALLOW_EXTRA_ARGS)(evaluate)
    yield fake_app


@pytest.fixture
def mock_evaluate():
    with patch.object(oumi, "evaluate", autospec=True) as m_evaluate:
        yield m_evaluate


def test_evaluate_runs(app, mock_evaluate):
    with tempfile.TemporaryDirectory() as output_temp_dir:
        yaml_path = str(Path(output_temp_dir) / "eval.yaml")
        config: EvaluationConfig = _create_eval_config()
        config.to_yaml(yaml_path)
        _ = runner.invoke(app, ["--config", yaml_path])
        mock_evaluate.assert_has_calls([call(config)])


def test_evaluate_unparsable_metrics(app, mock_evaluate):
    with tempfile.TemporaryDirectory() as output_temp_dir:
        mock_evaluate.return_value = [
            {
                "results": {
                    "mmlu_college_computer_science": {
                        "alias": "college_computer_science",
                        "acc,none": 0.24,
                        "acc_stderr,none": 0.042923469599092816,
                    }
                }
            },
            {
                "results": {
                    "leaderboard_bbh": {" ": " ", "alias": "leaderboard_bbh"},
                    "leaderboard_bbh_boolean_expressions": {
                        "alias": " - leaderboard_bbh_boolean_expressions",
                        "acc_norm,none": 0.436,
                        "acc_norm_stderr,none": 0.03142556706028128,
                    },
                    "leaderboard_bbh_causal_judgement": {
                        "alias": " - leaderboard_bbh_causal_judgement",
                        "acc_norm,none": 0.5187165775401069,
                        "acc_norm_stderr,none": 0.03663608375537842,
                    },
                    "leaderboard_bbh_date_understanding": {
                        "alias": " - leaderboard_bbh_date_understanding",
                        "acc_norm,none": 0.192,