Repository: instructlab/instructlab

Issue:
Print full scores along with delta for MMLU-Branch scores
Currently an MMLU-Run has the following output.

```
# KNOWLEDGE EVALUATION REPORT

## BASE MODEL
/home/ec2-user/.cache/instructlab/models/instructlab/granite-7b-lab

## MODEL
/home/ec2-user/models/4x-a10-np-4

### AVERAGE:
+0.08 (across 1)

### IMPROVEMENTS:
1. knowledge_pheonix (+0.08)

real    6m33.259s
user    2m45.545s
sys     0m44.396s
```

The user would benefit by does showing the full scores for each of the models, not just the deltas.
 Show individual qna scores for Mt-Bench-Branch
Right now the MT-Bench-Branch scores only show deltas not individual scores.



Discussion:
We should do this for all scores not just MMLU-Branch.
> We should do this for all scores not just MMLU-Branch.

https://github.com/instructlab/instructlab/issues/2125

That's the corresponding issue for mt_bench_branch.  Which I think covers all the current cases.





Patch:
diff --git a/requirements.txt b/requirements.txt
index 7ad5b5cd3f..286dea105d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ datasets>=2.18.0
 gguf>=0.6.0
 GitPython>=3.1.42
 httpx>=0.25.0
-instructlab-eval>=0.2.1
+instructlab-eval>=0.3.1
 instructlab-quantize>=0.1.0
 instructlab-schema>=0.4.0
 instructlab-sdg>=0.3.0
diff --git a/src/instructlab/model/evaluate.py b/src/instructlab/model/evaluate.py
index 92ff92cc3d..4d56d96f3f 100644
--- a/src/instructlab/model/evaluate.py
+++ b/src/instructlab/model/evaluate.py
@@ -139,24 +139,39 @@ def validate_model(model: str, model_arg: str = "--model"):
         raise click.exceptions.Exit(1)
 
 
-def sort_score(pairing: tuple[str, float]) -> float:
+def sort_score(pairing: tuple[str, float, float, float]) -> float:
     """helper func for display_branch_eval_summary
     takes a tuple pairing and returns just the score
     """
     return pairing[1]
 
 
-def display_models(model, base_model) -> None:
+def get_benchmark_max_score(benchmark: Benchmark) -> str:
+    # total score for Benchmark.MT_BENCH_BRANCH or Benchmark.MT_Bench
+    max_score = "10.0"
+    if benchmark in (Benchmark.MMLU_BRANCH, Benchmark.MMLU):
+        max_score = "1.0"
+    return max_score
+
+
+def display_models_and_scores(
+    benchmark, model, base_model, model_score, base_model_score
+) -> None:
     """prints the base_model and model with a header"""
-    print("## BASE MODEL")
-    print(base_model)
-    display_model(model)
+    max_score = get_benchmark_max_score(benchmark)
 
+    base_model_score = round(base_model_score, 2)
+    model_score = round(model_score, 2)
+    print("## BASE MODEL (SCORE)")
+    display_model(base_model, base_model_score, max_score)
+    print("\n## MODEL (SCORE)")
+    display_model(model, model_score, max_score)
 
-def display_model(model) -> None:
+
+def display_model(model, model_score, max_score) -> None:
     """prints the given model with a header"""
-    print("\n## MODEL")
-    print(model)
+    model_score = round(model_score, 2)
+    print(f"{model} ({model_score}/{max_score})")
 
 
 def display_error_rate(error_rate) -> None:
@@ -167,37 +182,49 @@ def display_error_rate(error_rate) -> None:
 
 
 def display_branch_eval_summary(
-    improvements: list[tuple[str, float]],
-    regressions: list[tuple[str, float]],
-    no_changes: list[str],
+    benchmark: Benchmark,
+    improvements: list[tuple[str, float, float, float]],
+    regressions: list[tuple[str, float, float, float]],
+    no_changes: list[tuple[str, float]],
     new=None,
 ):
     """takes in results lists from mt_bench_branch benchmark evaluation
     prints out diff between the branches to the user
     """
+    # total score for MT-BENCH-BRANCH
+    max_score = get_benchmark_max_score(benchmark)
+
     if len(improvements) > 0:
         improvements.sort(key=sort_score, reverse=True)
-        print("\n### IMPROVEMENTS:")
+        print(f"\n### IMPROVEMENTS (0.0 to {max_score}):")
         for index, improvement in enumerate(improvements):
-            task, delta = improvement
-            print(f"{index+1}. {task} (+{delta})")
+            task, delta, base_score, new_score = improvement
+            base_score = round(base_score, 2)
+            new_score = round(new_score, 2)
+            print(f"{index+1}. {task}: {base_score} -> {new_score} (+{delta})")
 
     if len(regressions) > 0:
         regressions.sort(key=sort_score)
-        print("\n### REGRESSIONS:")
+        print(f"\n### REGRESSIONS (0.0 to {max_score}):")
         for index, regression in enumerate(regressions):
-            task, delta = regression
-            print(f"{index+1}. {task} ({delta})")
+            task, delta, base_score, new_score = regression
+            base_score = round(base_score, 2)
+            new_score = round(new_score, 2)
+            print(f"{index+1}. {task}: {base_score} -> {new_score} ({delta})")
 
     if len(no_changes) > 0:
-        print("\n### NO CHANGE:")
-        for index, task in enumerate(no_changes):
-            print(f"{index+1}. {task}")
+        print(f"\n### NO CHANGE (0.0 to {max_score}):")
+        for index, entry in enumerate(no_changes):
+            task, avg_score = entry
+            avg_score = round(avg_score, 2)
+            print(f"{index+1}. {task} ({avg_score})")
 
     if new is not None and len(new) > 0:
-        print("\n### NEW:")
-        for index, qna in enumerate(new):
-            print(f"{index+1}. {qna}")
+        print(f"\n### NEW (0.0 to {max_score}):")
+        for index, entry in enumerate(new):
+            qna, avg_score = entry
+            avg_score = round(avg_score, 2)
+            print(f"{index+1}. {qna} ({avg_score})")
 
 
 def qa_pairs_to_qna_to_avg_scores(qa_pairs: list[dict]) -> dict[str, float]:
@@ -576,13 +603,13 @@ def evaluate(
                 if server is not None:
                     server.shutdown()
 
+            max_score = get_benchmark_max_score(Benchmark.MT_BENCH)
             print("# SKILL EVALUATION REPORT")
-            display_model(model)
-            print("\n### AVERAGE:")
-            print(f"{round(overall_score, 2)} (across {len(qa_pairs)})")
-            print("\n### TURN ONE:")
+            print("\n## MODEL (SCORE)")
+            display_model(model, overall_score, max_score)
+            print(f"\n### TURN ONE (0.0 to {max_score}):")
             print(round(turn_scores[0], 2))
-            print("\n### TURN TWO:")
+            print(f"\n### TURN TWO (0.0 to {max_score}):")
             turn2_score = turn_scores[1]
             if isinstance(turn2_score, float):
                 turn2_score = round(turn2_score, 2)
@@ -659,46 +686,64 @@ def evaluate(
                 for i, evaluator in enumerate(evaluators):
                     branch = branches[i]
                     print(f"Evaluating answers for branch {branch}...")
-                    judgement = evaluator.judge_answers(
+                    overall_score, qa_pairs, error_rate = evaluator.judge_answers(
                         api_base, max_workers=max_workers, serving_gpus=effective_gpus
                     )
-
-                    if len(judgement) == 3:
-                        qa_pairs = judgement[1]
-                        error_rate = judgement[2]
-                    else:
-                        qa_pairs = judgement[0]
-                        error_rate = judgement[1]
-
-                    qa_pairs_and_errors.append((qa_pairs, error_rate))
+                    qa_pairs_and_errors.append((overall_score, qa_pairs, error_rate))
             finally:
                 if server is not None:
                     server.shutdown()
 
-            qa_pairs, error_rate = qa_pairs_and_errors[0]
-            base_qa_pairs, base_error_rate = qa_pairs_and_errors[1]
+            overall_score, qa_pairs, error_rate = qa_pairs_and_errors[0]
+            base_overall_score, base_qa_pairs, base_error_rate = qa_pairs_and_errors[1]
 
             qna_to_avg_scores = qa_pairs_to_qna_to_avg_scores(qa_pairs)
             base_qna_to_avg_scores = qa_pairs_to_qna_to_avg_scores(base_qa_pairs)
 
             print("# SKILL EVALUATION REPORT\n")
-            display_models(model, base_model)
+            display_models_and_scores(
+                Benchmark.MT_BENCH_BRANCH,
+                model,
+                base_model,
+                overall_score,
+                base_overall_score,
+            )
 
             improvements, regressions, no_changes, new_qnas = [], [], [], []
             for qna, avg_score in qna_to_avg_scores.items():
                 base_avg_score = base_qna_to_avg_scores.get(qna)
                 if base_avg_score is not None:
                     if avg_score > base_avg_score:
-                        improvements.append((qna, round(avg_score - base_avg_score, 2)))
+                        improvements.append(
+                            (
+                                qna,
+                                round(avg_score - base_avg_score, 2),
+                                base_avg_score,
+                                avg_score,
+                            )
+                        )
                     elif avg_score == base_avg_score:
-                        no_changes.append(qna)
+                        no_changes.append((qna, avg_score))
                     else:
-                        regressions.append((qna, round(avg_score - base_avg_score, 2)))
+                        regressions.append(
+                            (
+                                qna,
+                                round(avg_score - base_avg_score, 2),
+                                base_avg_score,
+                                avg_score,
+                            )
+                        )
                 else:
-                    new_qnas.append((qna))
+                    new_qnas.append((qna, avg_score))
 
             # display summary of evaluation before exiting
-            display_branch_eval_summary(improvements, regressions, no_changes, new_qnas)
+            display_branch_eval_summary(
+                Benchmark.MT_BENCH_BRANCH,
+                improvements,
+                regressions,
+                no_changes,
+                new_qnas,
+            )
             display_error_rate((error_rate + base_error_rate) / 2)
 
         elif benchmark == Benchmark.MMLU:
@@ -732,12 +777,12 @@ def evaluate(
                 if server is not None:
                     server.shutdown()
 
+            max_score = get_benchmark_max_score(Benchmark.MMLU)
             print("# KNOWLEDGE EVALUATION REPORT")
-            display_model(model)
-            print("\n### AVERAGE:")
-            print(f"{round(overall_score, 2)} (across {len(individual_scores)})\n")
+            print("\n## MODEL (SCORE)")
+            display_model(model, overall_score, max_score)
 
-            print("### SCORES:")
+            print(f"\n### SCORES (0.0 to {max_score}):")
             for task, score in individual_scores.items():
                 s = round(score["score"], 2)
                 print(f"{task} - {s}")
@@ -791,16 +836,13 @@ def evaluate(
             base_individual_scores = individual_scores_list[1]
 
             print("# KNOWLEDGE EVALUATION REPORT\n")
-            display_models(model, base_model)
-
-            print("\n### AVERAGE:")
-            delta = round(overall_score - base_overall_score, 2)
-            if delta >= 0:
-                delta_display = f"+{delta}"
-            else:
-                delta_display = delta
-
-            print(f"{delta_display} (across {len(individual_scores)})")
+            display_models_and_scores(
+                Benchmark.MMLU_BRANCH,
+                model,
+                base_model,
+                overall_score,
+                base_overall_score,
+            )
 
             improvements, regressions, no_changes = [], [], []
             for task, score in individual_scores.items():
@@ -809,14 +851,16 @@ def evaluate(
                 b_s = base_score["score"]
                 d = round(s - b_s, 2)
                 if s > b_s:
-                    improvements.append((task, d))
+                    improvements.append((task, d, b_s, s))
                 elif b_s > s:
-                    regressions.append((task, d))
+                    regressions.append((task, d, b_s, s))
                 else:
-                    no_changes.append(task)
+                    no_changes.append((task, s))
 
             # display summary of evaluation before exiting
-            display_branch_eval_summary(improvements, regressions, no_changes)
+            display_branch_eval_summary(
+                Benchmark.MMLU_BRANCH, improvements, regressions, no_changes
+            )
     except EvalError as ee:
         print(ee.message)
         raise click.exceptions.Exit(1)


Test patch:
diff --git a/tests/test_lab_evaluate.py b/tests/test_lab_evaluate.py
index 0237ab5f60..822b635b69 100644
--- a/tests/test_lab_evaluate.py
+++ b/tests/test_lab_evaluate.py
@@ -111,16 +111,13 @@ def run_mt_bench(cli_runner, error_rate):
         Evaluating answers...
         # SKILL EVALUATION REPORT
 
-        ## MODEL
-        instructlab/granite-7b-lab
+        ## MODEL (SCORE)
+        instructlab/granite-7b-lab (1.5/10.0)
 
-        ### AVERAGE:
-        1.5 (across 2)
-
-        ### TURN ONE:
+        ### TURN ONE (0.0 to 10.0):
         1.0
 
-        ### TURN TWO:
+        ### TURN TWO (0.0 to 10.0):
         2
         """
     )
@@ -169,25 +166,25 @@ def run_mt_bench_branch(cli_runner, error_rate):
         Evaluating answers for branch main...
         # SKILL EVALUATION REPORT
 
-        ## BASE MODEL
-        instructlab/granite-7b-lab
+        ## BASE MODEL (SCORE)
+        instructlab/granite-7b-lab (0/10.0)
 
-        ## MODEL
-        instructlab/granite-7b-lab
+        ## MODEL (SCORE)
+        instructlab/granite-7b-lab (0/10.0)
 
-        ### IMPROVEMENTS:
-        1. category1/qna.yaml (+0.1)
-        2. category3/qna.yaml (+0.1)
+        ### IMPROVEMENTS (0.0 to 10.0):
+        1. category1/qna.yaml: 0.1 -> 0.2 (+0.1)
+        2. category3/qna.yaml: 0.1 -> 0.2 (+0.1)
 
-        ### REGRESSIONS:
-        1. category2/qna.yaml (-0.1)
-        2. category4/qna.yaml (-0.1)
+        ### REGRESSIONS (0.0 to 10.0):
+        1. category2/qna.yaml: 0.4 -> 0.3 (-0.1)
+        2. category4/qna.yaml: 0.4 -> 0.3 (-0.1)
 
-        ### NO CHANGE:
-        1. category5/qna.yaml
+        ### NO CHANGE (0.0 to 10.0):
+        1. category5/qna.yaml (0.5)
 
-        ### NEW:
-        1. category6/qna.yaml
+        ### NEW (0.0 to 10.0):
+        1. category6/qna.yaml (0.6)
         """
     )
     if error_rate > 0:
@@ -242,10 +239,10 @@ def test_evaluate_mt_bench(
 @patch(
     "instructlab.eval.mt_bench.MTBenchBranchEvaluator.judge_answers",
     side_effect=[
-        (gen_qa_pairs(True), 0),
-        (gen_qa_pairs(False), 0),
-        (gen_qa_pairs(True), 0.4567),
-        (gen_qa_pairs(False), 0.4567),
+        (0, gen_qa_pairs(True), 0),
+        (0, gen_qa_pairs(False), 0),
+        (0, gen_qa_pairs(True), 0.4567),
+        (0, gen_qa_pairs(False), 0.4567),
     ],
 )
 def test_evaluate_mt_bench_branch(
@@ -299,13 +296,10 @@ def test_evaluate_mmlu(
         """\
         # KNOWLEDGE EVALUATION REPORT
 
-        ## MODEL
-        instructlab/granite-7b-lab
-
-        ### AVERAGE:
-        0.5 (across 2)
+        ## MODEL (SCORE)
+        instructlab/granite-7b-lab (0.5/1.0)
 
-        ### SCORES:
+        ### SCORES (0.0 to 1.0):
         task1 - 0.1
         task2 - 0.9
         """
@@ -352,25 +346,22 @@ def test_evaluate_mmlu_branch(
         """\
         # KNOWLEDGE EVALUATION REPORT
 
-        ## BASE MODEL
-        instructlab/granite-7b-lab
-
-        ## MODEL
-        instructlab/granite-7b-lab
+        ## BASE MODEL (SCORE)
+        instructlab/granite-7b-lab (0.6/1.0)
 
-        ### AVERAGE:
-        -0.1 (across 5)
+        ## MODEL (SCORE)
+        instructlab/granite-7b-lab (0.5/1.0)
 
-        ### IMPROVEMENTS:
-        1. task1 (+0.1)
-        2. task3 (+0.1)
+        ### IMPROVEMENTS (0.0 to 1.0):
+        1. task1: 0.1 -> 0.2 (+0.1)
+        2. task3: 0.1 -> 0.2 (+0.1)
 
-        ### REGRESSIONS:
-        1. task2 (-0.1)
-        2. task4 (-0.1)
+        ### REGRESSIONS (0.0 to 1.0):
+        1. task2: 0.4 -> 0.3 (-0.1)
+        2. task4: 0.4 -> 0.3 (-0.1)
 
-        ### NO CHANGE:
-        1. task5
+        ### NO CHANGE (0.0 to 1.0):
+        1. task5 (0.5)
         """
     )
     assert result.output == expected