Repository: EleutherAI/lm-evaluation-harness
Pull request: add bypass metric
Started on #1152. Couple of issues:

- [x] Cannot amend `metric_list` in evaluate
- [x] print results crashing
- [x] repeated generations are not saved in `log_samples`

Problem statement:
Add `--predict_only` mode (run without scoring outputs)
As asked for by @dwadden from AI2.

We should support a CLI flag, `--predict_only`, which causes model outputs to be produced and saved, but to exit while reporting metric = `N/A` for all metrics. This is a useful feature in general.

This could also be extended, if we so desire, to code execution tasks--so that in lm-evaluation-harness, we support *the ability to run the generation passes needed for a HumanEval(+) score*, but tell the user to take those model outputs and score them offline, properly sandboxed, at their own risk.

Hints from issue discussion:
cc @StellaAthena --would you be willing to allow for us to support code generation benchmarks if we did not perform the code generation online, and just supported the ability to "dry-run" the model outputs?

I think this would be a good medium where we can still support code tasks (which are in high demand and people would appreciate having in LM-Eval-Harness). Currently, the standard practice is to just run the code without sandboxing in the community (e.g. Bigcode Eval Harness has just one CLI flag that turns on code execution), which is quite a bit different from before that bridge had been crossed, and we could still do significantly better / help raise the standard by not allowing for online evaluation of this code within our library and pointing to security practices users should take.
would a `metric: bypass` be a good abstraction do you think?
Yes, I think some "don't run" metric like that would be a good choice, for things like code tasks where we'd want that task's metric to be always bypassed at runtime.
> cc @StellaAthena --would you be willing to allow for us to support code generation benchmarks if we did not perform the code generation online, and just supported the ability to "dry-run" the model outputs?

Yes, this is an excellent idea.
Fantastic, I'm going to make trackers for some major code benchmarks soon then.

Commit message: add docs

Patch to review:
diff --git a/lm_eval/__main__.py b/lm_eval/__main__.py
index ebb1b6c4ab..2dc1c00449 100644
--- a/lm_eval/__main__.py
+++ b/lm_eval/__main__.py
@@ -142,6 +142,13 @@ def parse_eval_args() -> argparse.Namespace:
         metavar="CRITICAL|ERROR|WARNING|INFO|DEBUG",
         help="Controls the reported logging error level. Set to DEBUG when testing + adding new task configurations for comprehensive log output.",
     )
+    parser.add_argument(
+        "--predict_only",
+        "-x",
+        action="store_true",
+        default=False,
+        help="Use with --log_samples. Only model outputs will be saved and metrics will not be evaluated.",
+    )
     return parser.parse_args()
 
 
@@ -155,6 +162,11 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
     eval_logger.info(f"Verbosity set to {args.verbosity}")
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
+    if args.predict_only:
+        args.log_samples = True
+    if (args.log_samples or args.predict_only) and not args.output_path:
+        assert args.output_path, "Specify --output_path"
+
     initialize_tasks(args.verbosity)
 
     if args.limit:
@@ -222,8 +234,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         else:
             path.mkdir(parents=True, exist_ok=True)
             output_path_file = path.joinpath("results.json")
-    elif args.log_samples and not args.output_path:
-        assert args.output_path, "Specify --output_path"
 
     eval_logger.info(f"Selected Tasks: {task_names}")
 
@@ -242,6 +252,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         write_out=args.write_out,
         log_samples=args.log_samples,
         gen_kwargs=args.gen_kwargs,
+        predict_only=args.predict_only,
     )
 
     if results is not None:
diff --git a/lm_eval/api/metrics.py b/lm_eval/api/metrics.py
index 4eb68585b6..242e5e5bc7 100644
--- a/lm_eval/api/metrics.py
+++ b/lm_eval/api/metrics.py
@@ -14,6 +14,11 @@
 eval_logger = logging.getLogger("lm-eval")
 
 # Register Aggregations First
+@register_aggregation("null")
+def null(arr):
+    return -1
+
+
 @register_aggregation("mean")
 def mean(arr):
     return sum(arr) / len(arr)
@@ -206,6 +211,16 @@ def mean_stderr(arr):
     return sample_stddev(arr) / math.sqrt(len(arr))
 
 
+@register_metric(
+    metric="bypass",
+    higher_is_better=True,
+    output_type=["loglikelihood", "multiple_choice", "generate_until"],
+    aggregation="null",
+)
+def bypass(items):
+    return None
+
+
 @register_metric(
     metric="mcc",
     higher_is_better=True,
diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py
index 51248243e2..389a313228 100644
--- a/lm_eval/evaluator.py
+++ b/lm_eval/evaluator.py
@@ -42,6 +42,7 @@ def simple_evaluate(
     write_out: bool = False,
     log_samples: bool = True,
     gen_kwargs: str = None,
+    predict_only: bool = False,
 ):
     """Instantiate and evaluate a model on a list of tasks.
 
@@ -75,6 +76,9 @@ def simple_evaluate(
     :param gen_kwargs: str
         String arguments for model generation
         Ignored for all tasks with loglikelihood output_type
+    :param predict_only: bool
+        If true only model outputs will be generated and returned. Metrics will not be evaluated
+
     :return
         Dictionary of results
     """
@@ -130,6 +134,17 @@ def simple_evaluate(
                 continue
 
         config = task_obj._config
+        if predict_only:
+            log_samples = True
+            eval_logger.info(
+                f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
+            )
+            # we have to change the class properties post-hoc. This is pretty hacky.
+            task_obj._metric_fn_list = {"bypass": lambda x: None}
+            task_obj._metric_fn_kwargs = {"bypass": {}}
+            task_obj._aggregation_list = {"bypass": lambda x: -1}
+            config["metric_list"] = [{"metric": "bypass"}]
+
         if config["output_type"] == "generate_until" and gen_kwargs is not None:
             config["generation_kwargs"].update(gen_kwargs)
 
@@ -144,11 +159,11 @@ def simple_evaluate(
                     f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
                 )
 
-                task_obj._config["num_fewshot"] = num_fewshot
+                config["num_fewshot"] = num_fewshot
+        task_obj._config = config
 
     if check_integrity:
         run_task_tests(task_list=tasks)
-
     results = evaluate(
         lm=lm,
         task_dict=task_dict,
@@ -212,9 +227,16 @@ def evaluate(
     :return
         Dictionary of results
     """
-
     # decontaminate = decontamination_ngrams_path is not None
 
+    for task_name, task in task_dict.items():
+        if type(task) == tuple:
+            _, task = task
+        if not log_samples:
+            assert (
+                "bypass" not in getattr(task, "_metric_fn_list", {}).keys()
+            ), f"log_samples must be True for 'bypass' only tasks: {task_name}"
+
     # stores the final result for each task, for each metric/filter pair.
     results = collections.defaultdict(dict)
     # Tracks each task's version.


Code review comments:
diff_hunk: @@ -130,6 +134,17 @@ def simple_evaluate(
                 continue
 
         config = task_obj._config
+        if predict_only:
+            log_samples = True
+            eval_logger.info(
+                f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
+            )
+            # we have to change the class properties post-hoc. This is pretty hacky.
+            task_obj._metric_fn_list = {"bypass": lambda x: None}
path: lm_eval/evaluator.py
text: @user1:
maybe we can abstract this code that accesses private task class attributes into something like `ConfigurableTask.override_metrics()`? And/or an `ConfigurableTask.update_config()` method for updating the config it's based on