Repository: instructlab/instructlab

Issue:
Add option to take advantage of mt_bench max_workers=auto
**Is your feature request related to a problem? Please describe.**
Currently, max_workers with evaluate needs to be set manually.  If the value set isn't ideal, a the user is given a recommendation of better values.

**Describe the solution you'd like**
With https://github.com/instructlab/eval/pull/101, eval now has an option to auto tune max_workers.  This should probably become the default for the cli to consume for eval and training.



Discussion:
@JamesKunstle FYI



Patch:
diff --git a/.spellcheck-en-custom.txt b/.spellcheck-en-custom.txt
index 41a661bb0f..4e40197534 100644
--- a/.spellcheck-en-custom.txt
+++ b/.spellcheck-en-custom.txt
@@ -22,6 +22,7 @@ Config
 Containerfile
 coreutils
 cpp
+CPUs
 CRB
 cuBLAS
 cuda
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e531f889b..f63633106e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,5 @@
 ## v0.19
 
-### Features
-
-* Add `log_format` to the `config.yaml` file to allow for customizing the log format.
-
 ### Breaking Changes
 
 * InstructLab now uses XDG-based directories on macOS, similar to Linux.
@@ -24,6 +20,10 @@
 ### Features
 
 * `ilab config init` now auto detects your hardware when running on Nvidia enabled systems and chooses the best train profile. It does this by checking first if your system directly matches one of our supported train profiles and then attempts to match the vRAM for each profile to the total vRAM on your system.
+* Add `log_format` to the `config.yaml` file to allow for customizing the log format.
+* `ilab model evaluate --max-workers=auto` is now supported and is the default option. When
+  auto is specified, the optimal value is determined based on your GPUs, CPUs, and
+  configuration.
 
 ## v0.18.1
 
diff --git a/src/instructlab/configuration.py b/src/instructlab/configuration.py
index 6d5906148e..c209bdf4d1 100644
--- a/src/instructlab/configuration.py
+++ b/src/instructlab/configuration.py
@@ -307,9 +307,9 @@ class _mtbench(BaseModel):
         default_factory=lambda: DEFAULTS.EVAL_DATA_DIR,
         description="Directory where evaluation results are stored.",
     )
-    max_workers: int = Field(
-        default=16,
-        description="Number of workers to use for evaluation.",
+    max_workers: str | int = Field(
+        default="auto",
+        description="Number of workers to use for evaluation with mt_bench or mt_bench_branch. Must be a positive integer or 'auto'.",
     )
 
 
diff --git a/src/instructlab/model/evaluate.py b/src/instructlab/model/evaluate.py
index 7106f1c60f..f9c1b05559 100644
--- a/src/instructlab/model/evaluate.py
+++ b/src/instructlab/model/evaluate.py
@@ -9,7 +9,6 @@
 import multiprocessing
 import os
 import pathlib
-import typing
 
 # Third Party
 import click
@@ -21,10 +20,6 @@
 # Local
 from ..utils import http_client
 
-if typing.TYPE_CHECKING:
-    # Third Party
-    from instructlab.eval.evaluator import Evaluator
-
 logger = logging.getLogger(__name__)
 
 
@@ -36,7 +31,7 @@ class Benchmark(str, enum.Enum):
     MT_BENCH_BRANCH = "mt_bench_branch"
 
 
-def get_evaluator(
+def validate_options(
     model,
     base_model,
     benchmark,
@@ -49,11 +44,9 @@ def get_evaluator(
     few_shots,
     batch_size,
     tasks_dir,
-    merge_system_user_message,
-) -> "Evaluator":
+):
     """takes in arguments from the CLI and uses 'benchmark' to validate other arguments
-    if all needed configuration is present, returns the appropriate Evaluator class for the benchmark
-    otherwise raises an exception for the missing values
+    if all needed configuration is present, raises an exception for the missing values
     """
 
     # ensure skills benchmarks have proper arguments if selected
@@ -68,6 +61,7 @@ def get_evaluator(
             "model",
             "judge-model",
         ]
+
         if benchmark == Benchmark.MT_BENCH_BRANCH:
             required_args.append(taxonomy_path)
             required_args.append(branch)
@@ -86,32 +80,17 @@ def get_evaluator(
 
         validate_model(model)
         validate_model(judge_model, "--judge-model")
-        if benchmark == Benchmark.MT_BENCH:
-            # Third Party
-            from instructlab.eval.mt_bench import MTBenchEvaluator
+        if benchmark == Benchmark.MT_BENCH_BRANCH:
+            validate_model(base_model, "--base-model")
 
-            return MTBenchEvaluator(
-                get_model_name(model),
-                get_model_name(judge_model),
-                output_dir,
-                max_workers,
-                merge_system_user_message=merge_system_user_message,
+        if (isinstance(max_workers, str) and max_workers != "auto") or (
+            isinstance(max_workers, int) and max_workers < 1
+        ):
+            click.secho(
+                "max-workers must be specified as a positive integer or 'auto'",
+                fg="red",
             )
-
-        validate_model(base_model, "--base-model")
-
-        # Third Party
-        from instructlab.eval.mt_bench import MTBenchBranchEvaluator
-
-        return MTBenchBranchEvaluator(
-            get_model_name(model),
-            get_model_name(judge_model),
-            taxonomy_path,
-            branch,
-            output_dir,
-            max_workers,
-            merge_system_user_message=merge_system_user_message,
-        )
+            raise click.exceptions.Exit(1)
 
     # ensure knowledge benchmarks have proper arguments if selected
     if benchmark in [Benchmark.MMLU, Benchmark.MMLU_BRANCH]:
@@ -130,37 +109,8 @@ def get_evaluator(
             raise click.exceptions.Exit(1)
 
         validate_model(model)
-        if benchmark == Benchmark.MMLU:
-            # Third Party
-            from instructlab.eval.mmlu import MMLUEvaluator
-
-            min_tasks = os.environ.get("INSTRUCTLAB_EVAL_MMLU_MIN_TASKS")
-            if min_tasks is not None:
-                tasks = ["mmlu_abstract_algebra", "mmlu_anatomy", "mmlu_astronomy"]
-                evaluator = MMLUEvaluator(
-                    model,
-                    tasks=tasks,
-                    few_shots=few_shots,
-                    batch_size=batch_size,
-                )
-            else:
-                evaluator = MMLUEvaluator(
-                    model, few_shots=few_shots, batch_size=batch_size
-                )
-            return evaluator
-
-        validate_model(base_model, "--base-model")
-
-        # Third Party
-        from instructlab.eval.mmlu import MMLUBranchEvaluator
-
-        return MMLUBranchEvaluator(
-            model,
-            tasks_dir,
-            ["mmlu_pr"],
-            few_shots=few_shots,
-            batch_size=batch_size,
-        )
+        if benchmark == Benchmark.MMLU_BRANCH:
+            validate_model(base_model, "--base-model")
 
 
 def validate_model(model: str, model_arg: str = "--model"):
@@ -274,6 +224,7 @@ def get_model_name(model_path):
 
 
 def get_cpu_count():
+    """Returns the available cpu count to this process"""
     try:
         # Not available on all platforms
         return len(os.sched_getaffinity(0))  # type: ignore[attr-defined]
@@ -281,58 +232,77 @@ def get_cpu_count():
         return multiprocessing.cpu_count()
 
 
+def get_gpus(ctx, gpus=None) -> tuple[int | None, int]:
+    """Return the number of gpus explicitly selected through --gpus or config
+    The second value in the tuple is the effective gpus that will be used by
+    serving. If gpus is specified, the two values will be the same. 0 is the min
+    value for effective_gpus.
+    """
+    # First Party
+    from instructlab.model.backends.vllm import get_argument
+
+    eval_serve = ctx.obj.config.serve
+    gpus = gpus or eval_serve.vllm.gpus
+
+    effective_gpus = gpus
+    if effective_gpus is None:
+        try:
+            tps = get_argument("--tensor-parallel-size", eval_serve.vllm.vllm_args)
+            if tps is not None:
+                effective_gpus = int(tps)
+        except ValueError:
+            logger.warning("Invalid --tensor-parallel-size found in serve vllm_args")
+    effective_gpus = effective_gpus or 0
+    return gpus, effective_gpus
+
+
+def get_backend(backend, model):
+    """Return the backend based on specified backend and model detection"""
+    if backend is None:
+        try:
+            return backends.get(pathlib.Path(model), backend)
+        except ValueError as e:
+            click.secho(f"Failed to determine backend: {e}", fg="red")
+            raise click.exceptions.Exit(1)
+    return backend
+
+
 def launch_server(
     ctx: click.Context,
     model: str,
     model_name: str,
-    max_workers: int | None,
+    max_workers: str | int | None,
     gpus: int | None,
     backend: str | None,
     enable_serving_output: bool,
 ) -> tuple:
     eval_serve = deepcopy(ctx.obj.config.serve)
-    if backend is None:
-        try:
-            backend = eval_serve.backend = backends.get(pathlib.Path(model), backend)
-        except ValueError as e:
-            click.secho(f"Failed to determine backend: {e}", fg="red")
-            raise click.exceptions.Exit(1)
+    eval_serve.backend = backend = get_backend(backend, model)
 
+    effective_gpus = 0
     if backend == backends.VLLM:
-        # First Party
-        from instructlab.model.backends.vllm import contains_argument, get_argument
-
         eval_serve.vllm.vllm_args = eval_serve.vllm.vllm_args or []
         eval_serve.vllm.vllm_args.extend(["--served-model-name", model_name])
 
-        tps_prefix = "--tensor-parallel-size"
-        gpus = gpus or eval_serve.vllm.gpus
+        # First Party
+        from instructlab.model.backends.vllm import contains_argument
+
+        gpus, effective_gpus = get_gpus(ctx, gpus)
         if gpus:
+            tps_prefix = "--tensor-parallel-size"
             if contains_argument(tps_prefix, eval_serve.vllm.vllm_args):
                 click.secho(
-                    "Using gpus from --gpus or evaluate config and ignoring --tensor-parallel-size configured in serve vllm_args",
+                    "Using gpus from --gpus or config and ignoring --tensor-parallel-size configured in serve vllm_args",
                     fg="yellow",
                 )
             eval_serve.vllm.vllm_args.extend([tps_prefix, str(gpus)])
 
-        effective_gpus = gpus
-        if effective_gpus is None:
-            try:
-                tps = get_argument(tps_prefix, eval_serve.vllm.vllm_args)
-                if tps is not None:
-                    effective_gpus = int(tps)
-            except ValueError:
-                logger.warning(
-                    "Invalid --tensor-parallel-size found in serve vllm_args"
-                )
-        effective_gpus = effective_gpus or 1
-
-        if max_workers is not None:
+        if max_workers is not None and isinstance(max_workers, int):
             # Recommend max-workers based on hardware configuration: min(#GPUs being used * 10, #CPU cores) +- 50%
             # Edge cases:
             # - Many GPUs, not many CPUs: Unlikely, workers might not be able to keep the GPUs busy but recommendation can be ignored.
             # - Many CPUs, not many GPUs: More likely, 10 workers per GPU should still be reasonable.
-            target_max_workers = min(effective_gpus * 10, get_cpu_count())
+            target_max_workers = min(max(effective_gpus, 1) * 10, get_cpu_count())
             recommended_min_workers = max(target_max_workers // 2, 1)
             recommended_max_workers = max(int(target_max_workers // 0.5), 1)
             if (
@@ -348,8 +318,8 @@ def launch_server(
             logger.debug(
                 "Evaluate requires a context size of >= 5120, ignoring serve configuration for max_ctx_size"
             )
-        # llama-cpp fails fast on too many incoming requests and returns errors to client
-        if max_workers is not None:
+        if max_workers is not None and isinstance(max_workers, int):
+            # llama-cpp fails fast on too many incoming requests and returns errors to client
             recommended_workers = max(get_cpu_count() // 2, 1)
             if max_workers > recommended_workers:
                 logger.warning(
@@ -379,7 +349,7 @@ def launch_server(
     except Exception as exc:
         click.secho(f"Failed to start server: {exc}", fg="red")
         raise click.exceptions.Exit(1)
-    return backend_instance, api_base
+    return backend_instance, api_base, effective_gpus
 
 
 @click.command()
@@ -414,7 +384,7 @@ def launch_server(
 )
 @click.option(
     "--max-workers",
-    type=click.INT,
+    type=click.STRING,
     cls=clickext.ConfigOption,
     config_sections="mt_bench",
 )
@@ -511,12 +481,12 @@ def evaluate(
     benchmark,
     judge_model,
     output_dir,
-    max_workers,
+    max_workers: str | int,
     taxonomy_path,
     branch,
     base_branch,
     few_shots,
-    batch_size,
+    batch_size: str | int,
     tasks_dir,
     gpus,
     merge_system_user_message,
@@ -530,61 +500,77 @@ def evaluate(
 ):
     """Evaluates a trained model"""
 
-    with contextlib.suppress(ValueError):
-        batch_size = int(batch_size)
-
-    # get appropriate evaluator class from Eval lib
-    evaluator = get_evaluator(
-        model,
-        base_model,
-        benchmark,
-        judge_model,
-        output_dir,
-        max_workers,
-        taxonomy_path,
-        branch,
-        base_branch,
-        few_shots,
-        batch_size,
-        tasks_dir,
-        merge_system_user_message,
-    )
-
     # Third Party
     from instructlab.eval.exceptions import EvalError
 
+    with contextlib.suppress(ValueError):
+        max_workers = int(max_workers)
+    with contextlib.suppress(ValueError):
+        batch_size = int(batch_size)
+
     try:
+        # get appropriate evaluator class from Eval lib
+        validate_options(
+            model,
+            base_model,
+            benchmark,
+            judge_model,
+            output_dir,
+            max_workers,
+            taxonomy_path,
+            branch,
+            base_branch,
+            few_shots,
+            batch_size,
+            tasks_dir,
+        )
+
         if benchmark == Benchmark.MT_BENCH:
+            # Third Party
+            from instructlab.eval.mt_bench import MTBenchEvaluator
+
+            model_name = get_model_name(model)
+            judge_model_name = get_model_name(judge_model)
+            evaluator = MTBenchEvaluator(
+                model_name,
+                judge_model_name,
+                output_dir,
+                merge_system_user_message=merge_system_user_message,
+            )
             print("Generating answers...")
             server = None
             try:
-                server, api_base = launch_server(
+                server, api_base, effective_gpus = launch_server(
                     ctx,
                     model,
-                    get_model_name(model),
+                    model_name,
                     max_workers,
                     gpus,
                     backend,
                     enable_serving_output,
                 )
-                evaluator.gen_answers(api_base)
+                evaluator.gen_answers(
+                    api_base, max_workers=max_workers, serving_gpus=effective_gpus
+                )
             finally:
                 if server is not None:
                     server.shutdown()
 
             print("Evaluating answers...")
             try:
-                server, api_base = launch_server(
+                server, api_base, effective_gpus = launch_server(
                     ctx,
                     judge_model,
-                    get_model_name(judge_model),
+                    judge_model_name,
                     max_workers,
                     gpus,
                     judge_backend,
                     enable_serving_output,
                 )
                 overall_score, qa_pairs, turn_scores, error_rate = (
-                    evaluator.judge_answers(api_base)
+                    evaluator.judge_answers(
+                        api_base, max_workers=max_workers, serving_gpus=effective_gpus
+                    )
                 )
             finally:
                 if server is not None:
@@ -607,21 +593,30 @@ def evaluate(
             # Third Party
             from instructlab.eval.mt_bench import MTBenchBranchEvaluator
 
+            model_name = get_model_name(model)
+            base_model_name = get_model_name(base_model)
+            judge_model_name = get_model_name(judge_model)
             evaluators = [
-                evaluator,
                 MTBenchBranchEvaluator(
-                    get_model_name(base_model),
-                    get_model_name(judge_model),
+                    model_name,
+                    judge_model_name,
+                    taxonomy_path,
+                    branch,
+                    output_dir,
+                    merge_system_user_message=merge_system_user_message,
+                ),
+                MTBenchBranchEvaluator(
+                    base_model_name,
+                    judge_model_name,
                     taxonomy_path,
                     base_branch,
                     output_dir,
-                    max_workers,
                     merge_system_user_message=merge_system_user_message,
                 ),
             ]
             branches = [branch, base_branch]
             m_paths = [model, base_model]
-            m_names = [get_model_name(model), get_model_name(base_model)]
+            m_names = [model_name, base_model_name]
             qa_pairs_and_errors = []
             server = None
 
@@ -634,7 +629,7 @@ def evaluate(
                     f"Generating questions and reference answers from qna files for branch {branch}..."
                 )
                 try:
-                    server, api_base = launch_server(
+                    server, api_base, effective_gpus = launch_server(
                         ctx,
                         m_path,
                         m_name,
@@ -643,14 +638,16 @@ def evaluate(
                         backend,
                         enable_serving_output,
                     )
-                    evaluator.gen_answers(api_base)
+                    evaluator.gen_answers(
+                        api_base, max_workers=max_workers, serving_gpus=effective_gpus
+                    )
                 finally:
                     if server is not None:
                         server.shutdown()
 
             try:
                 # Share the judge model server for the two model evaluations
-                server, api_base = launch_server(
+                server, api_base, effective_gpus = launch_server(
                     ctx,
                     judge_model,
                     get_model_name(judge_model),
@@ -662,7 +659,9 @@ def evaluate(
                 for i, evaluator in enumerate(evaluators):
                     branch = branches[i]
                     print(f"Evaluating answers for branch {branch}...")
-                    qa_pairs, error_rate = evaluator.judge_answers(api_base)
+                    qa_pairs, error_rate = evaluator.judge_answers(
+                        api_base, max_workers=max_workers, serving_gpus=effective_gpus
+                    )
                     qa_pairs_and_errors.append((qa_pairs, error_rate))
             finally:
                 if server is not None:
@@ -695,9 +694,25 @@ def evaluate(
             display_error_rate((error_rate + base_error_rate) / 2)
 
         elif benchmark == Benchmark.MMLU:
+            # Third Party
+            from instructlab.eval.mmlu import MMLUEvaluator
+
+            min_tasks = os.environ.get("INSTRUCTLAB_EVAL_MMLU_MIN_TASKS")
+            if min_tasks is not None:
+                tasks = ["mmlu_abstract_algebra", "mmlu_anatomy", "mmlu_astronomy"]
+                evaluator = MMLUEvaluator(
+                    model,
+                    tasks=tasks,
+                    few_shots=few_shots,
+                    batch_size=batch_size,
+                )
+            else:
+                evaluator = MMLUEvaluator(
+                    model, few_shots=few_shots, batch_size=batch_size
+                )
             server = None
             try:
-                server, api_base = launch_server(
+                server, api_base, _ = launch_server(
                     ctx,
                     model,
                     model,
@@ -726,7 +741,13 @@ def evaluate(
             from instructlab.eval.mmlu import MMLUBranchEvaluator
 
             evaluators = [
-                evaluator,
+                MMLUBranchEvaluator(
+                    model,
+                    tasks_dir,
+                    ["mmlu_pr"],
+                    few_shots=few_shots,
+                    batch_size=batch_size,
+                ),
                 MMLUBranchEvaluator(
                     base_model,
                     tasks_dir,
@@ -742,7 +763,7 @@ def evaluate(
                 m_path = m_paths[i]
                 server = None
                 try:
-                    server, api_base = launch_server(
+                    server, api_base, _ = launch_server(
                         ctx,
                         m_path,
                         m_path,
diff --git a/src/instructlab/model/train.py b/src/instructlab/model/train.py
index 5e203c7a4b..0c9dc8f199 100644
--- a/src/instructlab/model/train.py
+++ b/src/instructlab/model/train.py
@@ -5,7 +5,6 @@
 import enum
 import functools
 import logging
-import multiprocessing
 import os
 import pathlib
 import pprint
@@ -801,7 +800,7 @@ def _mtbench(
     import torch
 
     # First Party
-    from instructlab.model.evaluate import launch_server
+    from instructlab.model.evaluate import get_gpus, get_model_name, launch_server
 
     # hard-override for local insecure vLLM serving
     ctx.params["tls_client_cert"] = None
@@ -809,48 +808,66 @@ def _mtbench(
     ctx.params["tls_client_passwd"] = None
     ctx.params["tls_insecure"] = True
 
+    explicit_gpus = None
+    gpus, effective_gpus = get_gpus(ctx, ctx.obj.evaluate.gpus)
+    if gpus and gpus > 0:
+        # gpus are specified in config for evaluate
+        logger.debug("Using gpus from config")
+        explicit_gpus = gpus
+    elif effective_gpus > 0:
+        # tensor-parallel size specified in serving config
+        logger.debug("Using gpus from --tensor-parallel-size in config")
+    else:
+        # TODO: Should be parameterized by the user specific for training
+        explicit_gpus = torch.cuda.device_count()
+        effective_gpus = explicit_gpus
+
+    model_name = get_model_name(str(model))
+    judge_model_name = get_model_name(str(mtbench_judge))
+
     evaluator = MTBenchEvaluator(
-        model_name=str(model),
-        judge_model_name=str(mtbench_judge),
+        model_name=model_name,
+        judge_model_name=judge_model_name,
         output_dir=str(eval_cache),
         merge_system_user_message=True,  # TODO: expose this to the user
     )
 
-    # TODO: Below, gpus=device_count() and max_workers=cpu_count() should be parameterized
-    #   by the user.
-
     server = None
     model_serve_url = None
     try:
         logger.debug("Starting model server for mt-bench answer generation")
-        server, model_serve_url = launch_server(
+        server, model_serve_url, effective_gpus = launch_server(
             ctx=ctx,
             model=str(model),
-            model_name=str(model),
-            gpus=torch.cuda.device_count(),
-            max_workers=multiprocessing.cpu_count(),
+            model_name=model_name,
+            gpus=explicit_gpus,
+            max_workers="auto",
             enable_serving_output=enable_serving_output,
             backend=backends.VLLM,
         )
         logger.debug("Generating mt-bench answers")
-        evaluator.gen_answers(model_serve_url)
+        evaluator.gen_answers(
+            model_serve_url, max_workers="auto", serving_gpus=effective_gpus
+        )
     finally:
         if server is not None:
             server.shutdown()
 
     try:
         logger.debug("Starting model server for mt-bench answer judgment")
-        server, model_serve_url = launch_server(
+        server, model_serve_url, effective_gpus = launch_server(
             ctx=ctx,
             model=str(mtbench_judge),
-            model_name=str(mtbench_judge),
-            gpus=torch.cuda.device_count(),
-            max_workers=multiprocessing.cpu_count(),
+            model_name=judge_model_name,
+            gpus=explicit_gpus,
+            max_workers="auto",
             backend=backends.VLLM,
             enable_serving_output=enable_serving_output,
         )
         logger.debug("Judging mt-bench answers")
-        mt_bench_results: tuple = evaluator.judge_answers(model_serve_url)
+        mt_bench_results: tuple = evaluator.judge_answers(
+            model_serve_url, max_workers="auto", serving_gpus=effective_gpus
+        )
         ckpt_score: float = mt_bench_results[0]
     finally:
         if server is not None:


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
index 330d954e3a..aa83b08179 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -292,7 +292,7 @@ def test_config_modified_settings(self, tmp_path_home):
   mt_bench:
     judge_model: prometheus
     output_dir: /dir/to/output
-    max_workers: 5
+    max_workers: auto
   mt_bench_branch:
     taxonomy_path: taxonomy
 general:
diff --git a/tests/test_lab_evaluate.py b/tests/test_lab_evaluate.py
index 4438c766e7..0237ab5f60 100644
--- a/tests/test_lab_evaluate.py
+++ b/tests/test_lab_evaluate.py
@@ -204,7 +204,7 @@ def run_mt_bench_branch(cli_runner, error_rate):
 @patch("instructlab.model.evaluate.validate_model")
 @patch(
     "instructlab.model.evaluate.launch_server",
-    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1"),
+    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1", 1),
 )
 @patch("instructlab.eval.mt_bench.MTBenchEvaluator.gen_answers")
 @patch(
@@ -236,7 +236,7 @@ def test_evaluate_mt_bench(
 @patch("instructlab.model.evaluate.validate_model")
 @patch(
     "instructlab.model.evaluate.launch_server",
-    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1"),
+    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1", 1),
 )
 @patch("instructlab.eval.mt_bench.MTBenchBranchEvaluator.gen_answers")
 @patch(
@@ -270,7 +270,7 @@ def test_evaluate_mt_bench_branch(
 @patch("instructlab.model.evaluate.validate_model")
 @patch(
     "instructlab.model.evaluate.launch_server",
-    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1"),
+    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1", 1),
 )
 @patch(
     "instructlab.eval.mmlu.MMLUEvaluator.run",
@@ -316,7 +316,7 @@ def test_evaluate_mmlu(
 @patch("instructlab.model.evaluate.validate_model")
 @patch(
     "instructlab.model.evaluate.launch_server",
-    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1"),
+    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1", 1),
 )
 @patch(
     "instructlab.eval.mmlu.MMLUBranchEvaluator.run",
@@ -427,6 +427,30 @@ def test_invalid_model_mt_bench(cli_runner: CliRunner):
     assert result.exit_code != 0
 
 
+@patch("instructlab.model.evaluate.validate_model")
+def test_invalid_max_workers(_, cli_runner: CliRunner):
+    result = cli_runner.invoke(
+        lab.ilab,
+        [
+            "--config=DEFAULT",
+            "model",
+            "evaluate",
+            "--benchmark",
+            "mt_bench",
+            "--model",
+            "instructlab/granite-7b-lab",
+            "--judge-model",
+            "instructlab/merlinite-7b-lab",
+            "--max-workers",
+            "invalid",
+        ],
+    )
+    assert (
+        "max-workers must be specified as a positive integer or 'auto'" in result.output
+    )
+    assert result.exit_code != 0
+
+
 def test_invalid_model_mmlu(cli_runner: CliRunner):
     result = cli_runner.invoke(
         lab.ilab,
@@ -470,7 +494,7 @@ def test_int_batchsize_mmlu(mmlu_mock, _, cli_runner: CliRunner):
 @patch("instructlab.model.evaluate.validate_model")
 @patch(
     "instructlab.model.evaluate.launch_server",
-    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1"),
+    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1", 1),
 )
 def test_invalid_taxonomy_mt_bench_branch(launch_server_mock, _, cli_runner: CliRunner):
     result = cli_runner.invoke(
@@ -503,7 +527,7 @@ def test_invalid_taxonomy_mt_bench_branch(launch_server_mock, _, cli_runner: Cli
 @patch("instructlab.model.evaluate.validate_model")
 @patch(
     "instructlab.model.evaluate.launch_server",
-    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1"),
+    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1", 1),
 )
 def test_invalid_branch_mt_bench_branch(launch_server_mock, _, cli_runner: CliRunner):
     setup_taxonomy("taxonomy")
@@ -537,7 +561,7 @@ def test_invalid_branch_mt_bench_branch(launch_server_mock, _, cli_runner: CliRu
 @patch("instructlab.model.evaluate.validate_model")
 @patch(
     "instructlab.model.evaluate.launch_server",
-    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1"),
+    return_value=(mock.MagicMock(), "http://127.0.0.1:8000/v1", 1),
 )
 def test_invalid_tasks_dir(_, __, cli_runner: CliRunner):
     result = cli_runner.invoke(
diff --git a/tests/testdata/default_config.yaml b/tests/testdata/default_config.yaml
index 9e0071c1e1..7ee9556962 100644
--- a/tests/testdata/default_config.yaml
+++ b/tests/testdata/default_config.yaml
@@ -66,9 +66,10 @@ evaluate:
     # Judge model for mt_bench and mt_bench_branch.
     # Default: prometheus-eval/prometheus-8x7b-v2.0
     judge_model: prometheus-eval/prometheus-8x7b-v2.0
-    # Number of workers to use for evaluation.
-    # Default: 16
-    max_workers: 16
+    # Number of workers to use for evaluation with mt_bench or mt_bench_branch. Must
+    # be a positive integer or 'auto'.
+    # Default: auto
+    max_workers: auto
     # Directory where evaluation results are stored.
     # Default: /data/instructlab/internal/eval_data
     output_dir: /data/instructlab/internal/eval_data