Repository: instructlab/instructlab

Issue:
evaluate: consider renaming --sdg-path to --tasks-dir or similar
This `model evaluate` option:

```
@click.option(
    "--sdg-path",
    type=click.Path(),
    cls=clickext.ConfigOption,
    config_sections="mmlu_branch",
    help="Path where all the MMLU Branch tasks are stored. Needed for running mmlu_branch.",
)
```

has nothing to do with the `instructlab.sdg` library, right? It's a directory that contains lm_eval tasks config files

This is pretty confusing, since the sdg library will have its own directory full of config files that is completely unrelated

I think it would be good to change to `--lm-eval-tasks-dir` or similar to avoid confusion

@danmcp 



Discussion:
I had been considering whether this should be renamed to something like --generated-tasks-path but hadn't thought about putting lm-eval in the name since the format isn't specific to lm-eval and we could potentially change that dependency and still implement mmlu.

tasks-dir is probably the appropriate name for the library since it doesn't care how you got the tasks.  I am debating whether --generated-tasks-dir or --tasks-dir is best for the cli.  The term generated could be a useful clue there.

@alimaredia @alinaryan
+1 for --tasks-dir
--tasks-dir seems fine with me.



Patch:
diff --git a/src/instructlab/configuration.py b/src/instructlab/configuration.py
index 6050068b5c..c3901d1a93 100644
--- a/src/instructlab/configuration.py
+++ b/src/instructlab/configuration.py
@@ -301,7 +301,7 @@ class _mtbenchbranch(BaseModel):
 
 
 class _mmlubranch(BaseModel):
-    sdg_path: str
+    tasks_dir: str
 
 
 class _evaluate(BaseModel):
@@ -415,7 +415,7 @@ def get_default_config() -> Config:
                 batch_size=5,
             ),
             mt_bench_branch=_mtbenchbranch(taxonomy_path=DEFAULTS.TAXONOMY_DIR),
-            mmlu_branch=_mmlubranch(sdg_path=DEFAULTS.DATASETS_DIR),
+            mmlu_branch=_mmlubranch(tasks_dir=DEFAULTS.DATASETS_DIR),
         ),
     )
 
diff --git a/src/instructlab/model/evaluate.py b/src/instructlab/model/evaluate.py
index b6b2204d83..cb423fa102 100644
--- a/src/instructlab/model/evaluate.py
+++ b/src/instructlab/model/evaluate.py
@@ -50,7 +50,7 @@ def get_evaluator(
     base_branch,
     few_shots,
     batch_size,
-    sdg_path,
+    tasks_dir,
     merge_system_user_message,
 ) -> "Evaluator":
     """takes in arguments from the CLI and uses 'benchmark' to validate other arguments
@@ -119,9 +119,9 @@ def get_evaluator(
         required_args = [model, few_shots, batch_size]
         required_arg_names = ["model"]
         if benchmark == Benchmark.MMLU_BRANCH:
-            required_args.append(sdg_path)
+            required_args.append(tasks_dir)
             required_args.append(base_model)
-            required_arg_names.append("sdg-path")
+            required_arg_names.append("tasks-dir")
             required_arg_names.append("base-model")
         if None in required_args:
             click.secho(
@@ -147,7 +147,7 @@ def get_evaluator(
             return evaluator
         return evaluator_class(
             model,
-            sdg_path,
+            tasks_dir,
             ["mmlu_pr"],
             few_shots=few_shots,
             batch_size=batch_size,
@@ -359,7 +359,7 @@ def launch_server(
     help="Number of GPUs. Needed for running mmlu or mmlu_branch.",
 )
 @click.option(
-    "--sdg-path",
+    "--tasks-dir",
     type=click.Path(),
     cls=clickext.ConfigOption,
     config_sections="mmlu_branch",
@@ -420,7 +420,7 @@ def evaluate(
     base_branch,
     few_shots,
     batch_size,
-    sdg_path,
+    tasks_dir,
     gpus,
     merge_system_user_message,
     backend,
@@ -442,7 +442,7 @@ def evaluate(
         base_branch,
         few_shots,
         batch_size,
-        sdg_path,
+        tasks_dir,
         merge_system_user_message,
     )
 
@@ -615,7 +615,7 @@ def evaluate(
             evaluator,
             MMLUBranchEvaluator(
                 base_model,
-                sdg_path,
+                tasks_dir,
                 ["mmlu_pr"],
                 few_shots=few_shots,
                 batch_size=batch_size,


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
index 5184e32f17..8ad8a4e382 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -200,7 +200,7 @@ def test_config_modified_settings(self, tmp_path_home):
     few_shots: 2
     batch_size: 5
   mmlu_branch:
-    sdg_path: /path/to/sdg
+    tasks_dir: /path/to/sdg
   mt_bench:
     judge_model: prometheus
     output_dir: /dir/to/output
diff --git a/tests/test_lab_evaluate.py b/tests/test_lab_evaluate.py
index b35f942fd1..5000bf3455 100644
--- a/tests/test_lab_evaluate.py
+++ b/tests/test_lab_evaluate.py
@@ -263,7 +263,7 @@ def test_evaluate_mmlu_branch(run_mock, cli_runner: CliRunner):
             "models/instructlab/granite-7b-lab",
             "--base-model",
             "models/instructlab/granite-7b-lab",
-            "--sdg-path",
+            "--tasks-dir",
             "generated",
         ],
     )