Repository: instructlab/instructlab

Issue:
set train profile keys as optional
`ilab model train` errors when train profile is missing keys that are set by default (should be made optional in the training class)

@cdoern can you add the exact keys that are erroring?


Discussion:
this will be handled by #1945 



Patch:
diff --git a/src/instructlab/configuration.py b/src/instructlab/configuration.py
index 436d5a9569..d8fb7c38a3 100644
--- a/src/instructlab/configuration.py
+++ b/src/instructlab/configuration.py
@@ -260,6 +260,13 @@ class _chat(BaseModel):
     greedy_mode: bool = False
     max_tokens: typing.Optional[int] = None
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {"model": DEFAULTS.DEFAULT_MODEL}
+        return finish_cfg_section(defaults, values)
+
 
 class _serve_vllm(BaseModel):
     """Class describing configuration of vllm serving backend."""
@@ -311,6 +318,17 @@ def api_base(self):
         """Returns server API URL, based on the configured host and port"""
         return get_api_base(self.host_port)
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {
+            "model_path": DEFAULTS.DEFAULT_MODEL,
+            "llama_cpp": {"gpu_layers": -1, "max_ctx_size": 4096, "llm_family": ""},
+            "vllm": {"llm_family": "", "vllm_args": [], "max_startup_attempts": 300},
+        }
+        return finish_cfg_section(defaults, values)
+
 
 class _generate(BaseModel):
     """Class describing configuration of the generate sub-command."""
@@ -339,25 +357,70 @@ class _generate(BaseModel):
     seed_file: StrictStr = Field(default_factory=lambda: DEFAULTS.SEED_FILE)
     gpus: Optional[int] = None
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {
+            "model": DEFAULTS.DEFAULT_MODEL,
+            "taxonomy_path": DEFAULTS.TAXONOMY_DIR,
+            "taxonomy_base": DEFAULTS.TAXONOMY_BASE,
+        }
+
+        return finish_cfg_section(defaults, values)
+
 
 class _mmlu(BaseModel):
     few_shots: int
     batch_size: str
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {"few_shots": 5, "batch_size": "auto"}
+        return finish_cfg_section(defaults, values)
+
 
 class _mtbench(BaseModel):
     judge_model: str
     output_dir: str
     max_workers: int
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {
+            "judge_model": DEFAULTS.JUDGE_MODEL_MT,
+            "output_dir": DEFAULTS.EVAL_DATA_DIR,
+            "max_workers": 16,
+        }
+        return finish_cfg_section(defaults, values)
+
 
 class _mtbenchbranch(BaseModel):
     taxonomy_path: str
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {"taxonomy_path": DEFAULTS.TAXONOMY_DIR}
+
+        return finish_cfg_section(defaults, values)
+
 
 class _mmlubranch(BaseModel):
     tasks_dir: str
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {"tasks_dir": DEFAULTS.DATASETS_DIR}
+        return finish_cfg_section(defaults, values)
+
 
 class _evaluate(BaseModel):
     # model configuration
@@ -373,6 +436,23 @@ class _evaluate(BaseModel):
     mt_bench: _mtbench
     mt_bench_branch: _mtbenchbranch
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {
+            "base_model": DEFAULTS.MODEL_REPO,
+            "mt_bench": {
+                "judge_model": DEFAULTS.JUDGE_MODEL_MT,
+                "output_dir": DEFAULTS.EVAL_DATA_DIR,
+                "max_workers": 16,
+            },
+            "mmlu": {"few_shots": 5, "batch_size": "auto"},
+            "mt_bench_branch": {"taxonomy_path": DEFAULTS.TAXONOMY_DIR},
+            "mmlu_branch": {"tasks_dir": DEFAULTS.DATASETS_DIR},
+        }
+        return finish_cfg_section(defaults, values)
+
 
 class _train(BaseModel):
     # model configuration
@@ -413,6 +493,30 @@ class _train(BaseModel):
 
     phased_mt_bench_judge: str | None = None
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {
+            "model_path": DEFAULTS.MODEL_REPO,
+            "data_path": DEFAULTS.DATASETS_DIR,
+            "ckpt_output_dir": DEFAULTS.CHECKPOINTS_DIR,
+            "data_output_dir": DEFAULTS.INTERNAL_DIR,
+            "max_seq_len": 4096,
+            "max_batch_len": 10000,
+            "num_epochs": 10,
+            "effective_batch_size": 3840,
+            "save_samples": 250000,
+            "lora_quantize_dtype": None,
+            "lora_rank": 4,
+            "nproc_per_node": 1,
+            "deepspeed_cpu_offload_optimizer": False,
+            "additional_args": {},
+            "is_padding_free": False,
+        }
+
+        return finish_cfg_section(defaults, values)
+
 
 class Config(BaseModel):
     """Configuration for the InstructLab CLI."""
@@ -435,6 +539,63 @@ class Config(BaseModel):
     # model configuration
     model_config = ConfigDict(extra="ignore")
 
+    @model_validator(mode="before")
+    def fill_defaults(
+        cls, values: typing.Dict[str, typing.Any]
+    ) -> typing.Dict[str, typing.Any]:
+        defaults = {
+            "chat": {"model": DEFAULTS.DEFAULT_MODEL},
+            "serve": {
+                "model_path": DEFAULTS.DEFAULT_MODEL,
+                "llama_cpp": {"gpu_layers": -1, "max_ctx_size": 4096, "llm_family": ""},
+                "vllm": {"llm_family": "", "vllm_args": []},
+            },
+            "generate": {
+                "model": DEFAULTS.DEFAULT_MODEL,
+                "taxonomy_path": DEFAULTS.TAXONOMY_DIR,
+                "taxonomy_base": DEFAULTS.TAXONOMY_BASE,
+            },
+            "evaluate": {
+                "model": DEFAULTS.DEFAULT_MODEL,
+                "taxonomy_path": DEFAULTS.TAXONOMY_DIR,
+                "taxonomy_base": DEFAULTS.TAXONOMY_BASE,
+            },
+            "train": {
+                "model_path": DEFAULTS.MODEL_REPO,
+                "data_path": DEFAULTS.DATASETS_DIR,
+                "ckpt_output_dir": DEFAULTS.CHECKPOINTS_DIR,
+                "data_output_dir": DEFAULTS.INTERNAL_DIR,
+                "max_seq_len": 4096,
+                "max_batch_len": 10000,
+                "num_epochs": 10,
+                "effective_batch_size": 3840,
+                "save_samples": 250000,
+                "lora_quantize_dtype": None,
+                "lora_rank": 4,
+                "nproc_per_node": 1,
+                "deepspeed_cpu_offload_optimizer": False,
+                "additional_args": {},
+                "is_padding_free": False,
+            },
+            "version": CONFIG_VERSION,
+        }
+
+        for key, val in defaults.items():
+            if values.get(key) is None and val is not None:
+                values[key] = val
+
+        return values
+
+
+def finish_cfg_section(
+    defaults: dict, values: typing.Dict[str, typing.Any]
+) -> typing.Dict[str, typing.Any]:
+    for key, val in defaults.items():
+        if values.get(key) is None and val is not None:
+            values[key] = val
+
+    return values
+
 
 def get_default_config() -> Config:
     """Generates default configuration for CLI"""
@@ -603,7 +764,6 @@ def ensure_storage_directories_exist():
     for dirpath in dirs_to_make:
         if not os.path.exists(dirpath):
             os.makedirs(dirpath, exist_ok=True)
-
     additional_args_and_defaults = {
         "learning_rate": 2e-5,
         "warmup_steps": 25,


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
index d35be83abe..2b1f7ad29c 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -15,6 +15,7 @@
 
 # First Party
 from instructlab import configuration as config
+from instructlab.configuration import DEFAULTS
 from instructlab.log import configure_logging
 
 
@@ -92,13 +93,12 @@ def _assert_model_defaults(self, cfg):
 
         assert cfg.evaluate is not None
         assert cfg.evaluate.base_model == "instructlab/granite-7b-lab"
-        assert cfg.evaluate.model is None
 
         assert cfg.generate is not None
-        assert cfg.generate.model == default_model
+        assert cfg.generate.model == DEFAULTS.DEFAULT_MODEL
 
         assert cfg.serve is not None
-        assert cfg.serve.model_path == default_model
+        assert cfg.serve.model_path == DEFAULTS.DEFAULT_MODEL
 
         assert cfg.train is not None
         assert cfg.train.model_path == "instructlab/granite-7b-lab"
@@ -109,7 +109,7 @@ def test_default_config(self, cli_runner):  # pylint: disable=unused-argument
         self._assert_defaults(cfg)
         self._assert_model_defaults(cfg)
 
-    def test_config_missing_required_field_groups(self, tmp_path_home):
+    def test_cfg_auto_fill(self, tmp_path_home):
         config_path = tmp_path_home / "config.yaml"
         with open(config_path, "w", encoding="utf-8") as config_file:
             config_file.write(
@@ -117,26 +117,17 @@ def test_config_missing_required_field_groups(self, tmp_path_home):
   log_level: INFO
 """
             )
-        with pytest.raises(
-            config.ConfigException,
-            match=r"""5 errors in [\/\w-]+config.yaml:
-- missing chat: field required
-- missing generate: field required
-- missing serve: field required
-- missing version: field required
-- missing evaluate: field required
-""",
-        ):
-            config.read_config(config_path)
-
-    def test_config_missing_required_fields(self, tmp_path_home):
+        cfg = config.read_config(config_path)
+        self._assert_defaults(cfg)
+        self._assert_model_defaults(cfg)
+
+    def test_cfg_auto_fill_with_large_config(self, tmp_path_home):  # pylint: disable=unused-argument
         config_path = tmp_path_home / "config.yaml"
         with open(config_path, "w", encoding="utf-8") as config_file:
             config_file.write(
                 """general:
   log_level: INFO
 generate:
-  model: models/merlinite-7b-lab-Q4_K_M.gguf
   teacher:
     model_path: models/granite-7b-lab-Q4_K_M.gguf
     chat_template: tokenizer
@@ -149,18 +140,10 @@ def test_config_missing_required_fields(self, tmp_path_home):
          - --tensor-parallel-size=8
 """
             )
-        with pytest.raises(
-            config.ConfigException,
-            match=r"""6 errors in [\/\w-]+config.yaml:
-- missing chat: field required
-- missing generate->taxonomy_path: field required
-- missing generate->taxonomy_base: field required
-- missing serve: field required
-- missing version: field required
-- missing evaluate: field required
-""",
-        ):
-            config.read_config(config_path)
+        cfg = config.read_config(config_path)
+        # make sure the generate cfg passed is preserved
+        assert cfg.generate.teacher.llama_cpp.max_ctx_size == 2048
+        self._assert_model_defaults(cfg)
 
     def test_validate_log_level_invalid(self):
         cfg = config.get_default_config()
@@ -323,8 +306,8 @@ def test_compare_default_config_testdata(
     [
         ("nf4", {}, False),
         (None, {}, False),
-        ("nf4", None, True),
-        (None, None, True),
+        ("nf4", None, False),
+        (None, None, False),
         ("valid-for-cli-but-not-for-training-library", {}, False),
         ("nf4", {"lora_alpha": 32}, False),
     ],
@@ -366,4 +349,5 @@ def test_read_train_profile(
         for k, v in result.dict().items():
             if k not in profile_data:
                 continue
-            assert v == profile_data[k]
+            if profile_data[k] is not None:
+                assert v == profile_data[k]