Repository: Project-MONAI/MONAI

Issue:
Auto3DSeg AutoRunner HPO training would save model checkpoint to wrong locations
**Describe the bug**

When the NNI HPO process starts, it will first use `generate` to generate templated algo with the new hyper-parameters and then `run_algo` with NNI to do `algo.train()`. However, because the generated algo doesn't update `bundle_root`, and the model checkpoints path is saved relative to the `bundle_root` config, the model checkpoints will appear in wrong locations.

**To Reproduce**
```
import os
import tempfile

from monai.bundle.config_parser import ConfigParser
from monai.apps import download_and_extract

from monai.apps.auto3dseg import AutoRunner

if __name__ == '__main__':
    directory = "./build"
    root_dir = tempfile.mkdtemp() if directory is None else directory
    print(root_dir)

    msd_task = "Task04_Hippocampus"
    resource = "https://msd-for-monai.s3-us-west-2.amazonaws.com/" + msd_task + ".tar"

    compressed_file = os.path.join(root_dir, msd_task + ".tar")


    dataroot = os.path.join(root_dir, msd_task)
    if not os.path.exists(dataroot):
        download_and_extract(resource, compressed_file, root_dir)

    datalist_file = os.path.join("../tutorials/auto3dseg/tasks", "msd", msd_task, "msd_" + msd_task.lower() + "_folds.json")

    input_cfg = {
        "name": msd_task,  # optional, it is only for your own record
        "task": "segmentation",  # optional, it is only for your own record
        "modality": "MRI",  # required
        "datalist": datalist_file,  # required
        "dataroot": dataroot,  # required
    }
    input = "./input.yaml"
    ConfigParser.export_config_file(input_cfg, input)


    runner = AutoRunner(work_dir="./work_dir", algos=('swinunetr', 'segresnet'), input=input, analyze=False, hpo=True, ensemble=False)
    num_epoch = 2
    hpo_params = {
        "maxTrialNumber": 20,
        "maxExperimentDuration": "30m",
        "num_epochs_per_validation": 1,
        "num_images_per_batch": 1,
        "num_epochs": 2,
        "num_warmup_epochs": 1,
        "training#num_epochs": 2,
        "training#num_epochs_per_validation": 1,
        "searching#num_epochs": 2,
        "searching#num_epochs_per_validation": 1,
        "searching#num_warmup_epochs": 1,
        "training#auto_scale_allowed": False,  # new
        "auto_scale_allowed": False,  # new
    }
    search_space = {"learning_rate": {"_type": "choice", "_value": [0.0001, 0.01]}}
    runner.set_num_fold(num_fold=1)
    runner.set_hpo_params(params=hpo_params)
    runner.set_nni_search_space(search_space)
    runner.run()
```

**Expected behavior**

Model checkpoint saved correctly.




Discussion:

Resolving patch:
diff --git a/monai/apps/auto3dseg/hpo_gen.py b/monai/apps/auto3dseg/hpo_gen.py
--- a/monai/apps/auto3dseg/hpo_gen.py
+++ b/monai/apps/auto3dseg/hpo_gen.py
@@ -193,7 +193,9 @@ def generate(self, output_folder: str = ".") -> None:
         self.obj_filename = os.path.join(write_path, "algo_object.pkl")
 
         if isinstance(self.algo, BundleAlgo):
-            self.algo.export_to_disk(output_folder, task_prefix + task_id, fill_with_datastats=False)
+            self.algo.export_to_disk(
+                output_folder, task_prefix + task_id, bundle_root=write_path, fill_with_datastats=False
+            )
         else:
             ConfigParser.export_config_file(self.params, write_path)
             logger.info(write_path)


Test patch:
diff --git a/tests/test_auto3dseg_hpo.py b/tests/test_auto3dseg_hpo.py
--- a/tests/test_auto3dseg_hpo.py
+++ b/tests/test_auto3dseg_hpo.py
@@ -181,7 +181,7 @@ def test_get_history(self) -> None:
 
         NNIGen().run_algo(obj_filename, self.work_dir)
         history = import_bundle_algo_history(self.work_dir, only_trained=True)
-        assert len(history) == 3
+        assert len(history) == 1
 
     def tearDown(self) -> None:
         self.test_dir.cleanup()