Repository: Asthestarsfalll/ExCore (language: python)

Issue:
PathManager
PathManager allows uesr to set some paths for logs, tesorboard , checkpoints and so on. It will automatically create some unique sub-folders in user-specified folders for each run. And will delete all sub-folders when the run fails. The structure of folder will be:
```
training_log/
└──  config_name
     ├── 2024-03-16-xx-xx-xx # default to time, can be override when running
     │   ├── config.toml  # automatically dump all loaded config into one file
     │   └── run.log
     └── tag # user can override the name of sub-folder 
         ├── config.toml
         └── run.log

```


Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index bcdb74a..1afd048 100644
--- a/README.md
+++ b/README.md
@@ -646,6 +646,77 @@ ResNet(torch.nn.ReLU())
 
 </details>
 
+### Plugins
+
+<details>
+  <summary>PathManager</summary>
+
+Manage paths in a structured manner for creating directories, if the scoped functions fail, it can automatically delete the created directories.
+
+```python
+from excore.plugins.path_manager import PathManager
+
+with PathManager(
+    base_path = "./exp",
+    sub_folders=["folder1", "folder2"],
+    config_name="config_dir",
+    instance_name="test1",
+    remove_if_fail=True,
+    sub_folder_exist_ok=False,
+    config_name_first=False,
+    return_str=True,
+) as pm:
+    folder1_path:str = pm.get("folder1")
+    folder2_path:str = pm.get("folder2")
+    do_sth(folder1_path, folder2_path)
+    train()
+```
+
+The structure will be
+
+```
+exp
+├── folder1
+│   └── config_dir
+│       └── test1
+└── folder2
+    └── config_dir
+        └── test1
+```
+
+You can also use the dataclass for a better experience:
+
+```python
+from dataclasses import dataclass
+
+from excore.plugins.path_manager import PathManager
+
+
+@dataclass
+class SubPath:
+    folder1: str = "folder1"
+    folder2: str = "folder2"
+
+sub_path = SubPath()
+
+with PathManager(
+    base_path = "./exp",
+    sub_folders=sub_path,
+    config_name="config_dir",
+    instance_name="test1",
+    remove_if_fail=True,
+    sub_folder_exist_ok=False,
+    config_name_first=False,
+    return_str=True,
+) as pm:
+    folder1_path:str = sub_path.folder1
+    folder2_path:str = sub_path.folder2
+    do_sth(folder1_path, folder2_path)
+    train()
+```
+
+</details>
+
 ### RoadMap
 
 For more features you may refer to [Roadmap of ExCore](https://github.com/users/Asthestarsfalll/projects/4)
diff --git a/excore/__init__.py b/excore/__init__.py
index fa80408..8419b86 100644
--- a/excore/__init__.py
+++ b/excore/__init__.py
@@ -16,7 +16,7 @@
     remove_logger,
 )
 from .engine.registry import Registry, load_registries
-from .utils import hub
+from . import plugins
 
 __all__ = [
     "__author__",
@@ -32,11 +32,11 @@
     "logging",
     "logger",
     "hook",
-    "hub",
     "Hook",
     "HookManager",
     "registry",
     "remove_logger",
+    "plugins",
     "Registry",
 ]
 
diff --git a/excore/utils/misc.py b/excore/_misc.py
similarity index 100%
rename from excore/utils/misc.py
rename to excore/_misc.py
diff --git a/excore/cli/_cache.py b/excore/cli/_cache.py
index 215c523..199e751 100644
--- a/excore/cli/_cache.py
+++ b/excore/cli/_cache.py
@@ -4,7 +4,7 @@
 
 from .._constants import _base_name, _cache_base_dir, _cache_dir
 from ..engine.logging import logger
-from ..utils.misc import _create_table
+from .._misc import _create_table
 from ._app import app
 
 
diff --git a/excore/cli/_registry.py b/excore/cli/_registry.py
index b8309c9..2f40fda 100644
--- a/excore/cli/_registry.py
+++ b/excore/cli/_registry.py
@@ -12,7 +12,7 @@
 from .._constants import _cache_base_dir, _workspace_cfg, _workspace_config_file
 from ..engine.logging import logger
 from ..engine.registry import Registry
-from ..utils.misc import _create_table
+from .._misc import _create_table
 from ._app import app
 
 
diff --git a/excore/config/model.py b/excore/config/model.py
index b3d0af7..58dcc06 100644
--- a/excore/config/model.py
+++ b/excore/config/model.py
@@ -7,7 +7,7 @@
 from ..engine.hook import ConfigArgumentHook
 from ..engine.logging import logger
 from ..engine.registry import Registry
-from ..utils.misc import CacheOut
+from .._misc import CacheOut
 
 __all__ = ["silent"]
 
diff --git a/excore/config/parse.py b/excore/config/parse.py
index b40ba44..fda6dd1 100644
--- a/excore/config/parse.py
+++ b/excore/config/parse.py
@@ -2,7 +2,7 @@
 
 from .._exceptions import CoreConfigParseError, ImplicitModuleParseError
 from ..engine import Registry, logger
-from ..utils.misc import _create_table
+from .._misc import _create_table
 from .model import (
     OTHER_FLAG,
     REFER_FLAG,
diff --git a/excore/engine/registry.py b/excore/engine/registry.py
index c7d4484..f479f9c 100644
--- a/excore/engine/registry.py
+++ b/excore/engine/registry.py
@@ -8,7 +8,7 @@
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
 from .._constants import _cache_dir, _registry_cache_file, _workspace_config_file
-from ..utils.misc import FileLock, _create_table
+from .._misc import FileLock, _create_table
 from .logging import logger
 
 _name_re = re.compile(r"^[A-Za-z0-9_]+$")
diff --git a/excore/utils/__init__.py b/excore/plugins/__init__.py
similarity index 100%
rename from excore/utils/__init__.py
rename to excore/plugins/__init__.py
diff --git a/excore/utils/cuda_helper.py b/excore/plugins/cuda_helper.py
similarity index 100%
rename from excore/utils/cuda_helper.py
rename to excore/plugins/cuda_helper.py
diff --git a/excore/utils/hub.py b/excore/plugins/hub.py
similarity index 100%
rename from excore/utils/hub.py
rename to excore/plugins/hub.py
diff --git a/excore/plugins/path_manager.py b/excore/plugins/path_manager.py
new file mode 100644
index 0000000..d944a3e
--- /dev/null
+++ b/excore/plugins/path_manager.py
@@ -0,0 +1,142 @@
+import shutil
+import time
+from dataclasses import is_dataclass
+from pathlib import Path
+from typing import Callable, Dict, List, Optional, Protocol, Union
+
+from ..engine.logging import logger
+
+__all__ = ["PathManager", "DataclassProtocol"]
+
+
+class DataclassProtocol(Protocol):
+    __dataclass_fields__: Dict
+    __dataclass_params__: Dict
+    __post_init__: Optional[Callable]
+
+
+class PathManager:
+    """Manage paths in a structured manner for creating directories, if the scoped functions fail,
+        it can automatically delete the created directories.
+
+    Args:
+        base_path (Path): The base path under which sub-folders will be created.
+        sub_folders (Union[List[str], DataclassProtocol]): List of sub-folders
+            or a dataclass representing sub-folders.
+        config_name (str): Name of the configuration directory.
+        instance_name (Optional[str]): Name for the current instance directory.
+            Defaults to the current timestamp.
+        remove_if_fail (bool): Flag to indicate whether to remove directories if creation fails.
+            Defaults to False.
+        sub_folder_exist_ok (bool): Flag to specify if existing sub-folders are allowed.
+            Defaults to False.
+        config_name_first (bool): Flag to determine if config name should be placed first in
+            the directory path. Defaults to False.
+        return_str (bool): Flag to indicate if string paths should be returned. Defaults to False.
+
+
+    Examples:
+        # Create a PathManager instance
+        with PathManager("/path/to/root", ["folder1", "folder2"], "config_dir") as pm:
+            folder1_path = pm.get("folder1")
+            folder2_path = pm.get("folder2")
+            do_sth(folder1_path, folder2_path)
+            train()
+
+        # Use dataclass
+        @dataclass
+        class SubPath:
+            tensorboard: str = "tensorboard"
+            ckpt: str = "checkpoints"
+            logs: str = "logs"
+        path_info = SubPath()
+        with PathManager("/path/to/root", ["folder1", "folder2"], "config_dir") as pm:
+            do_sth(path_info.logs)
+            train()
+    """
+
+    def __init__(
+        self,
+        /,
+        base_path: str,
+        sub_folders: Union[List[str], DataclassProtocol],
+        config_name: str,
+        instance_name: Optional[str] = None,
+        *,
+        remove_if_fail: bool = False,
+        sub_folder_exist_ok: bool = False,
+        config_name_first: bool = False,
+        return_str: bool = False,
+    ) -> None:
+        self.base_path = Path(base_path)
+        self._get_sub_folders(sub_folders)
+        self.config_name = Path(config_name)
+        self.instance_name = Path(instance_name or time.strftime("%Y-%m-%d-%H-%M-%S"))
+        self.remove_if_fail = remove_if_fail
+        self.sub_exist_ok = sub_folder_exist_ok
+        self.config_first = config_name_first
+        self.return_str = return_str
+        self._info = {}
+
+    def _get_sub_folders(self, sub_folders):
+        if not isinstance(sub_folders, list):
+            if not is_dataclass(sub_folders):
+                raise TypeError("Only Support dataclass or list of str")
+            self.sub_folders = [Path(f) for f in vars(sub_folders).values()]
+        else:
+            self.sub_folders = [Path(f) for f in set(sub_folders)]
+
+    def mkdir(self) -> None:
+        """
+        Create the base directory and sub-folders according to the specified configuration.
+        """
+        self.base_path.mkdir(exist_ok=True)
+        for f in self.sub_folders:
+            logger.ex(f"Create sub_folders {f}")
+            if self.config_first:
+                sub = self.base_path / self.config_name / f
+            else:
+                sub = self.base_path / f / self.config_name
+            sub = sub / self.instance_name
+            sub.mkdir(parents=True, exist_ok=self.sub_exist_ok)
+            self._info[str(f)] = sub
+
+    def get(self, name: str) -> Union[Path, str]:
+        """
+        Retrieve the path for a specific sub-folder by name.
+        """
+        path = self._info.get(name, None)
+        if self.return_str:
+            return str(path)
+        return path
+
+    def init(self) -> None:
+        """
+        Initialize the path manager by creating required directories.
+        """
+        self.mkdir()
+
+    def final(self) -> None:
+        """
+        Perform final actions like removing all directories when exiting the context.
+        """
+        self.remove_all()
+
+    def remove_all(self) -> None:
+        """
+        Remove all the previously created directories.
+        """
+
+        for f in self._info.values():
+            logger.info(f"Remove sub_folders {f}")
+            shutil.rmtree(str(f))
+
+    def __enter__(self):
+        self.init()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type is not None:
+            self.final()
+            return False
+        return True


Test patch:
diff --git a/tests/test_hub.py b/tests/test_hub.py
index a95c479..5245a62 100644
--- a/tests/test_hub.py
+++ b/tests/test_hub.py
@@ -1,6 +1,6 @@
 import pytest
 
-from excore import hub
+from excore.plugins import hub
 
 
 def test_hub_load():
diff --git a/tests/test_path_manager.py b/tests/test_path_manager.py
new file mode 100644
index 0000000..0e2cee3
--- /dev/null
+++ b/tests/test_path_manager.py
@@ -0,0 +1,135 @@
+import os
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+import pytest
+
+from excore.plugins.path_manager import PathManager
+
+BASE_PATH = os.path.abspath("./tmp/path_manager_test")
+SUB_FOLDERS = ["folder1", "folder2"]
+CONFIG_NAME = "config"
+INSTANCE_NAME = "instance_test"
+
+os.makedirs(BASE_PATH, exist_ok=True)
+
+
+class DataError:
+    def __init__(self) -> None:
+        self.a: str = ""
+
+
+@dataclass
+class Data:
+    floder1: str = "folder1"
+    floder2: str = "folder2"
+
+
+def teardown(func):
+    def _f():
+        func()
+        shutil.rmtree(BASE_PATH, ignore_errors=True)
+
+    return _f
+
+
+@teardown
+def test_init():
+    pm = PathManager(
+        base_path=BASE_PATH,
+        sub_folders=SUB_FOLDERS,
+        config_name=CONFIG_NAME,
+        instance_name=INSTANCE_NAME,
+    )
+    assert pm.base_path == Path(BASE_PATH)
+    assert pm.config_name == Path(CONFIG_NAME)
+    assert pm.instance_name == Path(INSTANCE_NAME)
+
+
+@teardown
+def test_get():
+    pm = PathManager(
+        base_path=BASE_PATH,
+        sub_folders=SUB_FOLDERS,
+        config_name=CONFIG_NAME,
+        instance_name=INSTANCE_NAME,
+        config_name_first=True,
+        return_str=True,
+    )
+    pm.mkdir()
+    path_str = pm.get("folder1")
+    assert os.path.exists(path_str)
+    expected_path = f"{BASE_PATH}/{CONFIG_NAME}/{'folder1'}/{pm.instance_name}"
+    assert path_str == expected_path
+
+
+@teardown
+def test_get_path():
+    pm = PathManager(
+        base_path=BASE_PATH,
+        sub_folders=SUB_FOLDERS,
+        config_name=CONFIG_NAME,
+        instance_name=INSTANCE_NAME,
+        config_name_first=True,
+        return_str=False,
+    )
+    pm.mkdir()
+    path = pm.get("folder1")
+    path.exists()
+    expected_path = f"{BASE_PATH}/{CONFIG_NAME}/{'folder1'}/{pm.instance_name}"
+    assert str(path) == expected_path
+
+
+@teardown
+def test_config_first_and_remove_all():
+    with PathManager(
+        base_path=BASE_PATH,
+        sub_folders=SUB_FOLDERS,
+        config_name=CONFIG_NAME,
+        instance_name=INSTANCE_NAME,
+        config_name_first=True,
+    ) as pm:
+        for sub_folder in pm.sub_folders:
+            full_path = Path(BASE_PATH) / Path(CONFIG_NAME) / sub_folder / pm.instance_name
+            assert full_path.exists()
+
+        pm.remove_all()
+        for sub_folder in pm.sub_folders:
+            full_path = Path(BASE_PATH) / Path(CONFIG_NAME) / sub_folder / pm.instance_name
+            assert not full_path.exists()
+
+
+@teardown
+def test_context_manager():
+    with pytest.raises(RuntimeError), PathManager(
+        base_path=BASE_PATH,
+        sub_folders=SUB_FOLDERS,
+        config_name=CONFIG_NAME,
+        instance_name=INSTANCE_NAME,
+    ):
+        raise RuntimeError
+
+    for sub_folder in SUB_FOLDERS:
+        full_path = Path(BASE_PATH) / Path(CONFIG_NAME) / Path(sub_folder) / Path(INSTANCE_NAME)
+        assert not full_path.exists()
+
+
+@teardown
+def test_dataclass():
+    with PathManager(
+        base_path=BASE_PATH,
+        sub_folders=Data(),
+        config_name=CONFIG_NAME,
+        instance_name=INSTANCE_NAME,
+        config_name_first=True,
+    ) as pm:
+        for sub_folder in pm.sub_folders:
+            full_path = Path(BASE_PATH) / Path(CONFIG_NAME) / sub_folder / pm.instance_name
+            assert full_path.exists()
+
+
+@teardown
+def test_error():
+    with pytest.raises(TypeError):
+        PathManager(BASE_PATH, DataError(), CONFIG_NAME, INSTANCE_NAME)