Repository: intel/dffml

Issue:
config: Directory ConfigLoader
It will look in the directory and start building a `dict`, each folder becomes a key in the dict.

Go good way to test this is to export the `service/http/tests/dataflow.py` `HELLO_BLANK_DATAFLOW` and then split out the subflow for `remap_to_response`:

### Old Version

**hello_blank.yaml**

```yaml
configs:
  hello_blank:
    formatting: Hello {}
  remap_to_response:
    dataflow:
      configs: {}
      flow:
        get_formatted_message:
          inputs:
            spec:
            - seed
      operations:
        get_formatted_message:
          inputs:
            spec:
              name: get_single_spec
              primitive: array
          name: get_single
          outputs:
            output:
              name: get_single_output
              primitive: map
          stage: output
      seed:
      - definition:
          name: get_single_spec
          primitive: array
        value:
        - message
definitions:
  format_data:
    name: format_data
    primitive: string
  message:
    name: message
    primitive: string
  remap_output:
    name: remap_output
    primitive: map
  remap_spec:
    name: remap_spec
    primitive: map
flow:
  hello_blank:
    inputs:
      data:
      - seed
  remap_to_response:
    inputs:
      spec:
      - seed
linked: true
operations:
  hello_blank:
    inputs:
      data: format_data
    name: formatter
    outputs:
      string: message
    stage: processing
  remap_to_response:
    inputs:
      spec: remap_spec
    name: remap
    outputs:
      response: remap_output
    stage: output
seed:
- definition: remap_spec
  value:
    response:
    - message
```

### New Version

**hello_blank.yaml**

```yaml
configs:
  hello_blank:
    formatting: Hello {}
definitions:
  format_data:
    name: format_data
    primitive: string
  message:
    name: message
    primitive: string
  remap_output:
    name: remap_output
    primitive: map
  remap_spec:
    name: remap_spec
    primitive: map
flow:
  hello_blank:
    inputs:
      data:
      - seed
  remap_to_response:
    inputs:
      spec:
      - seed
linked: true
operations:
  hello_blank:
    inputs:
      data: format_data
    name: formatter
    outputs:
      string: message
    stage: processing
  remap_to_response:
    inputs:
      spec: remap_spec
    name: remap
    outputs:
      response: remap_output
    stage: output
seed:
- definition: remap_spec
  value:
    response:
    - message
```

**configs/remap_to_response/dataflow.yaml**

```yaml
configs: {}
flow:
  get_formatted_message:
    inputs:
      spec:
      - seed
operations:
  get_formatted_message:
    inputs:
      spec:
        name: get_single_spec
        primitive: array
    name: get_single
    outputs:
      output:
        name: get_single_output
        primitive: map
    stage: output
seed:
- definition:
    name: get_single_spec
    primitive: array
  value:
  - message
```


Discussion:

Resolving patch:
diff --git a/dffml/config/config.py b/dffml/config/config.py
--- a/dffml/config/config.py
+++ b/dffml/config/config.py
@@ -1,14 +1,18 @@
 import abc
+import os
 import pathlib
+import json
 import contextlib
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, Union
 
 from ..util.entrypoint import base_entry_point
+from ..util.asynchelper import AsyncContextManagerList
 from ..base import (
     BaseConfig,
     BaseDataFlowFacilitatorObjectContext,
     BaseDataFlowFacilitatorObject,
 )
+from dffml.util.data import explore_directories, nested_apply
 
 
 class BaseConfigLoaderContext(BaseDataFlowFacilitatorObjectContext):
@@ -37,7 +41,7 @@ def __call__(self) -> BaseConfigLoaderContext:
         return self.CONTEXT(self)
 
     @classmethod
-    async def load_file(
+    async def load_single_file(
         cls,
         parsers: Dict[str, "BaseConfigLoader"],
         exit_stack: contextlib.AsyncExitStack,
@@ -71,3 +75,62 @@ async def load_file(
         config_path = tuple(config_path)
         # Load the file
         return config_path, await parsers[filetype].loadb(path.read_bytes())
+
+    @classmethod
+    async def load_file(
+        cls,
+        parsers: Dict[str, "BaseConfigLoader"],
+        exit_stack: contextlib.AsyncExitStack,
+        path: pathlib.Path,
+        *,
+        base_dir: Optional[pathlib.Path] = None,
+    ) -> Dict:
+        async def _get_config(temp_filepath):
+            if not isinstance(temp_filepath, pathlib.Path):
+                temp_filepath = pathlib.Path(temp_filepath)
+            config_path, loaded = await BaseConfigLoader.load_single_file(
+                parsers, exit_stack, temp_filepath, base_dir=base_dir
+            )
+            return config_path, loaded
+
+        async def _get_config_aux(temp_filepath):
+            _, loaded = await _get_config(temp_filepath)
+            return loaded
+
+        if len(path.suffixes) >= 2 and path.suffixes[-2] == ".dirconf":
+            dir_name = path.parts[-1].split(".")[0]
+            dir_path = os.path.join(*(path.parts[:-1] + (dir_name,)))
+
+            temp_conf_dict = {dir_name: dir_path}
+            config_path, conf_dict = await _get_config(path)
+            explored = explore_directories(temp_conf_dict)
+            explored = await nested_apply(explored, _get_config_aux)
+            conf_dict.update(explored[dir_name])
+        else:
+            config_path, conf_dict = await _get_config(path)
+        return config_path, conf_dict
+
+
+class ConfigLoaders(AsyncContextManagerList):
+    """
+    A class similar to Sources in that it will hold all the config
+    loaders that get loaded as needed (based on filetype).
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.parsers: Dict[str, "BaseConfigLoader"] = {}
+
+    async def load_file(
+        self,
+        filepath: Union[str, pathlib.Path],
+        *,
+        base_dir: Optional[pathlib.Path] = None,
+    ) -> Dict:
+
+        if not isinstance(filepath, pathlib.Path):
+            filepath = pathlib.Path(filepath)
+        conf_dict = await BaseConfigLoader.load_file(
+            self.parsers, self.async_exit_stack, filepath, base_dir=base_dir
+        )
+        return conf_dict
diff --git a/dffml/util/asynchelper.py b/dffml/util/asynchelper.py
--- a/dffml/util/asynchelper.py
+++ b/dffml/util/asynchelper.py
@@ -74,6 +74,10 @@ async def __aenter__(self):
     async def __aexit__(self, exc_type, exc_value, traceback):
         await self.__stack.aclose()
 
+    @property
+    def async_exit_stack(self):
+        return self.__stack
+
 
 async def concurrently(
     work: Dict[asyncio.Task, Any],
diff --git a/dffml/util/data.py b/dffml/util/data.py
--- a/dffml/util/data.py
+++ b/dffml/util/data.py
@@ -1,9 +1,13 @@
 """
 Various helper functions for manipulating python data structures and values
 """
+
+import inspect
 import pydoc
 import inspect
 from functools import wraps
+import pathlib
+from typing import Callable
 
 
 def merge(one, two):
@@ -123,3 +127,61 @@ def export_dict(**kwargs):
         elif isinstance(kwargs[key], list):
             kwargs[key] = export_list(kwargs[key])
     return kwargs
+
+
+def explore_directories(path_dict: dict):
+    """
+    Recursively explores any path binded to a key in `path_dict`
+
+    eg:
+        | root
+            |deadbeef
+                |file1.txt
+                |colosseum
+                    | battle.rst
+            |face
+                |file2.jpg
+
+        >>>explore_directories(path_dict ={
+                "hello":"there",
+                "deadbeef" : path_to_deadbeef
+                })
+
+            {
+                "hello":"there",
+                "deadbeef":{
+                    "file1" : path_to_file1.txt
+                    "colosseum":{
+                        "battle":path_to_battle.rst
+                    }
+                }
+                "face":{
+                    "file2":path_to_file2.txt
+                }
+            }
+    """
+    for key, val in path_dict.items():
+        t_path = pathlib.Path(val)
+        if t_path.is_dir():
+            temp_path_dict = {}
+            for _path in pathlib.Path(val).glob("*"):
+                t_path = pathlib.Path(_path)
+                temp_path_dict[t_path.stem] = _path
+            explore_directories(temp_path_dict)
+            path_dict[key] = temp_path_dict
+    return path_dict
+
+
+async def nested_apply(target: dict, func: Callable):
+    """
+    Applies `func` recursively to all non dict types in `target`
+    """
+    for key, val in target.items():
+        if isinstance(val, dict):
+            target[key] = await nested_apply(val, func)
+        else:
+            if inspect.iscoroutinefunction(func):
+                target[key] = await func(val)
+            else:
+                target[key] = func(val)
+    return target


Test patch:
diff --git a/tests/config/test_config_loader.py b/tests/config/test_config_loader.py
new file mode 100644
--- /dev/null
+++ b/tests/config/test_config_loader.py
@@ -0,0 +1,60 @@
+import os
+import json
+import pathlib
+import tempfile
+from dffml.util.asynctestcase import AsyncTestCase
+from dffml.config.config import ConfigLoaders
+from dffml.util.data import explore_directories, nested_apply
+
+
+class TestConfigLoader(AsyncTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.test_dir = tempfile.TemporaryDirectory()
+
+        cls.config_folder_name = "the_config_name"
+        cls.config_folder_path = os.path.join(
+            cls.test_dir.name, cls.config_folder_name
+        )
+        os.mkdir(cls.config_folder_path)
+        temp_path = cls.config_folder_path
+        pathlib.Path(
+            cls.test_dir.name, cls.config_folder_name + ".dirconf.json"
+        ).write_text(json.dumps({"hello": "there"}))
+        pathlib.Path(temp_path, "deadbeef.json").write_text(
+            json.dumps({"massive": "hax"})
+        )
+        os.mkdir(os.path.join(temp_path, "feed"))
+        pathlib.Path(temp_path, "feed", "face.json").write_text(
+            json.dumps({"so": "secure"})
+        )
+
+    @classmethod
+    def tearDown(cls):
+        cls.test_dir.cleanup()
+
+    def setUp(self):
+        self.config_loader = ConfigLoaders()
+
+    async def test_run(self):
+        expected = {
+            "hello": "there",
+            "deadbeef": {"massive": "hax"},
+            "feed": {"face": {"so": "secure"}},
+        }
+        temp_path = self.config_folder_path
+        async with self.config_loader as cfgl:
+            _, conf_dict = await cfgl.load_file(
+                os.path.join(
+                    self.test_dir.name, "the_config_name.dirconf.json"
+                )
+            )
+            self.assertEqual(expected, conf_dict)
+
+        expected = {"massive": "hax"}
+        temp_path = self.config_folder_path
+        async with self.config_loader as cfgl:
+            _, conf_dict = await cfgl.load_file(
+                os.path.join(temp_path, "deadbeef.json")
+            )
+            self.assertEqual(expected, conf_dict)