Repository: caikit/caikit (language: python)

Issue:
Consolidate config
## Description

As a library extending caikit (or somebody deploying a container of that library) I would like a single file defining all of my config.

(Instead of one file for core config and one for runtime config)

## Discussion

The `caikit.core.config` and `caikit.runtime.config` packages developed separately, and now could use merging.
It would be great to have 
- only a single package deal with reading config
- only a single config yml baked into the caikit release
- only a single config file required to be written by the extension library
- the same `CONFIG_FILES` sources override from the runtime config package so that configs can be easily supplied by files at runime

The `environment` overrides (e.g. test.config.name vs. prod.config.name, _not_ environment variable overrides) could be considered for deletion, it's unclear if they are actually useful

## Acceptance Criteria

<!-- Remove any that don't apply -->

- [ ] Unit tests cover new/changed code
- [ ] Examples build against new/changed code
- [ ] READMEs are updated
- [ ] Type of [semantic version](https://semver.org/) change is identified



Discussion / hints:

Fix patch:
diff --git a/caikit/core/module_config.py b/caikit/core/module_config.py
index bfc82e5d6..d176caa6f 100644
--- a/caikit/core/module_config.py
+++ b/caikit/core/module_config.py
@@ -7,7 +7,7 @@
 
 # Local
 from . import toolkit
-from .toolkit import error_handler
+from .toolkit.errors import error_handler
 
 log = alog.use_channel("MODULE")
 error = error_handler.get(log)
diff --git a/caikit/core/toolkit/__init__.py b/caikit/core/toolkit/__init__.py
index 1fb87ee07..62042cff5 100644
--- a/caikit/core/toolkit/__init__.py
+++ b/caikit/core/toolkit/__init__.py
@@ -19,7 +19,6 @@
 # Local
 from . import compatibility, logging
 from .errors import *
-from .extension_utils import *
 from .fileio import *
 from .isa import *
 from .quality_evaluation import (
diff --git a/caikit/core/toolkit/extension_utils.py b/caikit/core/toolkit/extension_utils.py
deleted file mode 100644
index c4615e699..000000000
--- a/caikit/core/toolkit/extension_utils.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# Copyright The Caikit Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Standard
-# pylint: disable=deprecated-module
-from imp import new_module
-from importlib import import_module
-from types import ModuleType
-
-# First Party
-import alog
-
-# Local
-from .errors import error_handler
-
-log = alog.use_channel("TLKIT")
-error = error_handler.get(log)
-
-EXTENSIONS_ATTR_NAME = "extensions"
-
-
-def enable_extension_binding(lib_handle):
-    """Idempotent initialization of extensions subpackage under a provided package. Assuming
-    one doesn't exist, binds an empty "extensions" subpackage to the lib_handle. We can then
-    bind individual extensions to the library.
-
-    This should be called in the top level package initialization of the library being extended
-    to enable extension module support by attribute binding.
-
-    Args:
-        lib_handle: Module
-            caikit * library on which we would like to create an empty extensions subpackage,
-            accessible by <lib_handle>.extensions
-    """
-    error.type_check("<COR64728193E>", ModuleType, lib_handle=lib_handle)
-    # Ensure that we have a caikit * like module to start with, and that
-    # it's initialized dynamic extension package binding.
-    error.value_check(
-        "<COR10841029E>",
-        is_extension_like(lib_handle),
-        "lib_handle must be a caikit * like Module",
-    )
-    if not hasattr(lib_handle, EXTENSIONS_ATTR_NAME):
-        extension_pkg = new_module(EXTENSIONS_ATTR_NAME)
-        setattr(lib_handle, EXTENSIONS_ATTR_NAME, extension_pkg)
-
-
-def is_extension_like(lib_handle):
-    """Given a handle to a module, determine if it is caikit * like. Currently, this is checked
-    to see if <lib_handle>.lib_config.library_version is a string type.
-
-    Args:
-        lib_handle: Module
-            caikit * library on which we would like to create an empty extensions subpackage,
-            accessible by <lib_handle>.extensions
-
-    Returns:
-        bool
-            True if the library handle is caikit * like.
-    """
-    error.type_check("<COR64748191E>", ModuleType, lib_handle=lib_handle)
-    lib_version = getattr(
-        getattr(lib_handle, "lib_config", None), "library_version", None
-    )
-    return isinstance(lib_version, str)
-
-
-def bind_extensions(extension_names, lib_handle):
-    """Given an iterable of extension names and a handle to a caikit library, consider each
-    extension name. For each, import it & ensure it looks like a caikit library by checking to
-    see if it have a lib_config attribute with a defined library version.
-
-    For all caikit * like libraries, bind them to an extensions package on the library.
-
-    Ex)
-        Say we provide inputs:
-            extension_names = ["sample_module"], a caikit extension
-            lib_handle = caikit_nlp
-
-        Register sample_module onto caikit_nlp.extensions.sample_module.
-
-    Args:
-        extension_names: list | tuple | set
-            Iterable of (string) module names to be imported and bound. Objects are presumed
-            to be unique & will be cast to a set for consideration. If an existing extension
-            of the same name is already registered, the extension will be skipped.
-        lib_handle: Module
-            caikit * library on which we would like to create an empty extensions subpackage,
-            accessible by <lib_handle>.extensions
-    """
-    error.type_check(
-        "<COR64140091E>", set, list, tuple, extension_names=extension_names
-    )
-    error.type_check_all("<COR34341191E>", str, extension_names=extension_names)
-    error.type_check("<COR64748191E>", ModuleType, lib_handle=lib_handle)
-    error.value_check(
-        "<COR12831731E>",
-        hasattr(lib_handle, EXTENSIONS_ATTR_NAME),
-        "Library {} has not enabled extension binding",
-        lib_handle.__name__,
-    )
-
-    ext_subpkg = getattr(lib_handle, EXTENSIONS_ATTR_NAME)
-    for ext_name in set(extension_names):
-        # Skip any extension package name that's already registered
-        if hasattr(ext_subpkg, ext_name):
-            log.debug("Extension [{}] is already bound".format(ext_name))
-            continue
-        # Dynamically import each extension module; skip if we can't import
-        try:
-            ext_lib = import_module(ext_name)
-        except ImportError:
-            log.warning(
-                "<COR81130101W>",
-                "Module [{}] is not importable and could not be bound".format(ext_name),
-            )
-            continue
-        # If this is a caikit * like library, bind it to the extensions property
-        if is_extension_like(ext_lib):
-            setattr(ext_subpkg, ext_name, ext_lib)
-            log.debug("Bound extension [{}] successfully".format(ext_name))
-        else:
-            log.warning(
-                "<COR81130101W>",
-                "Skipping binding for [{}]; module is not caikit Library like".format(
-                    ext_name
-                ),
-            )


Test patch:
diff --git a/tests/core/toolkit/test_extension_utils.py b/tests/core/toolkit/test_extension_utils.py
deleted file mode 100644
index 2e973b140..000000000
--- a/tests/core/toolkit/test_extension_utils.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# Copyright The Caikit Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Standard
-from types import ModuleType
-import os
-
-# Local
-from caikit.core.toolkit import extension_utils
-
-# Unit Test Infrastructure
-from tests.base import TestCaseBase
-import caikit.core
-
-
-class TestExtensionUtils(TestCaseBase):
-    @classmethod
-    def setUpClass(cls):
-        cls.caikit_lib = caikit.core
-        cls.extension_like_lib = caikit.core
-        cls.nonextension_like_lib = os
-        cls.extension_like_lib_name = cls.extension_like_lib.__name__
-        cls.nonextension_like_lib_name = cls.nonextension_like_lib.__name__
-
-    def test_is_extension_like_happy(self):
-        """Ensure caikit core looks like a caikit * library"""
-        self.assertTrue(extension_utils.is_extension_like(self.extension_like_lib))
-
-    def test_is_extension_like_sad(self):
-        """Ensure something like os does NOT look like a caikit * library"""
-        self.assertFalse(extension_utils.is_extension_like(self.nonextension_like_lib))
-
-    def test_enable_extension_binding(self):
-        """Ensure that enabling extension binding creates an extensions subpackage"""
-        self.assertFalse(hasattr(self.caikit_lib, extension_utils.EXTENSIONS_ATTR_NAME))
-        extension_utils.enable_extension_binding(self.caikit_lib)
-        self.assertTrue(hasattr(self.caikit_lib, extension_utils.EXTENSIONS_ATTR_NAME))
-        self.assertIsInstance(self.caikit_lib, ModuleType)
-
-    def test_enable_extension_binding_non_extension_like(self):
-        """Ensure that we need to pass a ModuleType to extension binding enablement"""
-        with self.assertRaises(TypeError):
-            extension_utils.enable_extension_binding(42)
-
-    def test_bind_extension_like(self):
-        """Ensure we can bind caikit / ext like libs, and skip other modules"""
-        extension_utils.enable_extension_binding(self.caikit_lib)
-        extension_utils.bind_extensions(
-            [self.extension_like_lib_name, self.nonextension_like_lib_name],
-            self.caikit_lib,
-        )
-        # Grab our extensions that have been bound
-        exts = getattr(self.caikit_lib, extension_utils.EXTENSIONS_ATTR_NAME)
-        # Ensure only extension like libs are bound
-        self.assertTrue(hasattr(exts, self.extension_like_lib_name))
-        self.assertFalse(hasattr(exts, self.nonextension_like_lib_name))
-        # Ensure that the bound lib is a ModuleType object
-        bound_lib = getattr(exts, self.extension_like_lib_name)
-        self.assertIsInstance(bound_lib, ModuleType)
-
-    def test_bind_requires_enablement(self):
-        """Ensure that we can't bind prior to initializing the extensions subpackage"""
-        with self.assertRaises(ValueError):
-            extension_utils.bind_extensions(
-                [self.extension_like_lib_name], self.caikit_lib
-            )
-
-    def test_bind_invalid_extensions(self):
-        """Ensure we type check our lib extensions properly"""
-        extension_utils.enable_extension_binding(self.caikit_lib)
-        # Ensure that we catch bad types for for the arg itself
-        with self.assertRaises(TypeError):
-            extension_utils.bind_extensions([42], self.caikit_lib)
-        # And also within the nested type of the iterable
-        with self.assertRaises(TypeError):
-            extension_utils.bind_extensions(42, self.caikit_lib)
-
-    def test_bind_invalid_lib_handle_type(self):
-        """Ensure we type check our lib handle properly"""
-        extension_utils.enable_extension_binding(self.caikit_lib)
-        with self.assertRaises(TypeError):
-            extension_utils.bind_extensions([self.extension_like_lib_name], 42)
-
-    def tearDown(self):
-        # Disable extension binding on the caikit library between tests
-        if hasattr(self.caikit_lib, extension_utils.EXTENSIONS_ATTR_NAME):
-            delattr(self.caikit_lib, extension_utils.EXTENSIONS_ATTR_NAME)