Repository: jordantshaw/pydantic-config (language: python)

Issue:
Refactoring for pydantic v2



Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 846c273..a453346 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,14 @@ Support for Pydantic settings configuration file loading
 ## Installation
 `pip install pydantic-config`
 
+### Optional Dependencies
+
+Pydantic-Config has the following optional dependencies:
+  - yaml - `pip install pydantic-config[yaml]`
+  - toml - `pip install pydantic-config[toml]`
+
+You can install all the optional dependencies with `pip install pydantic-config[all]`
+
 ## Usage
 
 ```toml
@@ -13,7 +21,7 @@ description = "Test application description"
 ```
 
 ```python
-from pydantic_config import SettingsModel
+from pydantic_config import SettingsModel, SettingsConfig
 
 
 class Settings(SettingsModel):
@@ -21,9 +29,10 @@ class Settings(SettingsModel):
     app_name: str = None
     description: str = None
     log_level: str = 'INFO'
-
-    class Config:
-        config_file = 'config.toml'
+    
+    model_config = SettingsConfig(
+        config_file='config.toml',
+    )
 
 
 settings = Settings()
@@ -53,7 +62,7 @@ description = "Test application description"
 ```
 
 ```python
-from pydantic_config import SettingsModel
+from pydantic_config import SettingsModel, SettingsConfig
 
 
 class Settings(SettingsModel):
@@ -61,16 +70,23 @@ class Settings(SettingsModel):
     app_name: str = 'App Name'
     description: str = None
     log_level: str = 'INFO'
-
-    class Config:
-        config_file = ['config.toml', 'config.json']  # The config.json file will take priority over config.toml
-
+    
+    model_config = SettingsConfig(
+        config_file=['config.toml', 'config.json']  # The config.json file will take priority over config.toml
+    )
 
 settings = Settings()
 print(settings)
 # app_id='1' app_name='Python Application' description='Description from JSON file' log_level='WARNING'
 ```
 
+## Supported file formats
+Currently, the following file formats are supported:
+  - `.yaml` _Requires `pyyaml` package_
+  - `.toml` _Requires `toml` package_
+  - `.json`
+  - `.ini`
+
 
 ## Merging
 If your configurations have existing `list` or `dict` variables the contents will be merged by default. To disable
@@ -91,15 +107,16 @@ item2 = "value2"
 ```
 
 ```python
-from pydantic_config import SettingsModel
+from pydantic_config import SettingsModel, SettingsConfig
 
 
 class Settings(SettingsModel):
     foo: dict = {}
-
-    class Config:
-        config_file = ['config.toml', 'config2.toml']
-        config_merge: bool = True
+    
+    model_config = SettingsConfig(
+        config_file=['config.toml', 'config2.toml'],
+        config_merge= True,
+    )
 
 
 settings = Settings()
diff --git a/environment.yml b/environment.yml
index 1f3ecb4..3029696 100644
--- a/environment.yml
+++ b/environment.yml
@@ -4,7 +4,7 @@ channels:
   - defaults
 dependencies:
   - python=3.10
-  - pydantic<2.0.0, >=1.10.0
+  - pydantic-settings>=2.0.1
   - toml=0.10.2
   - python-dotenv=0.21.0
   - pyyaml=6.0
diff --git a/pyproject.toml b/pyproject.toml
index a0de24f..ad13162 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "pydantic-config"
 description = "Support for Pydantic settings configuration file loading"
-version = "0.1.2"
+version = "0.2.0"
 authors = [{name="Jordan Shaw"}]
 readme = "README.md"
 requires-python = ">=3.7"
@@ -19,24 +19,41 @@ classifiers = [
 ]
 
 dependencies = [
-    'pydantic<2.0.0, >=1.10.0',
-    'pyyaml>=6.0',
-    'python-dotenv>=0.21.0',
-    'toml>=0.10.2'
+    'pydantic-settings>=2.0.2',
+
 ]
 
 [project.optional-dependencies]
+yaml = [
+    "pyyaml>=5.1"
+]
+toml = [
+    'toml>=0.10.0'
+]
+all = [
+    'pyyaml>=5.1',
+    'toml>=0.10.0'
+]
 dev = [
-    "pytest", 'twine', 'build'
+    'pytest',
+    'twine',
+    'build',
+    'pyyaml>=5.1',
+    'python-dotenv>=0.15.0',
+    'toml>=0.10.0'
 ]
 test = [
-    "pytest"
+    'pytest',
+    'pyyaml==5.1',
+    'python-dotenv>=0.15.0',
+    'toml>=0.10.0'
 ]
 
 [project.urls]
 "Homepage" = "https://github.com/jordantshaw/pydantic-config"
 
 [tool.pytest.ini_options]
-addopts = [
-    "--import-mode=importlib",
+pythonpath = [
+  "src"
 ]
+
diff --git a/src/pydantic_config/__init__.py b/src/pydantic_config/__init__.py
index a565ae2..d77de1d 100644
--- a/src/pydantic_config/__init__.py
+++ b/src/pydantic_config/__init__.py
@@ -1,1 +1,1 @@
-from .main import SettingsModel
\ No newline at end of file
+from .main import SettingsModel, SettingsConfig
diff --git a/src/pydantic_config/main.py b/src/pydantic_config/main.py
index d1975de..4ff11a2 100644
--- a/src/pydantic_config/main.py
+++ b/src/pydantic_config/main.py
@@ -1,106 +1,145 @@
-from copy import deepcopy
+import os
 from pathlib import Path
-from typing import Dict, Any, Union, List
-
-from pydantic import BaseSettings, BaseModel
-
-from .loaders import (
-    ini_file_loader,
-    toml_file_loader,
-    yaml_file_loader,
-    json_file_loader
+from typing import Dict, Any, Union, List, Type, Tuple, Mapping
+
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings, SettingsConfigDict, PydanticBaseSettingsSource
+from pydantic_settings.sources import PydanticBaseEnvSettingsSource
+
+from .merge import deep_merge
+from .readers import (
+    ini_file_reader,
+    toml_file_reader,
+    yaml_file_reader,
+    json_file_reader
 )
 
 
-class SettingsModel (BaseSettings):
-
-    class Config:
-
-        config_files: List = []
-        config_file_encoding: str = None
-        config_merge: bool = True
-        config_merge_unique: bool = True
-
-        @classmethod
-        def customise_sources(
-                cls,
-                init_settings,
-                env_settings,
-                file_secret_settings,
-        ):
-            return (
-                init_settings,
-                env_settings,
-                file_secret_settings,
-                config_file_settings,
-            )
-
-
-def config_file_settings(settings: BaseSettings) -> Dict[str, Any]:
-    encoding = getattr(settings.__config__, 'config_file_encoding', None)
-    files = getattr(settings.__config__, 'config_file', [])
-    config_merge = getattr(settings.__config__, 'config_merge', True)
-    config_merge_unique = getattr(settings.__config__, 'config_merge_unique', True)
-
-    if isinstance(files, str):
-        files = [files]
-
-    config = {}
-    for file in files:
-        if config_merge:
-            config = _deep_merge(
-                base=config,
-                nxt=_load_config_file(file, encoding),
-                unique=config_merge_unique,
-            )
-        else:
-            config.update(_load_config_file(file, encoding))
-
-    return config
-
-
-def _deep_merge(base: dict, nxt: dict, unique: bool = True) -> dict:
-    """
-    Merges nested dictionaries.
-
-    Parameters
-    ----------
-    base: dict
-        The base dictionary to merge into
-    nxt: dict
-        The dictionary to merge into base
-    unique: bool
-        This determines the behavior when merging list values. Lists are appended together which could result
-        in duplicate values.  To avoid duplicates set this value to True.
-
-    """
-    result = deepcopy(base)
-
-    for key, value in nxt.items():
-        if isinstance(result.get(key), dict) and isinstance(value, dict):
-            result[key] = _deep_merge(result.get(key), value)
-        elif isinstance(result.get(key), list) and isinstance(value, list):
-            if unique:
-                result[key] = result.get(key) + [i for i in deepcopy(value) if i not in set(result.get(key))]
+ConfigFileType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]]
+
+
+class SettingsError(ValueError):
+    pass
+
+
+class SettingsConfig(SettingsConfigDict):
+    config_file: Union[ConfigFileType, None]
+    config_file_encoding: Union[str, None]
+    config_merge: bool
+    config_merge_unique: bool
+
+
+class SettingsModel(BaseSettings):
+
+    @classmethod
+    def settings_customise_sources(
+            cls,
+            settings_cls: Type[BaseSettings],
+            init_settings: PydanticBaseSettingsSource,
+            env_settings: PydanticBaseSettingsSource,
+            dotenv_settings: PydanticBaseSettingsSource,
+            file_secret_settings: PydanticBaseSettingsSource
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        return (
+            init_settings,
+            env_settings,
+            dotenv_settings,
+            file_secret_settings,
+            ConfigFileSettingsSource(settings_cls),
+        )
+
+
+class ConfigFileSettingsSource(PydanticBaseEnvSettingsSource):
+    """ Settings source class that loads values from one more configuration files. """
+    def __init__(
+            self,
+            settings_cls: Type[BaseSettings],
+            case_sensitive: Union[bool, None] = None,
+            config_file: Union[ConfigFileType, None] = None,
+            file_encoding: Union[str, None] = None,
+            config_merge: bool = True,
+            config_merge_unique: bool = True,
+
+    ) -> None:
+        super().__init__(settings_cls, case_sensitive)
+        self.config_file = config_file or self.config.get('config_file', None)
+        self.file_encoding = file_encoding or self.config.get('file_encoding', None)
+        self.config_merge = config_merge or self.config.get('config_merge', True)
+        self.config_merge_unique = config_merge_unique or self.config.get('config_merge_unique', True)
+        self.config_values = self._load_config_values()
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> Tuple[Any, str, bool]:
+        config_val: Any = self.config_values.get(field_name)
+        return config_val, field_name, False
+
+    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
+        if value is None:
+            value = self.config_values.get(field_name if self.case_sensitive else field_name.lower())
+            if value:
+                return value
+
+        return value
+
+    def _lowercase_dict_keys(self, d: Dict):
+        return {k.lower(): self._lowercase_dict_keys(v) if isinstance(v, dict) else v for k, v in d.items()}
+
+    def _load_config_values(self) -> Dict[str, Any]:
+        """ Gets the config values from the configuration files """
+        config_values = self._read_config_files()
+        if self.case_sensitive:
+            return config_values
+
+        return self._lowercase_dict_keys(config_values)
+
+    def _read_config_files(self) -> Dict[str, Any]:
+        """ Reads config files and merges config values if merging is enabled """
+        config_files = self.config_file
+        if config_files is None:
+            return {}
+
+        if isinstance(config_files, (str, os.PathLike)):
+            config_files = [config_files]
+
+        config = {}
+        for file in config_files:
+            file_path = Path(file).expanduser()
+            if not file_path.exists():
+                raise OSError(f"Config file `{file}` not found")
+
+            if self.config_merge:
+                config = deep_merge(
+                    base=config,
+                    nxt=self._read_config_file(file_path),
+                    unique=self.config_merge_unique,
+                )
             else:
-                result[key] = result.get(key) + deepcopy(value)
-        else:
-            result[key] = deepcopy(value)
-    return result
-
-
-def _load_config_file(file: str, encoding: str) -> Dict[str, Any]:
-    """ Loads config file with selected encoding """
-    file = Path(file)
-    if not file.exists():
-        raise OSError(f"Could not load file from provided config file: {file}")
-
-    file_loaders = {
-        '.ini': ini_file_loader,
-        '.toml': toml_file_loader,
-        '.yaml': yaml_file_loader,
-        '.json': json_file_loader,
-    }
-
-    return file_loaders.get(file.suffix)(str(file), encoding)
-
+                config.update(self._read_config_file(file_path))
+        return config
+
+    def _read_config_file(self, file: Path) -> Dict[str, Any]:
+        """ Reads single config file based on file extension """
+        file_loaders = {
+            '.ini': ini_file_reader,
+            '.toml': toml_file_reader,
+            '.yaml': yaml_file_reader,
+            '.json': json_file_reader,
+        }
+
+        return file_loaders.get(file.suffix)(str(file), self.file_encoding)
+
+    def __call__(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = super().__call__()
+
+        data_lower_keys: List[str] = []
+        if not self.case_sensitive:
+            data_lower_keys = [x.lower() for x in data.keys()]
+
+        # As `extra` config is allowed in config settings source, We have to
+        # update data with extra config values from the config files.
+        for config_name, config_value in self.config_values.items():
+            if config_value is not None:
+                if (data_lower_keys and config_name not in data_lower_keys) or (
+                        not data_lower_keys and config_name not in data):
+                    data[config_name] = config_value
+
+        return data
diff --git a/src/pydantic_config/merge.py b/src/pydantic_config/merge.py
new file mode 100644
index 0000000..2be1774
--- /dev/null
+++ b/src/pydantic_config/merge.py
@@ -0,0 +1,31 @@
+from copy import deepcopy
+
+
+def deep_merge(base: dict, nxt: dict, unique: bool = True) -> dict:
+    """
+    Merges nested dictionaries.
+
+    Parameters
+    ----------
+    base: dict
+        The base dictionary to merge into
+    nxt: dict
+        The dictionary to merge into base
+    unique: bool
+        This determines the behavior when merging list values. Lists are appended together which could result
+        in duplicate values.  To avoid duplicates set this value to True.
+
+    """
+    result = deepcopy(base)
+
+    for key, value in nxt.items():
+        if isinstance(result.get(key), dict) and isinstance(value, dict):
+            result[key] = deep_merge(result.get(key), value)
+        elif isinstance(result.get(key), list) and isinstance(value, list):
+            if unique:
+                result[key] = result.get(key) + [i for i in deepcopy(value) if i not in set(result.get(key))]
+            else:
+                result[key] = result.get(key) + deepcopy(value)
+        else:
+            result[key] = deepcopy(value)
+    return result
\ No newline at end of file
diff --git a/src/pydantic_config/loaders.py b/src/pydantic_config/readers.py
similarity index 52%
rename from src/pydantic_config/loaders.py
rename to src/pydantic_config/readers.py
index d8b7755..13f6598 100644
--- a/src/pydantic_config/loaders.py
+++ b/src/pydantic_config/readers.py
@@ -4,26 +4,26 @@
 import yaml
 
 
-def json_file_loader(file: str, encoding: str = None):
-    """ .json file type loader """
+def json_file_reader(file: str, encoding: str = None):
+    """ .json file type reader """
     with open(file, 'r', encoding=encoding) as file:
         return json.load(file)
 
 
-def ini_file_loader(file: str, encoding: str = None):
-    """ .ini file type loader """
+def ini_file_reader(file: str, encoding: str = None):
+    """ .ini file type reader """
     config = configparser.ConfigParser()
     config.read(file, encoding=encoding)
     return {k: dict(v) for k, v in config.items()}
 
 
-def toml_file_loader(file: str, encoding: str = None):
-    """ .toml file type loader """
+def toml_file_reader(file: str, encoding: str = None):
+    """ .toml file type reader """
     return toml.load(file)
 
 
-def yaml_file_loader(file: str, encoding: str = None):
-    """ .yaml file type loader """
+def yaml_file_reader(file: str, encoding: str = None):
+    """ .yaml file type reader """
     with open(file, 'r', encoding=encoding) as file:
         return yaml.safe_load(file)
 


Test patch:
diff --git a/tests/test_loaders.py b/tests/test_loaders.py
index 3ac39e9..1eb958d 100644
--- a/tests/test_loaders.py
+++ b/tests/test_loaders.py
@@ -1,21 +1,21 @@
-from pydantic_config.loaders import toml_file_loader, ini_file_loader, yaml_file_loader, json_file_loader
+from pydantic_config.readers import toml_file_reader, ini_file_reader, yaml_file_reader, json_file_reader
 
 
 def test_toml_load(config_toml_file):
-    data = toml_file_loader(config_toml_file)
+    data = toml_file_reader(config_toml_file)
     assert data == {'app': {'description': 'description from config.toml'}}
 
 
 def test_ini_load(config_ini_file):
-    data = ini_file_loader(config_ini_file)
+    data = ini_file_reader(config_ini_file)
     assert data == {'DEFAULT': {}, 'APP': {'description': 'description from config.ini'}}
 
 
 def test_yaml_load(config_yaml_file):
-    data = yaml_file_loader(config_yaml_file)
+    data = yaml_file_reader(config_yaml_file)
     assert data == {'app': {'description': 'description from config.yaml'}}
 
 
 def test_json_load(config_json_file):
-    data = json_file_loader(config_json_file)
+    data = json_file_reader(config_json_file)
     assert data == {'app': {'description': 'description from config.json'}}
diff --git a/tests/test_settings_model.py b/tests/test_settings_model.py
index 0c37b46..0102fcc 100644
--- a/tests/test_settings_model.py
+++ b/tests/test_settings_model.py
@@ -1,7 +1,7 @@
 import pytest
 from pydantic import BaseModel
 
-from pydantic_config import SettingsModel
+from pydantic_config import SettingsModel, SettingsConfig
 
 
 def test_config_file(config_toml_file):
@@ -12,11 +12,67 @@ class App(BaseModel):
     class Settings(SettingsModel):
         app: App
 
-        class Config:
-            config_file = [config_toml_file]
+        model_config = SettingsConfig(
+            config_file=[config_toml_file],
+        )
 
     settings = Settings()
-    assert settings.dict() == {'app': {'name': 'AppName', 'description': 'description from config.toml'}}
+    assert settings.model_dump() == {'app': {'name': 'AppName', 'description': 'description from config.toml'}}
+
+
+def test_case_sensitive_config(config_toml_file):
+    class App(BaseModel):
+        name: str = 'AppName'
+        description: str = None
+
+    class Settings(SettingsModel):
+        APP: App = App()
+
+        model_config = SettingsConfig(
+            case_sensitive=True,
+            extra='allow',
+            config_file=[config_toml_file],
+        )
+
+    settings = Settings()
+    assert settings.model_dump() == {
+        'APP': {'name': 'AppName', 'description': None},
+        'app': {'description': 'description from config.toml'}
+    }
+
+
+def test_case_insensitive_config(config_toml_file):
+    class App(BaseModel):
+        Name: str = 'AppName'
+        description: str = None
+
+    class Settings(SettingsModel):
+        APP: App = App()
+
+        model_config = SettingsConfig(
+            case_sensitive=False,
+            config_file=[config_toml_file],
+        )
+
+    settings = Settings()
+    assert settings.model_dump() == {'APP': {'Name': 'AppName', 'description': 'description from config.toml'}}
+
+
+def test_extra_config(config_toml_file):
+
+    class Settings(SettingsModel):
+        foo: str = 'bar'
+
+        model_config = SettingsConfig(
+            extra='allow',
+            config_file=[config_toml_file],
+        )
+
+    settings = Settings()
+    assert settings.model_dump() == {
+        'foo': 'bar',
+        'app': {'description': 'description from config.toml'}
+    }
 
 
 def test_invalid_config_file():
@@ -25,8 +81,9 @@ def test_invalid_config_file():
     class Settings(SettingsModel):
         foo: str = 'bar'
 
-        class Config:
-            config_file = [file]
+        model_config = SettingsConfig(
+            config_file=[file]
+        )
 
     with pytest.raises(OSError) as exc:
         Settings()
@@ -40,9 +97,10 @@ class App(BaseModel):
     class Settings(SettingsModel):
         app: App
 
-        class Config:
-            config_file = [config_toml_file, config_yaml_file]
+        model_config = SettingsConfig(
+            config_file=[config_toml_file, config_yaml_file]
+        )
 
     settings = Settings()
-    assert settings.dict() == {'app': {'name': 'AppName', 'description': 'description from config.yaml'}}
+    assert settings.model_dump() == {'app': {'name': 'AppName', 'description': 'description from config.yaml'}}