Repository: dynaconf/dynaconf

Issue:
[RFC] Add Lazy Read file Converter
Problem

in a `.py` settings file:

```py
SECRET_KEY = open('/etc/tower/SECRET_KEY', 'rb').read().strip()
```

When converting to YAML I want:

```yaml
secret_key: "@read /etc/tower/SECRET_KEY"
```

Evaluate the need to have multiple variants: read and read_b

Discussion:

Solution patch:
diff --git a/docs/dynamic.md b/docs/dynamic.md
index 6f3794f..b8517db 100644
--- a/docs/dynamic.md
+++ b/docs/dynamic.md
@@ -166,6 +166,92 @@ other_number = "@get number @int 12"
 The `@get` is lazily evaluated and subject to `DynaconfFormatError` in case of
 malformed expression, it is recommended to add validators.
 
+## Reading from files
+
+You can read from text files using the `@read_file` token. This token will read the file
+and return its content as a string. The file path can be absolute or relative to the current working directory.
+
+Example:
+
+```toml
+SECRET_KEY = "@read_file /path/to/secret_key"
+```
+```bash
+echo "my_secret_key" > /path/to/secret_key
+```
+
+```py
+from dynaconf import Dynaconf
+settings = Dynaconf(settings_files=["settings.toml"])
+assert settings.SECRET_KEY == "my_secret_key"
+```
+
+You can also specify a default value in case the file does not exist or isn't readable.
+
+Example:
+
+```toml
+SECRET_KEY = "@read_file /path/to/secret_key default_value"
+```
+
+If the file does not exist and there is no default value, a `FileNotFoundError` will be raised upon access.
+
+
+### Combining with other tokens
+
+You can combine the `@read_file` with other interpolation tokens like `@format` or `@jinja` and `@get` to read files with dynamic paths or content.
+
+Example:
+
+```toml
+FILENAME = "secret_key"
+DIRECTORY = "/path/to"
+
+
+SECRET_KEY = "@read_file @format {this.DIRECTORY}/{this.FILENAME}"
+SECRET_KEY = "@read_file @jinja {{ this.DIRECTORY }}/{{ this.FILENAME }}"
+
+
+COMPLETE_PATH = "@format {this.DIRECTORY}/{this.FILENAME}"
+SECRET_KEY = "@read_file @get COMPLETE_PATH"
+```
+
+### Combining with string utils
+
+If the file needs stripping or other string manipulations, you can combine the `@read_file` with the string utils tokens
+
+```toml
+SECRET_KEY = "@strip @read_file /path/to/secret_key"
+```
+
+## String utils
+
+Dynaconf provides some string utils to manipulate strings. The following utils are available:
+
+```py
+"@upper foo" == "FOO"
+"@lower FOO" == "foo"
+"@title foo bar" == "Foo Bar"
+"@capitalize foo bar" == "Foo bar"
+"@strip foo bar \n" == "foo bar"
+"@lstrip foo bar \n" == "foo bar \n"
+"@rstrip foo bar \n" == " foo bar"
+"@split foo bar" == ["foo", "bar"]
+"@casefold Foo BAR" == "foo bar"
+"@swapcase Foo BAR" == "fOO bar"
+```
+
+- `@upper`: Converts the string to uppercase.
+- `@lower`: Converts the string to lowercase.
+- `@title`: Converts the string to title case.
+- `@capitalize`: Capitalizes the first character of the string.
+- `@strip`: Removes leading and trailing whitespace from the string.
+- `@lstrip`: Removes leading whitespace from the string.
+- `@rstrip`: Removes trailing whitespace from the string.
+- `@split`: Splits the string into a list of words.
+- `@casefold`: Converts the string to casefolded form (for case-insensitive comparisons).
+- `@swapcase`: Swaps the case of all characters in the string.
+
 
 ## Access Hooks
 
diff --git a/dynaconf/utils/parse_conf.py b/dynaconf/utils/parse_conf.py
index 79ad551..152400e 100644
--- a/dynaconf/utils/parse_conf.py
+++ b/dynaconf/utils/parse_conf.py
@@ -40,6 +40,10 @@ class DynaconfParseError(Exception):
     """Error to raise when parsing @casts"""
 
 
+class DynaconfFileNotFoundError(FileNotFoundError):
+    """Error to raise when a file is not found"""
+
+
 class MetaValue:
     """A Marker to trigger specific actions on `set` and `object_merge`"""
 
@@ -244,11 +248,69 @@ def _get_formatter(value, **context):
     )
     if match := pattern.match(value.strip()):
         data = match.groupdict()
-        return context["this"].get(
-            key=data["key"],
-            default=data["default"],
-            cast=data["cast"],
-        )
+        key = data["key"]
+        default = data["default"]
+        cast = data["cast"]
+
+        params = {"key": key}
+        if default:
+            params["default"] = default
+        if cast:
+            params["cast"] = cast
+
+        if not default and key not in context["this"]:
+            raise DynaconfParseError(
+                f"Key {key} not found in settings and no default value provided."
+            )
+        return context["this"].get(**params)
+    else:
+        raise DynaconfFormatError(f"Error parsing {value} malformed syntax.")
+
+
+def _read_file_formatter(value, **context):
+    """Reads a file and returns its content.
+    takes a file path, reads the content and returns it.
+
+    @read_file /abspath/path/to/file
+    @read_file relative/path/to/file
+    @read_file file
+    @read_file file default_value
+
+    @marker FILEPATH OPTIONAL_DEFAULT_VALUE
+
+    The path can be absolute or relative to the current working directory,
+    default_value can be set to return if the file does not exist.
+    Raises error if file cannot be read.
+    Sets empty string if file is empty.
+
+    Can be composed with @get, @jinja and @format
+
+    @read_file @jinja /path/to/{{this.FILENAME}}
+    @read_file @format /path/to/{this.FILENAME}
+    @read_file @get FILENAME
+    """
+    pattern = re.compile(
+        r"(?P<path>[^ ]+)\s*"
+        r'(?P<quote>["\']?)'
+        r'\s*(?P<default>[^"\']*)\s*(?P=quote)?'
+    )
+
+    if isinstance(value, Lazy):
+        value = value(context["this"], context["env"])
+
+    if match := pattern.match(value.strip()):
+        data = match.groupdict()
+        path = data["path"]
+        default = data["default"]
+        if os.path.exists(path):
+            with open(path) as file:
+                return file.read()
+        elif default:
+            return default
+        else:
+            raise DynaconfFileNotFoundError(
+                f"File {path} does not exist and no default value provided."
+            )
     else:
         raise DynaconfFormatError(f"Error parsing {value} malformed syntax.")
 
@@ -259,6 +321,7 @@ class Formatters:
     python_formatter = BaseFormatter(str.format, "format")
     jinja_formatter = BaseFormatter(_jinja_formatter, "jinja")
     get_formatter = BaseFormatter(_get_formatter, "get")
+    read_file_formatter = BaseFormatter(_read_file_formatter, "read_file")
 
 
 class Lazy:
@@ -331,29 +394,51 @@ def evaluate_lazy_format(f):
     return evaluate
 
 
-converters = {
-    "@str": lambda value: value.set_casting(str)
-    if isinstance(value, Lazy)
-    else str(value),
-    "@int": lambda value: value.set_casting(int)
-    if isinstance(value, Lazy)
-    else int(value),
-    "@float": lambda value: value.set_casting(float)
-    if isinstance(value, Lazy)
-    else float(value),
-    "@bool": lambda value: value.set_casting(
-        lambda x: str(x).lower() in true_values
+def lazy_casting(value, cast_func):
+    """Helper function to handle Lazy casting."""
+    return (
+        value.set_casting(cast_func)
+        if isinstance(value, Lazy)
+        else cast_func(value)
     )
-    if isinstance(value, Lazy)
-    else str(value).lower() in true_values,
-    "@json": lambda value: value.set_casting(
-        lambda x: json.loads(x.replace("'", '"'))
+
+
+def json_casting(value):
+    """Helper function to handle JSON casting."""
+    return (
+        value.set_casting(lambda x: json.loads(x.replace("'", '"')))
+        if isinstance(value, Lazy)
+        else json.loads(value)
     )
-    if isinstance(value, Lazy)
-    else json.loads(value),
+
+
+def bool_casting(value):
+    """Helper function to handle boolean casting."""
+    return (
+        value.set_casting(lambda x: str(x).lower() in true_values)
+        if isinstance(value, Lazy)
+        else str(value).lower() in true_values
+    )
+
+
+def string_casting(value, str_func):
+    """Helper function to handle string transformations."""
+    return (
+        value.set_casting(str_func)
+        if isinstance(value, Lazy)
+        else str_func(str(value))
+    )
+
+
+converters = {
+    "@str": lambda value: lazy_casting(value, str),
+    "@int": lambda value: lazy_casting(value, int),
+    "@float": lambda value: lazy_casting(value, float),
+    "@bool": bool_casting,
+    "@json": json_casting,
     "@format": lambda value: Lazy(value),
     "@jinja": lambda value: Lazy(value, formatter=Formatters.jinja_formatter),
-    # Meta Values to trigger pre assignment actions
+    # Meta Values to trigger pre-assignment actions
     "@reset": Reset,  # @reset is DEPRECATED on v3.0.0
     "@del": Del,
     "@merge": Merge,
@@ -362,7 +447,21 @@ converters = {
     ),
     "@insert": Insert,
     "@get": lambda value: Lazy(value, formatter=Formatters.get_formatter),
-    # Special markers to be used as placeholders e.g: in prefilled forms
+    "@read_file": lambda value: Lazy(
+        value, formatter=Formatters.read_file_formatter
+    ),
+    # String utilities
+    "@upper": lambda value: string_casting(value, str.upper),
+    "@lower": lambda value: string_casting(value, str.lower),
+    "@title": lambda value: string_casting(value, str.title),
+    "@capitalize": lambda value: string_casting(value, str.capitalize),
+    "@strip": lambda value: string_casting(value, str.strip),
+    "@lstrip": lambda value: string_casting(value, str.lstrip),
+    "@rstrip": lambda value: string_casting(value, str.rstrip),
+    "@split": lambda value: string_casting(value, str.split),
+    "@casefold": lambda value: string_casting(value, str.casefold),
+    "@swapcase": lambda value: string_casting(value, str.swapcase),
+    # Special markers to be used as placeholders e.g., in prefilled forms
     # will always return None when evaluated
     "@note": lambda value: None,
     "@comment": lambda value: None,
@@ -452,7 +551,7 @@ def _parse_conf_data(data, tomlfy=False, box_settings=None):
     ):
         # Check combination token is used
         comb_token = re.match(
-            f"^({'|'.join(converters.keys())}) @(jinja|format)",
+            f"^({'|'.join(converters.keys())}) @(jinja|format|read_file|get)",
             data,
         )
         if comb_token:


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index ca7743a..c289084 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -13,6 +13,7 @@ from dynaconf import add_converter
 from dynaconf import default_settings
 from dynaconf import Dynaconf
 from dynaconf import DynaconfFormatError
+from dynaconf import DynaconfParseError
 from dynaconf.loaders.json_loader import DynaconfEncoder
 from dynaconf.utils import build_env_list
 from dynaconf.utils import ensure_a_list
@@ -544,6 +545,14 @@ def test_get_converter_error_when_converting(settings):
         settings.BLA
 
 
+def test_get_converter_error_when_not_found(settings):
+    """Malformed declaration errors"""
+    settings.set("BLA", "@get BATATA123")
+
+    with pytest.raises(DynaconfParseError):
+        settings.BLA
+
+
 @pytest.mark.parametrize(
     "input_expected",
     [
@@ -559,3 +568,210 @@ def test_get_converter_error_when_converting(settings):
 def test_prepare_json(input_expected):
     data, expected = input_expected
     assert prepare_json(data) == expected
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Doesn't work on windows due to backslash decoding errors",
+)
+def test_read_file_converter(tmp_path):
+    """Assert read_file converter works"""
+    # 1. Create a temp file with FOOBAR as content
+    filename = tmp_path / "SECRET_KEY"
+    create_file(filename, "FOOBAR")
+    # 2. Create a settings file with the read_file converter
+    settings_file = tmp_path / "settings.toml"
+    create_file(
+        settings_file,
+        f"""\
+        SECRET_KEY = "@read_file {filename}"
+        """,
+    )
+    # 3. Load the settings
+    settings = Dynaconf(
+        settings_file=settings_file,
+    )
+    # 4. Assert the SECRET_KEY is the content of the file
+    assert settings.SECRET_KEY == "FOOBAR"
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Doesn't work on windows due to backslash decoding errors",
+)
+def test_read_file_notfound_error(tmp_path):
+    """Assert read_file converter raises error on invalid file"""
+    # 1. Create a settings file with the read_file converter
+    settings_file = tmp_path / "settings.toml"
+    create_file(
+        settings_file,
+        f"""\
+        SECRET_KEY = "@read_file {tmp_path}/non_existent_file"
+        """,
+    )
+    # 2. Load the settings
+    with pytest.raises(FileNotFoundError):
+        settings = Dynaconf(
+            settings_file=settings_file,
+        )
+        settings.SECRET_KEY
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Doesn't work on windows due to backslash decoding errors",
+)
+def test_read_file_notfound_error_with_default(tmp_path):
+    """Assert read_file converter uses default on invalid file"""
+    # 1. Create a settings file with the read_file converter
+    settings_file = tmp_path / "settings.toml"
+    create_file(
+        settings_file,
+        f"""\
+        SECRET_KEY = "@read_file {tmp_path}/non_existent_file DEFAULT_VALUE"
+        """,
+    )
+    # 2. Load the settings
+    settings = Dynaconf(
+        settings_file=settings_file,
+    )
+    assert settings.SECRET_KEY == "DEFAULT_VALUE"
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Doesn't work on windows due to backslash decoding errors",
+)
+def test_read_file_syntax_error(tmp_path):
+    """Assert read_file converter raises error on invalid syntax"""
+    # 1. Create a settings file with the read_file converter
+    settings_file = tmp_path / "settings.toml"
+    create_file(
+        settings_file,
+        """\
+        SECRET_KEY = "@read_file "
+        """,
+    )
+    # 2. Load the settings
+    with pytest.raises(DynaconfFormatError):
+        settings = Dynaconf(
+            settings_file=settings_file,
+        )
+        settings.SECRET_KEY
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Doesn't work on windows due to backslash decoding errors",
+)
+def test_read_file_with_format(tmp_path):
+    """Assert read_file converter works with format"""
+    # 1. Create a temp file with FOOBAR as content
+    filename = tmp_path / "SECRET_KEY"
+    create_file(filename, "BATATA")
+    # 2. Create a settings file with the read_file converter
+    settings_file = tmp_path / "settings.toml"
+    create_file(
+        settings_file,
+        f"""\
+        FILENAME = "{filename}"
+        SECRET_KEY = "@read_file @format {{this.FILENAME}}"
+        """,
+    )
+    # 3. Load the settings
+    settings = Dynaconf(
+        settings_file=settings_file,
+    )
+    # 4. Assert the SECRET_KEY is the content of the file
+    assert settings.SECRET_KEY == "BATATA"
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Doesn't work on windows due to backslash decoding errors",
+)
+def test_read_file_with_jinja(tmp_path):
+    """Assert read_file converter works with format"""
+    # 1. Create a temp file with FOOBAR as content
+    filename = tmp_path / "SECRET_KEY"
+    create_file(filename, "POTATO")
+    # 2. Create a settings file with the read_file converter
+    settings_file = tmp_path / "settings.toml"
+    create_file(
+        settings_file,
+        f"""\
+        FILENAME = "{filename}"
+        SECRET_KEY = "@read_file @jinja {{{{this.FILENAME}}}}"
+        """,
+    )
+    # 3. Load the settings
+    settings = Dynaconf(
+        settings_file=settings_file,
+    )
+    # 4. Assert the SECRET_KEY is the content of the file
+    assert settings.SECRET_KEY == "POTATO"
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Doesn't work on windows due to backslash decoding errors",
+)
+def test_read_file_with_get(tmp_path):
+    """Assert read_file converter works with format"""
+    # 1. Create a temp file with FOOBAR as content
+    filename = tmp_path / "SECRET_KEY"
+    create_file(filename, "POMME")
+    # 2. Create a settings file with the read_file converter
+    settings_file = tmp_path / "settings.toml"
+    create_file(
+        settings_file,
+        f"""\
+        SECRET_FILENAME = "{filename}"
+        SECRET_KEY = "@read_file @get SECRET_FILENAME"
+        """,
+    )
+    # 3. Load the settings
+    settings = Dynaconf(
+        settings_file=settings_file,
+    )
+    # 4. Assert the SECRET_KEY is the content of the file
+    assert settings.SECRET_KEY == "POMME"
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Doesn't work on windows due to backslash decoding errors",
+)
+def test_read_file_converter_with_strip(tmp_path):
+    """Assert read_file converter works"""
+    # 1. Create a temp file with FOOBAR as content
+    filename = tmp_path / "SECRET_KEY"
+    create_file(filename, "FOOBAR \n")
+    # 2. Create a settings file with the read_file converter
+    settings_file = tmp_path / "settings.toml"
+    create_file(
+        settings_file,
+        f"""\
+        SECRET_KEY = "@strip @read_file {filename}"
+        """,
+    )
+    # 3. Load the settings
+    settings = Dynaconf(
+        settings_file=settings_file,
+    )
+    # 4. Assert the SECRET_KEY is the content of the file
+    assert settings.SECRET_KEY == "FOOBAR"
+
+
+def test_string_utils():
+    """Test string utils"""
+    assert parse_conf_data("@upper foo") == "FOO"
+    assert parse_conf_data("@lower FOO") == "foo"
+    assert parse_conf_data("@title foo bar") == "Foo Bar"
+    assert parse_conf_data("@capitalize foo bar") == "Foo bar"
+    assert parse_conf_data("@strip foo bar \n") == "foo bar"
+    assert parse_conf_data("@lstrip foo bar \n") == "foo bar \n"
+    assert parse_conf_data("@rstrip foo bar \n") == "foo bar"
+    assert parse_conf_data("@split foo bar") == ["foo", "bar"]
+    assert parse_conf_data("@casefold Foo BAR") == "foo bar"
+    assert parse_conf_data("@swapcase Foo BAR") == "fOO bar"