Repository: rochacbruno/dynaconf
Pull request #45 (2018-05-11 20:01:51)

Added support for vaultproject (hashi corp) loader


Patch:
diff --git a/.coveragerc b/.coveragerc
index 84cd899..8ef8432 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -10,4 +10,5 @@ omit =
     dynaconf/utils/functional.py
     dynaconf/utils/redis_writer.py
     dynaconf/loaders/redis_loader.py
+    dynaconf/loaders/vault_loader.py
     dynaconf/loaders/__init__.py
diff --git a/HISTORY b/HISTORY
index 25ddcf9..4c4515a 100644
--- a/HISTORY
+++ b/HISTORY
@@ -3,6 +3,11 @@ Changelog
 
 These are all the changes in Dynaconf since the first public release.
 
+0.7.3
+-----
+- Added AUTO_CAST_FOR_DYNACONF=off to disable the @type casters
+- Added loader for https://www.vaultproject.io/
+
 0.7.2
 -----
 - SETTINGS_MODULE translated to SETTINGS_MODULE_FOR_DYNACONF on compat
diff --git a/README.md b/README.md
index 32d95fb..cb08c79 100644
--- a/README.md
+++ b/README.md
@@ -383,7 +383,8 @@ You have 2 ways to use the casts.
 
 ### Casting on declaration
 
-Just start your ENV settigs with this
+Dynaconf uses the `AUTO_CAST` of envvars, just start your ENV vars names with a
+typed prefix like in the examples below.
 
 ```bash
 export DYNACONF_DEFAULT_THEME='material'
@@ -396,6 +397,10 @@ export DYNACONF_ALLOWED_EXTENSIONS='@json ["jpg", "png"]'
 
 Starting the settings values with @ will make dynaconf.settings to cast it in the time od load.
 
+There is support for `@int, @float, @bool, @json, @note`
+
+> **NOTE:** To disable the AUTO_CAST you can set a env var `export AUTO_CAST_FOR_DYNACONF=off`
+
 ### Casting on access
 
 ```bash
@@ -478,6 +483,46 @@ NAMESPACE_FOR_DYNACONF = 'MYPROGRAM'
 
 # Storing settings in databases
 
+## Using Hashicorp Vault
+
+The https://www.vaultproject.io/ is a key:value store for secrets and Dynaconf can load
+variables from a Vault secret.
+
+1. Run a vault server
+Run a Vault server installed or via docker:
+
+```bash
+docker run -e 'VAULT_DEV_ROOT_TOKEN_ID=myroot' -p 8200:8200 vault
+```
+
+2. Install support for vault in dynaconf
+
+```bash
+pip install dynaconf[vault]
+```
+
+3. In your `.env` file or in environment variables do:
+
+```bash
+VAULT_FOR_DYNACONF_ENABLED=1
+VAULT_FOR_DYNACONF_URL="http://localhost:8200"
+VAULT_FOR_DYNACONF_TOKEN="myroot"
+VAULT_FOR_DYNACONF_PATH="/secret/data/dynaconf"
+```
+
+Now you can have keys like `PASSWORD` and `TOKEN` defined in the vault and
+dynaconf will read it.
+
+To write a new secret you can use http://localhost:8200 web admin and write keys
+under the `/secret/dynaconf` secret database.
+
+You can also use the Dynaconf writer via Python terminal
+
+```python
+from dynaconf.loaders.vault_loader import write
+write({'password': '123456'})
+```
+
 ## Using REDIS
 
 
@@ -489,23 +534,14 @@ REDIS_FOR_DYNACONF = {
     'db': 0,
     'decode_responses': True
 }
-
 ```
 
 > NOTE: if running on Python 3 include `'decode_responses': True` in `REDIS_FOR_DYNACONF`
 
+Enable the loader in `.env` or as environment variable
 
-Include **redis_loader** in dynaconf LOADERS_FOR_DYNACONF
-
-> the order is the precedence
-
-```python
-
-# Loaders to read namespace based vars from diferent data stores
-LOADERS_FOR_DYNACONF = [
-    'dynaconf.loaders.env_loader',
-    'dynaconf.loaders.redis_loader'
-]
+```bash
+REDIS_FOR_DYNACONF_ENABLED=1
 ```
 
 You can now write variables direct in to a redis hash named `DYNACONF_< NAMESPACE >`
diff --git a/dynaconf/base.py b/dynaconf/base.py
index 97e83a5..42349a4 100644
--- a/dynaconf/base.py
+++ b/dynaconf/base.py
@@ -9,7 +9,12 @@ import types
 from six import string_types
 
 from dynaconf import default_settings
-from dynaconf.loaders import default_loader, settings_loader, yaml_loader
+from dynaconf.loaders import (
+    default_loader,
+    settings_loader,
+    yaml_loader,
+    enable_external_loaders
+)
 from dynaconf.transformator import TransformatorList
 from dynaconf.utils.functional import LazyObject, empty
 from dynaconf.utils.parse_conf import converters, parse_conf_data
@@ -525,6 +530,7 @@ class Settings(object):
             default_loader(self, self._defaults)
         settings_loader(self, namespace=namespace, silent=silent, key=key)
         self.load_extra_yaml(namespace, silent, key)  # DEPRECATED
+        enable_external_loaders(self)
         for loader in self.loaders:
             self.logger.debug('Loading %s', loader.__name__)
             loader.load(self, namespace, silent=silent, key=key)
diff --git a/dynaconf/constants.py b/dynaconf/constants.py
index fb7861a..ae8c2a2 100644
--- a/dynaconf/constants.py
+++ b/dynaconf/constants.py
@@ -5,3 +5,9 @@ YAML_EXTENSIONS = ('.yaml', '.yml',)
 JSON_EXTENSIONS = ('.json',)
 
 ALL_EXTENSIONS = INI_EXTENSIONS + TOML_EXTENSIONS + YAML_EXTENSIONS + JSON_EXTENSIONS  # noqa
+
+EXTERNAL_LOADERS = {
+    'ENV': 'dynaconf.loaders.env_loader',
+    'REDIS': 'dynaconf.loaders.redis_loader',
+    'VAULT': 'dynaconf.loaders.vault_loader',
+}
diff --git a/dynaconf/default_settings.py b/dynaconf/default_settings.py
index f8ba722..172d41f 100644
--- a/dynaconf/default_settings.py
+++ b/dynaconf/default_settings.py
@@ -1,9 +1,17 @@
+import os
+from dynaconf.utils.parse_conf import parse_conf_data
+
+
+def get(key, default=None):
+    return parse_conf_data(os.environ.get(key.upper(), default))
+
+
 # default proj root
 # pragma: no cover
-PROJECT_ROOT_FOR_DYNACONF = "."
+PROJECT_ROOT_FOR_DYNACONF = get('PROJECT_ROOT_FOR_DYNACONF', ".")
 
 # Default settings file
-SETTINGS_MODULE_FOR_DYNACONF = (
+default_paths = (
     'settings.py,.secrets.py,'
     'settings.yaml,settings.yml,.secrets.yaml,.secrets.yml,'
     'settings.toml,settings.tml,.secrets.toml,.secrets.tml,'
@@ -11,40 +19,67 @@ SETTINGS_MODULE_FOR_DYNACONF = (
     '.secrets.ini,.secrets.conf,.secrets.properties,'
     'settings.json,.secrets.json'
 )
+SETTINGS_MODULE_FOR_DYNACONF = get('SETTINGS_MODULE_FOR_DYNACONF',
+                                   default_paths)
 
 # Namespace for envvars
-NAMESPACE_FOR_DYNACONF = 'DYNACONF'
+NAMESPACE_FOR_DYNACONF = get('NAMESPACE_FOR_DYNACONF', 'DYNACONF')
 
 # The env var specifying settings module
-ENVVAR_FOR_DYNACONF = "DYNACONF_SETTINGS"
+ENVVAR_FOR_DYNACONF = get('ENVVAR_FOR_DYNACONF', 'DYNACONF_SETTINGS')
 
 # Default values for redis configs
-REDIS_FOR_DYNACONF = {
-    'host': 'localhost',
-    'port': 6379,
-    'db': 0
+default_redis = {
+    'host': get('REDIS_FOR_DYNACONF_HOST', 'localhost'),
+    'port': int(get('REDIS_FOR_DYNACONF_PORT', 6379)),
+    'db': int(get('REDIS_FOR_DYNACONF_DB', 0))
+}
+REDIS_FOR_DYNACONF = get('REDIS_FOR_DYNACONF', default_redis)
+REDIS_FOR_DYNACONF_ENABLED = get('REDIS_FOR_DYNACONF_ENABLED', False)
+
+# Hashicorp Vault Project
+vault_scheme = get('VAULT_FOR_DYNACONF_SCHEME', 'http')
+vault_url = get('VAULT_FOR_DYNACONF_HOST', 'localhost')
+vault_port = get('VAULT_FOR_DYNACONF_PORT', '8200')
+default_vault = {
+    'url': get('VAULT_FOR_DYNACONF_URL', '{}://{}:{}'.format(
+        vault_scheme, vault_url, vault_port)
+    ),
+    'token': get('VAULT_FOR_DYNACONF_TOKEN', None),
+    'cert': get('VAULT_FOR_DYNACONF_CERT', None),
+    'verify': get('VAULT_FOR_DYNACONF_VERIFY', None),
+    'timeout': get('VAULT_FOR_DYNACONF_TIMEOUT', None),
+    'proxies': get('VAULT_FOR_DYNACONF_PROXIES', None),
+    'allow_redirects': get('VAULT_FOR_DYNACONF_ALLOW_REDIRECTS', None),
+    'session': get('VAULT_FOR_DYNACONF_SESSION', None),
 }
+VAULT_FOR_DYNACONF = get('VAULT_FOR_DYNACONF', default_vault)
+VAULT_FOR_DYNACONF_ENABLED = get('VAULT_FOR_DYNACONF_ENABLED', False)
+VAULT_FOR_DYNACONF_PATH = get('VAULT_FOR_DYNACONF_PATH',
+                              '/secret/data/dynaconf')
 
 # Loaders to read namespace based vars from different data stores
-LOADERS_FOR_DYNACONF = [
+default_loaders = [
     'dynaconf.loaders.env_loader',
     # 'dynaconf.loaders.redis_loader'
+    # 'dynaconf.loaders.vault_loader'
 ]
+LOADERS_FOR_DYNACONF = get('LOADERS_FOR_DYNACONF', default_loaders)
 
 # Errors in loaders should be silenced?
-SILENT_ERRORS_FOR_DYNACONF = False
+SILENT_ERRORS_FOR_DYNACONF = get('SILENT_ERRORS_FOR_DYNACONF', False)
 
 # always fresh variables
-FRESH_VARS_FOR_DYNACONF = []
+FRESH_VARS_FOR_DYNACONF = get('FRESH_VARS_FOR_DYNACONF', [])
 
 # debug
-DEBUG_LEVEL_FOR_DYNACONF = 'ERROR'
+DEBUG_LEVEL_FOR_DYNACONF = get('DEBUG_LEVEL_FOR_DYNACONF', 'ERROR')
 
-YAML = None
-TOML = None
-JSON = None
-INI = None
+YAML = get('YAML', None)
+TOML = get('TOML', None)
+JSON = get('JSON', None)
+INI = get('INI', None)
 
-DOTENV_PATH_FOR_DYNACONF = None
-DOTENV_VERBOSE_FOR_DYNACONF = False
-DOTENV_OVERRIDE_FOR_DYNACONF = False
+DOTENV_PATH_FOR_DYNACONF = get('DOTENV_PATH_FOR_DYNACONF', None)
+DOTENV_VERBOSE_FOR_DYNACONF = get('DOTENV_VERBOSE_FOR_DYNACONF', False)
+DOTENV_OVERRIDE_FOR_DYNACONF = get('DOTENV_OVERRIDE_FOR_DYNACONF', False)
diff --git a/dynaconf/loaders/__init__.py b/dynaconf/loaders/__init__.py
index f400695..002ed7f 100644
--- a/dynaconf/loaders/__init__.py
+++ b/dynaconf/loaders/__init__.py
@@ -112,6 +112,18 @@ def settings_loader(obj, settings_module=None, namespace=None,
             )
 
 
+def enable_external_loaders(obj):
+    """Enable external service loaders like VAULT_ and REDIS_"""
+    for name, loader in ct.EXTERNAL_LOADERS.items():
+        enabled = getattr(
+            obj,
+            '{}_FOR_DYNACONF_ENABLED'.format(name.upper()),
+            False
+        )
+        if enabled and loader not in obj.LOADERS_FOR_DYNACONF:
+            obj.LOADERS_FOR_DYNACONF.append(loader)
+
+
 def load_from_module(obj, settings_module,
                      identifier='PY_MODULE', silent=False, key=None):
     obj.logger.debug('executing load_from_module: %s', settings_module)
diff --git a/dynaconf/loaders/redis_loader.py b/dynaconf/loaders/redis_loader.py
index 0e99fbb..42bb650 100644
--- a/dynaconf/loaders/redis_loader.py
+++ b/dynaconf/loaders/redis_loader.py
@@ -39,10 +39,9 @@ def load(obj, namespace=None, silent=True, key=None):
             if data:
                 obj.update(data, loader_identifier=IDENTIFIER)
     except Exception as e:
-        e.message = 'Unable to load config from redis (%s)' % e.message
         if silent:
             if hasattr(obj, 'logger'):
-                obj.logger.error(e.message)
+                obj.logger.error(str(e))
             return False
         raise
 
diff --git a/dynaconf/loaders/vault_loader.py b/dynaconf/loaders/vault_loader.py
new file mode 100644
index 0000000..174ba21
--- /dev/null
+++ b/dynaconf/loaders/vault_loader.py
@@ -0,0 +1,39 @@
+# docker run -e 'VAULT_DEV_ROOT_TOKEN_ID=myroot' -p 8200:8200 vault
+# pip install hvac
+
+from hvac import Client
+from dynaconf.utils.parse_conf import parse_conf_data
+
+IDENTIFIER = 'vault_loader'
+
+
+def get_client(obj):
+    client = Client(
+        **{k: v for k, v in obj.VAULT_FOR_DYNACONF.items() if v is not None}
+    )
+    assert client.is_authenticated(), "Vault authentication error"
+    return client
+
+
+def load(obj, namespace=None, silent=None, key=None):
+    client = get_client(obj)
+    data = client.read(obj.VAULT_FOR_DYNACONF_PATH).get('data', {}).get('data')
+
+    try:
+        if key:
+            value = parse_conf_data(data.get(key))
+            if value:
+                obj.set(key, value)
+        else:
+                obj.update(data, loader_identifier=IDENTIFIER)
+    except Exception as e:
+        if silent:
+            if hasattr(obj, 'logger'):
+                obj.logger.error(str(e))
+            return False
+        raise
+
+
+def writer(obj, data, lease='1h'):
+    client = get_client(obj)
+    client.write(obj.VAULT_FOR_DYNACONF_PATH, data=data, lease=lease)
diff --git a/dynaconf/utils/boxing.py b/dynaconf/utils/boxing.py
index 0d1d1a5..872203a 100644
--- a/dynaconf/utils/boxing.py
+++ b/dynaconf/utils/boxing.py
@@ -5,25 +5,25 @@ from box import Box
 
 class DynaBox(Box):
     """Specialized Box for dynaconf
-    it allows items/attrs to be founf both in upper or lower case"""
+    it allows items/attrs to be found both in upper or lower case"""
 
-    def __getattr__(self, item):
+    def __getattr__(self, item, *args, **kwargs):
         try:
-            return super(DynaBox, self).__getattr__(item)
+            return super(DynaBox, self).__getattr__(item, *args, **kwargs)
         except (AttributeError, KeyError):
             n_item = item.lower() if item.isupper() else item.upper()
-            return super(DynaBox, self).__getattr__(n_item)
+            return super(DynaBox, self).__getattr__(n_item, *args, **kwargs)
 
-    def __getitem__(self, item):
+    def __getitem__(self, item, *args, **kwargs):
         try:
-            return super(DynaBox, self).__getitem__(item)
+            return super(DynaBox, self).__getitem__(item, *args, **kwargs)
         except (AttributeError, KeyError):
             n_item = item.lower() if item.isupper() else item.upper()
-            return super(DynaBox, self).__getitem__(n_item)
+            return super(DynaBox, self).__getitem__(n_item, *args, **kwargs)
 
-    def get(self, item, default=None):
-        value = super(DynaBox, self).get(item, default)
+    def get(self, item, default=None, *args, **kwargs):
+        value = super(DynaBox, self).get(item, default, *args, **kwargs)
         if value is None or value == default:
             n_item = item.lower() if item.isupper() else item.upper()
-            value = super(DynaBox, self).get(n_item, default)
+            value = super(DynaBox, self).get(n_item, default, *args, **kwargs)
         return value
diff --git a/dynaconf/utils/parse_conf.py b/dynaconf/utils/parse_conf.py
index ff1f1b3..1fb16b6 100644
--- a/dynaconf/utils/parse_conf.py
+++ b/dynaconf/utils/parse_conf.py
@@ -1,9 +1,12 @@
+import os
 import json
 
 from six import string_types
 from dynaconf.utils.boxing import DynaBox
 
 true_values = ('t', 'true', 'enabled', '1', 'on', 'yes')
+false_values = ('f', 'false', 'disabled', '0', 'off', 'no')
+
 converters = {
     '@int': int,
     '@float': float,
@@ -32,6 +35,10 @@ def _parse_conf_data(data):
     export DYNACONF_MONGODB_SETTINGS='@json {"DB": "quokka_db"}'
     export DYNACONF_ALLOWED_EXTENSIONS='@json ["jpg", "png"]'
     """
+    cast_toggler = os.environ.get('AUTO_CAST_FOR_DYNACONF', '').lower()
+    if cast_toggler in false_values:
+        return data
+
     if data and isinstance(
             data, string_types
     ) and data.startswith(tuple(converters.keys())):
diff --git a/requirements_dev.txt b/requirements_dev.txt
index df6003b..f1b95ac 100644
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -10,6 +10,7 @@ python-box
 python-dotenv
 setuptools>=38.6.0
 configobj
+hvac
 
 # testing
 coveralls
diff --git a/setup.py b/setup.py
index f771c7d..a824082 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@ def read(*names, **kwargs):
 
 setup(
     name='dynaconf',
-    version="0.7.2",
+    version="0.7.3",
     url='https://github.com/rochacbruno/dynaconf',
     license='MIT',
     author="Bruno Rocha",
@@ -46,11 +46,12 @@ setup(
     ],
     extras_require={
         "redis": ['redis'],
+        "vault": ['hvac'],
         "yaml": ['PyYAML'],
         "toml": ['toml'],
         "ini": ['configobj'],
         "configobj": ['configobj'],
-        "all": ['redis', 'PyYAML', 'toml', 'configobj'],
+        "all": ['redis', 'PyYAML', 'toml', 'configobj', 'hvac'],
     },
     setup_requires=['setuptools>=38.6.0'],
     classifiers=[


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 6fa5da5..1427369 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -61,3 +61,11 @@ def test_find_file():
     with open(filename, 'w') as f:
         f.write("TEST=test\n")
     assert find_file(usecwd=True) == filename
+
+
+def test_disable_cast():
+    # this casts for int
+    assert parse_conf_data('@int 42') == 42
+    # now gives pure string
+    os.environ['AUTO_CAST_FOR_DYNACONF'] = 'off'
+    assert parse_conf_data('@int 42') == '@int 42'