Repository: elastic/apm-agent-python

Issue:
Remote Config: Add callbacks on config change
The upcoming remote config option for [`log_level`](https://github.com/elastic/apm/pull/332) will require us to be able to execute code when it changes, so we can change the log level in the `elasticapm` logger.

This should be generalized to allow for functions to be called when a remote config option changes.


Discussion:

Resolving patch:
diff --git a/elasticapm/conf/__init__.py b/elasticapm/conf/__init__.py
--- a/elasticapm/conf/__init__.py
+++ b/elasticapm/conf/__init__.py
@@ -53,10 +53,51 @@ def __init__(self, msg, field_name):
 
 
 class _ConfigValue(object):
-    def __init__(self, dict_key, env_key=None, type=compat.text_type, validators=None, default=None, required=False):
+    """
+    Base class for configuration values
+
+    dict_key
+        String representing the key used for this config value in dict configs.
+    env_key
+        String representing the key used in environment variables for this
+        config value. If not specified, will be set to `"ELASTIC_APM_" + dict_key`.
+    type
+        Type of value stored in this config value.
+    validators
+        List of validator classes. Must be callables, which will be called with
+        a value and the dict_key for the config value. The validator either
+        returns the validated value or raises a ConfigurationError if validation
+        fails.
+    callbacks
+        List of functions which will be called when the config value is updated.
+        The callbacks must match this signature: callback(dict_key, old_value, new_value)
+    default
+        The default for this config value if not user-configured.
+    required
+        Whether this config value is required. If a default is specified,
+        this is a redundant option (except to ensure that this config value
+        is specified if a default were ever to be removed).
+
+    Note that _ConfigValues and any inheriting classes must implement __set__
+    and __get__. The calling instance will always be a _ConfigBase descendant
+    and the __set__ and __get__ calls will access `instance._values[self.dict_key]`
+    to get and set values.
+    """
+
+    def __init__(
+        self,
+        dict_key,
+        env_key=None,
+        type=compat.text_type,
+        validators=None,
+        callbacks=None,
+        default=None,
+        required=False,
+    ):
         self.type = type
         self.dict_key = dict_key
         self.validators = validators
+        self.callbacks = callbacks
         self.default = default
         self.required = required
         if env_key is None:
@@ -71,6 +112,7 @@ def __get__(self, instance, owner):
 
     def __set__(self, config_instance, value):
         value = self._validate(config_instance, value)
+        self._callback_if_changed(config_instance, value)
         config_instance._values[self.dict_key] = value
 
     def _validate(self, instance, value):
@@ -89,6 +131,28 @@ def _validate(self, instance, value):
         instance._errors.pop(self.dict_key, None)
         return value
 
+    def _callback_if_changed(self, instance, new_value):
+        """
+        If the value changed (checked against instance._values[self.dict_key]),
+        then run the callback function (if defined)
+        """
+        old_value = instance._values.get(self.dict_key, self.default)
+        if old_value != new_value:
+            self.call_callbacks(old_value, new_value)
+
+    def call_callbacks(self, old_value, new_value):
+        if not self.callbacks:
+            return
+        for callback in self.callbacks:
+            try:
+                callback(self.dict_key, old_value, new_value)
+            except Exception as e:
+                raise ConfigurationError(
+                    "Callback {} raised an exception when setting {} to {}: {}".format(
+                        callback, self.dict_key, new_value, e
+                    )
+                )
+
 
 class _ListConfigValue(_ConfigValue):
     def __init__(self, dict_key, list_separator=",", **kwargs):
@@ -102,6 +166,7 @@ def __set__(self, instance, value):
             value = list(value)
         if value:
             value = [self.type(item) for item in value]
+        self._callback_if_changed(instance, value)
         instance._values[self.dict_key] = value
 
 
@@ -118,6 +183,7 @@ def __set__(self, instance, value):
         elif not isinstance(value, dict):
             # TODO: better error handling
             value = None
+        self._callback_if_changed(instance, value)
         instance._values[self.dict_key] = value
 
 
@@ -133,6 +199,7 @@ def __set__(self, instance, value):
                 value = True
             elif value.lower() == self.false_string:
                 value = False
+        self._callback_if_changed(instance, value)
         instance._values[self.dict_key] = bool(value)
 
 
@@ -209,6 +276,11 @@ class _ConfigBase(object):
     def __init__(self, config_dict=None, env_dict=None, inline_dict=None):
         self._values = {}
         self._errors = {}
+        self._dict_key_lookup = {}
+        for config_value in self.__class__.__dict__.values():
+            if not isinstance(config_value, _ConfigValue):
+                continue
+            self._dict_key_lookup[config_value.dict_key] = config_value
         self.update(config_dict, env_dict, inline_dict)
 
     def update(self, config_dict=None, env_dict=None, inline_dict=None):
@@ -218,7 +290,7 @@ def update(self, config_dict=None, env_dict=None, inline_dict=None):
             env_dict = os.environ
         if inline_dict is None:
             inline_dict = {}
-        for field, config_value in self.__class__.__dict__.items():
+        for field, config_value in compat.iteritems(self.__class__.__dict__):
             if not isinstance(config_value, _ConfigValue):
                 continue
             new_value = self._NO_VALUE
@@ -243,6 +315,15 @@ def update(self, config_dict=None, env_dict=None, inline_dict=None):
                     config_value.dict_key
                 )
 
+    def call_callbacks(self, callbacks):
+        """
+        Call callbacks for config options matching list of tuples:
+
+        (dict_key, old_value, new_value)
+        """
+        for dict_key, old_value, new_value in callbacks:
+            self._dict_key_lookup[dict_key].call_callbacks(old_value, new_value)
+
     @property
     def values(self):
         return self._values
@@ -270,7 +351,7 @@ class Config(_ConfigBase):
     api_key = _ConfigValue("API_KEY")
     debug = _BoolConfigValue("DEBUG", default=False)
     server_url = _ConfigValue("SERVER_URL", default="http://localhost:8200", required=True)
-    server_cert = _ConfigValue("SERVER_CERT", default=None, required=False, validators=[FileIsReadableValidator()])
+    server_cert = _ConfigValue("SERVER_CERT", default=None, validators=[FileIsReadableValidator()])
     verify_server_cert = _BoolConfigValue("VERIFY_SERVER_CERT", default=True)
     include_paths = _ListConfigValue("INCLUDE_PATHS")
     exclude_paths = _ListConfigValue("EXCLUDE_PATHS", default=compat.get_default_library_patters())
@@ -418,11 +499,23 @@ def update(self, version, **config):
     def reset(self):
         """
         Reset state to the original configuration
+
+        Note that because ConfigurationValues can have callbacks, we need to
+        note any differences between the original configuration and the most
+        recent configuration and run any callbacks that might exist for those
+        values.
         """
+        callbacks = []
+        for key in compat.iterkeys(self._config.values):
+            if self._config.values[key] != self._first_config.values[key]:
+                callbacks.append((key, self._config.values[key], self._first_config.values[key]))
+
         with self._lock:
             self._version = self._first_version
             self._config = self._first_config
 
+        self._config.call_callbacks(callbacks)
+
     @property
     def changed(self):
         return self._config != self._first_config


Test patch:
diff --git a/tests/config/tests.py b/tests/config/tests.py
--- a/tests/config/tests.py
+++ b/tests/config/tests.py
@@ -43,6 +43,7 @@
     ConfigurationError,
     FileIsReadableValidator,
     RegexValidator,
+    VersionedConfig,
     _BoolConfigValue,
     _ConfigBase,
     _ConfigValue,
@@ -315,3 +316,39 @@ class MyConfig(_ConfigBase):
     c = MyConfig({"this_one_is_required": 1})
     c.update({"this_one_isnt": 0})
     assert not c.errors
+
+
+def test_callback():
+    test_var = {"foo": 0}
+
+    def set_global(dict_key, old_value, new_value):
+        # TODO make test_var `nonlocal` once we drop py2 -- it can just be a
+        # basic variable then instead of a dictionary
+        test_var[dict_key] += 1
+
+    class MyConfig(_ConfigBase):
+        foo = _ConfigValue("foo", callbacks=[set_global])
+
+    c = MyConfig({"foo": "bar"})
+    assert test_var["foo"] == 1
+    c.update({"foo": "baz"})
+    assert test_var["foo"] == 2
+
+
+def test_callback_reset():
+    test_var = {"foo": 0}
+
+    def set_global(dict_key, old_value, new_value):
+        # TODO make test_var `nonlocal` once we drop py2 -- it can just be a
+        # basic variable then instead of a dictionary
+        test_var[dict_key] += 1
+
+    class MyConfig(_ConfigBase):
+        foo = _ConfigValue("foo", callbacks=[set_global])
+
+    c = VersionedConfig(MyConfig({"foo": "bar"}), version=None)
+    assert test_var["foo"] == 1
+    c.update(version=2, **{"foo": "baz"})
+    assert test_var["foo"] == 2
+    c.reset()
+    assert test_var["foo"] == 3