Repository: kangasta/fdbk

Issue:
Data tools should support passing through metadata

Discussion:

Patch:
diff --git a/fdbk/_db_connection.py b/fdbk/_db_connection.py
index e1f665f..7176026 100644
--- a/fdbk/_db_connection.py
+++ b/fdbk/_db_connection.py
@@ -1,4 +1,4 @@
-from fdbk.data_tools import functions as data_functions, post_process
+from fdbk.data_tools import post_process, run_data_tools
 
 
 class DBConnection:
@@ -103,37 +103,6 @@ class DBConnection:
         '''
         return self.get_data(topic_id)[-1]
 
-    def _run_data_tools(self, topic_d, data):
-        results = []
-        warnings = []
-
-        for instruction in topic_d['data_tools']:
-            if instruction["method"] not in data_functions:
-                warnings.append("The requested method '" +
-                                instruction["method"] + "' is not supported.")
-                continue
-            if instruction["field"] not in topic_d["fields"]:
-                warnings.append("The requested field '" +
-                                instruction["field"] + "' is undefined.")
-                continue
-
-            result = data_functions[instruction["method"]](
-                data, instruction["field"]
-            )
-            if result is not None:
-                result["payload"]["topic_name"] = topic_d["name"]
-                try:
-                    result["payload"]["unit"] = next(
-                        i["unit"] for i in topic_d["units"] if (
-                            i["field"] == instruction["field"])
-                    )
-                except StopIteration:
-                    pass
-
-            results.append(result)
-
-        return (results, warnings,)
-
     def get_summary(self, topic_id, since=None, until=None, limit=None):
         '''Get summary of the topic data
 
@@ -161,7 +130,7 @@ class DBConnection:
             "warnings": []
         }
 
-        results, warnings = self._run_data_tools(topic_d, data_d)
+        results, warnings = run_data_tools(topic_d, data_d)
         summary_d["statistics"] = post_process(results)
         summary_d["warnings"].extend(warnings)
 
@@ -192,7 +161,7 @@ class DBConnection:
             result_d["topic_names"].append(topic_d["name"])
             result_d["fields"].extend(topic_d["fields"])
 
-            results, warnings = self._run_data_tools(topic_d, data_d)
+            results, warnings = run_data_tools(topic_d, data_d)
             result_d["statistics"].extend(results)
             result_d["warnings"].extend(warnings)
 
diff --git a/fdbk/data_tools/__init__.py b/fdbk/data_tools/__init__.py
index 59a61fe..a60d0f6 100644
--- a/fdbk/data_tools/__init__.py
+++ b/fdbk/data_tools/__init__.py
@@ -4,8 +4,5 @@ Functions to ease the simple data analysis done by the DBConnection.
 
 '''
 
-from ._chart_funcs import *
+from .functions import *
 from ._utils import *
-from ._value_funcs import *
-
-functions = {**VALUE_FUNCS, **CHART_FUNCS}  # pylint: disable=invalid-name
diff --git a/fdbk/data_tools/_utils.py b/fdbk/data_tools/_utils.py
index 4eebc1b..8b9e135 100644
--- a/fdbk/data_tools/_utils.py
+++ b/fdbk/data_tools/_utils.py
@@ -1,13 +1,7 @@
-def _statistics_dict(type_, **kwargs):
-    return dict(type=type_, payload=kwargs)
+from fdbk.utils.messages import method_not_supported, field_is_undefined
 
-
-def _chart_dict(**kwargs):
-    return _statistics_dict("chart", **kwargs)
-
-
-def _value_dict(**kwargs):
-    return _statistics_dict("value", **kwargs)
+from .functions import functions as data_functions
+from .functions.utils import chart_dict
 
 
 def _create_chart(type_, field):
@@ -25,6 +19,38 @@ def _visualization_to_dataset(visualization):
     )
 
 
+def run_data_tools(topic_d, data):
+    results = []
+    warnings = []
+
+    for instruction in topic_d['data_tools']:
+        if instruction["method"] not in data_functions:
+            warnings.append(method_not_supported(instruction["method"]))
+            continue
+        if instruction["field"] not in topic_d["fields"]:
+            warnings.append(field_is_undefined(instruction["field"]))
+            continue
+
+        result = data_functions[instruction["method"]](
+            data, instruction["field"]
+        )
+        if result is not None:
+            result["payload"]["topic_name"] = topic_d["name"]
+            if instruction.get("metadata"):
+                result["metadata"] = instruction.get("metadata")
+            try:
+                result["payload"]["unit"] = next(
+                    i["unit"] for i in topic_d["units"] if (
+                        i["field"] == instruction["field"])
+                )
+            except StopIteration:
+                pass
+
+        results.append(result)
+
+    return (results, warnings,)
+
+
 def process_charts(statistics):
     charts = {}
     other = []
@@ -36,6 +62,7 @@ def process_charts(statistics):
             other.append(i)
             continue
 
+        metadata = i.get("metadata")
         i = i.get('payload')
 
         field = i.get('field')
@@ -48,8 +75,11 @@ def process_charts(statistics):
         charts[key]['data']['labels'] = (
             list(set(charts[key]['data']['labels'] + i.get("labels", []))))
         charts[key]["data"]['datasets'].append(_visualization_to_dataset(i))
+        if metadata:
+            charts[key]["metadata"] = {
+                **charts[key].get("metadata", {}), **metadata}
 
-    return list(_chart_dict(**chart) for chart in charts.values()) + other
+    return list(chart_dict(**chart) for chart in charts.values()) + other
 
 
 def post_process(statistics):
diff --git a/fdbk/data_tools/functions/__init__.py b/fdbk/data_tools/functions/__init__.py
new file mode 100644
index 0000000..0c500c8
--- /dev/null
+++ b/fdbk/data_tools/functions/__init__.py
@@ -0,0 +1,4 @@
+from ._chart_funcs import *
+from ._value_funcs import *
+
+functions = {**VALUE_FUNCS, **CHART_FUNCS}  # pylint: disable=invalid-name
diff --git a/fdbk/data_tools/_chart_funcs.py b/fdbk/data_tools/functions/_chart_funcs.py
similarity index 83%
rename from fdbk/data_tools/_chart_funcs.py
rename to fdbk/data_tools/functions/_chart_funcs.py
index 9062ab1..c78a18f 100644
--- a/fdbk/data_tools/_chart_funcs.py
+++ b/fdbk/data_tools/functions/_chart_funcs.py
@@ -1,4 +1,4 @@
-from ._utils import _chart_dict
+from .utils import chart_dict
 
 
 def doughnut(data, field, type_="doughnut"):
@@ -7,7 +7,7 @@ def doughnut(data, field, type_="doughnut"):
     if not field_data:
         return None
 
-    return _chart_dict(
+    return chart_dict(
         type=type_,
         field=field,
         data=[field_data.count(label) for label in set(field_data)],
@@ -19,7 +19,7 @@ def line(data, field):
     if not data:
         return None
 
-    return _chart_dict(
+    return chart_dict(
         type="line",
         field=field,
         data=[{"x": a["timestamp"], "y": a[field]} for a in data],
diff --git a/fdbk/data_tools/_value_funcs.py b/fdbk/data_tools/functions/_value_funcs.py
similarity index 87%
rename from fdbk/data_tools/_value_funcs.py
rename to fdbk/data_tools/functions/_value_funcs.py
index 813c04d..8440905 100644
--- a/fdbk/data_tools/_value_funcs.py
+++ b/fdbk/data_tools/functions/_value_funcs.py
@@ -1,6 +1,6 @@
 from numbers import Number
 
-from ._utils import _value_dict
+from .utils import value_dict
 
 
 def average(data, field):
@@ -9,7 +9,7 @@ def average(data, field):
     if not filtered_data:
         return None
 
-    return _value_dict(
+    return value_dict(
         type="average",
         field=field,
         value=sum(i / float(len(filtered_data)) for i in filtered_data)
@@ -20,7 +20,7 @@ def latest(data, field):
     if not data:
         return None
 
-    return _value_dict(
+    return value_dict(
         type="latest",
         field=field,
         value=data[-1][field] if data else None
@@ -36,7 +36,7 @@ def last(truthy_or_falsy, data, field):
     type_str = "last_truthy" if truthy_or_falsy else "last_falsy"
     value = filtered_data[-1]["timestamp"]
 
-    return _value_dict(
+    return value_dict(
         type=type_str,
         field=field,
         value=value
diff --git a/fdbk/data_tools/functions/utils.py b/fdbk/data_tools/functions/utils.py
new file mode 100644
index 0000000..c14b296
--- /dev/null
+++ b/fdbk/data_tools/functions/utils.py
@@ -0,0 +1,14 @@
+def _statistics_dict(type_, metadata=None, **kwargs):
+    statistic_d = dict(type=type_, payload=kwargs)
+    if metadata:
+        statistic_d["metadata"] = metadata
+
+    return statistic_d
+
+
+def chart_dict(**kwargs):
+    return _statistics_dict("chart", **kwargs)
+
+
+def value_dict(metadata=None, **kwargs):
+    return _statistics_dict("value", **kwargs)
diff --git a/fdbk/schemas/topic-in.json b/fdbk/schemas/topic-in.json
index 95c1e29..dc9037b 100644
--- a/fdbk/schemas/topic-in.json
+++ b/fdbk/schemas/topic-in.json
@@ -13,7 +13,8 @@
       "type": "object",
       "properties": {
         "field": { "type": "string"},
-        "method": { "type": "string"}
+        "method": { "type": "string"},
+        "metadata": {"type": "object"}
       },
       "required": ["field", "method"]
     }
diff --git a/fdbk/utils/messages.py b/fdbk/utils/messages.py
index 9c3ae4a..36f77c6 100644
--- a/fdbk/utils/messages.py
+++ b/fdbk/utils/messages.py
@@ -1,2 +1,10 @@
 def topic_not_found(id_):
-    return f"Topic ID '{id_}' not found from database"
+    return f'Topic ID "{id_}" not found from database'
+
+
+def method_not_supported(method):
+    return f'The requested method "{method}" is not supported.'
+
+
+def field_is_undefined(field):
+    return f'The requested field "{field}" is undefined.'


Test patch:
diff --git a/tst/test_data_tools.py b/tst/test_data_tools.py
index 385f500..ca9a2ff 100644
--- a/tst/test_data_tools.py
+++ b/tst/test_data_tools.py
@@ -3,7 +3,7 @@ from unittest.mock import Mock, patch
 
 from fdbk.data_tools import functions
 
-class DBConnectionTest(TestCase):
+class DataToolsTest(TestCase):
     def test_summary_funcs_return_none_on_empty_data(self):
         for fn in functions.values():
             self.assertIsNone(fn([], 'field'))
diff --git a/tst/test_db_connection.py b/tst/test_db_connection.py
index a4a8057..5a78477 100644
--- a/tst/test_db_connection.py
+++ b/tst/test_db_connection.py
@@ -136,8 +136,8 @@ class DBConnectionTest(TestCase):
         C.add_data(topic_id, {"number": 3})
         summary = C.get_summary(topic_id)
         self.assertEqual(summary["warnings"], [
-            "The requested method 'cow' is not supported.",
-            "The requested method 'moose' is not supported."
+            'The requested method "cow" is not supported.',
+            'The requested method "moose" is not supported.'
         ])
 
     def test_get_latest_handles_empty_list(self):
@@ -173,7 +173,7 @@ class DBConnectionTest(TestCase):
         self.assertEqual(summary["statistics"][0]["payload"]["value"], data[3]["timestamp"])
         self.assertEqual(summary["statistics"][1]["payload"]["value"], data[2]["timestamp"])
 
-    def _test_run_data_tools_for_many(self, fn, data_tools, statistics_len):
+    def _test_run_data_tools_for_many(self, fn, data_tools):
         topic_ids = []
         C = DictConnection()
         for i in range(3):
@@ -194,15 +194,21 @@ class DBConnectionTest(TestCase):
         self.assertEqual(result["topic_names"], ["topic_0", "topic_1", "topic_2"])
         self.assertEqual(result["fields"], ["number"])
 
-        self.assertEqual(len(result["statistics"]), statistics_len)
+        return result
 
     def test_get_comparison_produces_comparison(self):
-        data_tools = [{"field":"number", "method":"doughnut"}]
-        return self._test_run_data_tools_for_many('get_comparison', data_tools, 1)
+        data_tools = [{"field":"number", "method":"doughnut", "metadata": dict(asd=123)}]
+        result = self._test_run_data_tools_for_many('get_comparison', data_tools)
+
+        self.assertEqual(len(result["statistics"]), 1)
+        self.assertEqual(result["statistics"][0]["metadata"]["asd"], 123)
 
     def test_get_overview_produces_overview(self):
         data_tools = [{"field":"number", "method":"latest"}]
-        return self._test_run_data_tools_for_many('get_overview', data_tools, 3)
+        result = self._test_run_data_tools_for_many('get_overview', data_tools)
+
+        self.assertEqual(len(result["statistics"]), 3)
+
 
     def test_overview_and_comparison_include_units(self):
         data_tools = [