Repository: privacyidea/privacyidea

Issue:
Enhance /audit/statistics in WebUI
The webui at Audit->Statistics should allow

1. for a flexible number of days
2. a time range from -> to, specifying date and time.

In addition the API should return the plain numbers.

This way we can print it in the web UI as normal table and click a link to the corresponding tokens or users.


Discussion:

Resolving patch:
diff --git a/privacyidea/api/audit.py b/privacyidea/api/audit.py
--- a/privacyidea/api/audit.py
+++ b/privacyidea/api/audit.py
@@ -43,6 +43,8 @@
 from ..lib.stats import get_statistics
 import datetime
 from privacyidea.lib.utils import parse_timedelta
+from dateutil.parser import parse as parse_date_string
+from dateutil.tz import tzlocal
 
 log = logging.getLogger(__name__)
 
@@ -160,6 +162,15 @@ def statistics():
     """
     get the statistics values from the audit log
 
+    :jsonparam days: The number of days to run the stats
+    :jsonparam start: The start time to run the stats
+    :jsonparam end: The end time to run the stats
+
+    If start or end is missing, the ``days`` are used.
+
+    The time is to be passed in the format
+        yyyy-MM-ddTHH:mmZ
+
     **Example request**:
 
     .. sourcecode:: http
@@ -190,8 +201,27 @@ def statistics():
         }
     """
     days = int(getParam(request.all_data, "days", default=7))
+    start = getParam(request.all_data, "start")
+    if start:
+        start = parse_date_string(start)
+
+    end = getParam(request.all_data, "end")
+    if end:
+        end = parse_date_string(end)
+
+    if not end and not start:
+        end = datetime.datetime.now(tzlocal())
+        start = end - datetime.timedelta(days=days)
+
+    else:
+        if not end:
+            end = start + datetime.timedelta(days=days)
+        elif not start:
+            start = end - datetime.timedelta(days=days)
+
     stats = get_statistics(g.audit_object,
-                           start_time=datetime.datetime.now()
-                                      -datetime.timedelta(days=days))
+                           start_time=start, end_time=end)
+    stats["time_start"] = start
+    stats["time_end"] = end
     g.audit_object.log({'success': True})
     return send_result(stats)
diff --git a/privacyidea/lib/stats.py b/privacyidea/lib/stats.py
--- a/privacyidea/lib/stats.py
+++ b/privacyidea/lib/stats.py
@@ -24,6 +24,7 @@
 This module is tested in tests/test_lib_stats.py
 """
 import logging
+import traceback
 from privacyidea.lib.log import log_with
 import datetime
 import StringIO
@@ -64,13 +65,13 @@ def get_statistics(auditobject, start_time=datetime.datetime.now()
 
     # get simple usage
     for key in ["serial", "action"]:
-        result["{0!s}_plot".format(key)] = _get_number_of(df, key)
+        result["{0!s}_plot".format(key)], result["{0!s}".format(key)] = _get_number_of(df, key)
 
     # failed authentication requests
     for key in ["user", "serial"]:
-        result["validate_failed_{0!s}_plot".format(key)] = _get_fail(df, key)
+        result["validate_failed_{0!s}_plot".format(key)], result["validate_failed_{0!s}".format(key)] = _get_fail(df, key)
 
-    result["admin_plot"] = _get_number_of(df, "action", nums=20)
+    result["admin_plot"], result["admin"] = _get_number_of(df, "action", nums=20)
 
     return result
 
@@ -79,6 +80,7 @@ def _get_success_fail(df, key):
 
     try:
         output = StringIO.StringIO()
+        # in this case series is actually a dataframe
         series = df[df.action.isin(["POST /validate/check",
                                     "GET /validate/check"])].groupby([key,
                                                                 'success']).size().unstack()
@@ -94,19 +96,21 @@ def _get_success_fail(df, key):
         image_uri = 'data:image/png;base64,{0!s}'.format(image_data)
     except Exception as exx:
         log.info(exx)
-        image_uri = "{0!s}".format(exx)
+        log.debug(traceback.format_exc())
+        image_uri = "No statistics available. Maybe you are missing the matplotlib installation."
     return image_uri
 
 
-def _get_fail(df, key):
+def _get_fail(df, key, nums=5):
 
     try:
         output = StringIO.StringIO()
         series = df[(df.success==0)
                     & (df.action.isin(["POST /validate/check",
                                        "GET /validate/check"]))][
-                     key].value_counts()[:5]
+                     key].value_counts(sort=True, dropna=True)[:nums]
 
+        series_list = [{"key": series.keys()[i], "count": series.tolist()[i]} for i in range(series.size)]
         plot_canvas = matplotlib.pyplot.figure()
         ax = plot_canvas.add_subplot(1,1,1)
 
@@ -123,8 +127,10 @@ def _get_fail(df, key):
         image_uri = 'data:image/png;base64,{0!s}'.format(image_data)
     except Exception as exx:
         log.info(exx)
-        image_uri = "{0!s}".format(exx)
-    return image_uri
+        log.debug(traceback.format_exc())
+        image_uri = "No statistics available. Maybe you are missing the matplotlib installation."
+        series_list = []
+    return image_uri, series_list
 
 
 def _get_number_of(df, key, nums=5):
@@ -135,7 +141,7 @@ def _get_number_of(df, key, nums=5):
     :param df: The DataFrame
     :type df: Pandas DataFrame
     :param key: The key, which should be plotted.
-    :param count: how many of the most often values should be plotted
+    :param num: how many of the most often values should be plotted
     :return: A data url
     """
     output = StringIO.StringIO()
@@ -144,7 +150,8 @@ def _get_number_of(df, key, nums=5):
         plot_canvas = matplotlib.pyplot.figure()
         ax = plot_canvas.add_subplot(1, 1, 1)
 
-        series = df[key].value_counts()[:nums]
+        series = df[key].value_counts(sort=True, dropna=True)[:nums]
+        series_list = [{"key": series.keys()[i], "count": series.tolist()[i]} for i in range(series.size)]
         fig = series.plot(ax=ax, kind="bar", colormap="Blues",
                           legend=False,
                           stacked=False,
@@ -157,5 +164,7 @@ def _get_number_of(df, key, nums=5):
         image_uri = 'data:image/png;base64,{0!s}'.format(image_data)
     except Exception as exx:
         log.info(exx)
-        image_uri = "No data"
-    return image_uri
+        log.debug(traceback.format_exc())
+        image_uri = "No statistics available. Maybe you are missing the matplotlib installation."
+        series_list = []
+    return image_uri, series_list


Test patch:
diff --git a/tests/test_api_audit.py b/tests/test_api_audit.py
--- a/tests/test_api_audit.py
+++ b/tests/test_api_audit.py
@@ -2,6 +2,9 @@
 from .base import MyTestCase
 from privacyidea.lib.policy import set_policy, SCOPE, ACTION, delete_policy
 from privacyidea.models import Audit
+import datetime
+from dateutil.parser import parse as parse_time_string
+from dateutil.tz import tzlocal
 
 PWFILE = "tests/testdata/passwords"
 POLICYFILE = "tests/testdata/policy.cfg"
@@ -31,6 +34,41 @@ def test_01_get_statistics(self):
             self.assertTrue("serial_plot" in json_response.get(
                 "result").get("value"), json_response.get("result"))
 
+        start = datetime.datetime.now(tzlocal()) - datetime.timedelta(days=10)
+        start_str = start.strftime("%Y-%m-%dT%H:%M%Z")
+        # We need to reparse it, to get rid of the seconds and milliseconds
+        start = parse_time_string(start_str)
+        with self.app.test_request_context('/audit/statistics',
+                                           method='GET',
+                                           data={"start": start_str},
+                                           headers={'Authorization': self.at}):
+            res = self.app.full_dispatch_request()
+            self.assertTrue(res.status_code == 200, res)
+            json_response = json.loads(res.data)
+            self.assertTrue(json_response.get("result").get("status"), res)
+            self.assertTrue("serial_plot" in json_response.get(
+                "result").get("value"), json_response.get("result"))
+            d = parse_time_string(json_response.get("result").get("value").get("time_start"))
+            self.assertEqual(d, start)
+
+        end = datetime.datetime.now(tzlocal()) - datetime.timedelta(days=10)
+        end_str = end.strftime("%Y-%m-%dT%H:%M%Z")
+        # We need to reparse it, to get rid of the seconds and milliseconds
+        end = parse_time_string(end_str)
+        with self.app.test_request_context('/audit/statistics',
+                                           method='GET',
+                                           data={"end": end_str},
+                                           headers={'Authorization': self.at}):
+            res = self.app.full_dispatch_request()
+            self.assertTrue(res.status_code == 200, res)
+            json_response = json.loads(res.data)
+            self.assertTrue(json_response.get("result").get("status"), res)
+            self.assertTrue("serial_plot" in json_response.get(
+                "result").get("value"), json_response.get("result"))
+            d = parse_time_string(json_response.get("result").get("value").get("time_end"))
+            self.assertEqual(d, end)
+
+
     def test_02_get_allowed_audit_realm(self):
         # Check that an administrator is only allowed to see log entries of
         # the defined realms.