Repository: openaddresses/machine
Pull request #485 (2016-12-10 20:21:41)

Add basic stats view
Adds a view at `/dashboard` which reflects a static point-in-time display of OA stats. Dynamic updating to come later.


Patch:
diff --git a/openaddr/ci/dashboard_stats.py b/openaddr/ci/dashboard_stats.py
new file mode 100644
index 0000000..4f17dd3
--- /dev/null
+++ b/openaddr/ci/dashboard_stats.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+"Python script to pull stats about OpenAddresses runs from the database and convert to JSON"
+
+import psycopg2, json, os, sys
+
+def make_stats(db_connect_string):
+    "Connect to the database and extract stats data, transforming into a JSON-friendly object"
+    db = psycopg2.connect(db_connect_string)
+    cur = db.cursor()
+
+    ### JSON output data object
+    data = {
+        'timeseries': [],
+        'last_address_counts': [],
+        'last_process_times': [],
+        'last_cache_times': [],
+        'lost_sources': { 'headers': ['Source', 'Max Addresses', 'Last Good'],
+                          'rows': []},
+    }
+
+    ### Get the timestamp of the last run
+    cur.execute('select max(tsName) from dashboard_stats')
+    last_ts = cur.fetchone()[0]
+
+    ### data['timeseries']: summary statistics for all runs
+    cur.execute('''
+        select tsName,
+               sum(address_count) as addresses,
+               count(*) as successful_sources,
+               avg(cache_time) as average_cache_time,
+               avg(process_time) as average_process_time
+        from dashboard_stats
+        where address_count > 0
+        group by tsName
+        order by tsName;
+    ''')
+    for row in cur.fetchall():
+        data['timeseries'].append({
+            'ts': int(float(row[0])*1000),
+            'addresses': row[1],
+            'successful_sources': row[2],
+            'average_cache_time': row[3],
+            'average_process_time': row[4]
+            })
+
+
+    ### data['last_address_counts'] and friends: detailed performance stats for the last run
+    cur.execute('''
+        select address_count, cache_time, process_time
+        from dashboard_stats
+        where tsName = %s
+    ''', (last_ts,))
+    results = cur.fetchall()
+    address_counts, cache_times, process_times = zip(*results)
+    def strip_and_sort(d):
+        return sorted((x for x in d if x is not None))
+    data['last_address_counts'] = strip_and_sort(address_counts)
+    data['last_process_times'] = strip_and_sort(process_times)
+    data['last_cache_times'] = strip_and_sort(cache_times)
+
+
+    ### data['lost_sources']: sources which previously had addresses, but not in the last run
+
+    # First calculate a list of all sources that did work once, but didn't last run
+    # Could probably use a SQL subselect for this, but let's just do it in Python
+    cur.execute('''
+        select source
+        from dashboard_stats
+        where tsName = %s
+              and (address_count = 0 or address_count is null);
+    ''', (last_ts,))
+    r = cur.fetchall()
+    no_address_list = [row[0] for row in r]
+
+    # Now get details from the last successful run of each of those failed sources
+    cur.execute('''
+        select source, max(address_count) as ac, max(tsName)
+        from dashboard_stats
+        where source = any(%s)
+              and address_count > 0
+        group by source
+        order by ac desc
+    ''', (no_address_list,))
+    for source, address_count, ts in cur.fetchall():
+        data['lost_sources']['rows'].append((source, address_count, ts))
+
+    return data
+
+if __name__ == '__main__':
+    data = make_stats(sys.argv[1])
+    sys.stdout.write(json.dumps(data))
diff --git a/openaddr/ci/templates/dashboard.html b/openaddr/ci/templates/dashboard.html
new file mode 100644
index 0000000..6bd4495
--- /dev/null
+++ b/openaddr/ci/templates/dashboard.html
@@ -0,0 +1,235 @@
+<html lang='en'>
+<head>
+<script src='https://d3js.org/d3.v3.min.js' charset='utf-8'></script>
+<link rel="stylesheet" href="/static/writ.css">
+<style>
+
+/* SVG styles for graph */
+.axis path, .axis line { fill: none; stroke: #000; shape-rendering: crispEdges; }
+.line { fill: none; stroke: #000; stroke-width: 2.5px; }
+.bar rect { fill: #000; shape-rendering: crispEdges; }
+.bar text { fill: #fff; font-size: 10px; }
+
+/* Inline HTML styles */
+
+td, th { border: none }
+
+/* Inline style lifted from http://results.openaddresses.io/index.html */
+body
+{
+    margin-bottom: 2em;
+    font-size: 1em;
+}
+
+body>*, main>*, footer>*
+{
+    margin: 1em auto;
+    width: 960px;
+}
+
+body>p, main>p, footer>p { line-height: 1.5em }
+
+tr:nth-child(2n+1) td { background-color: #f8f8f8 }
+tr:nth-child(2n+2) td { background-color: #ffffff }
+
+</style> </head>
+<body>
+
+<h1>Summary Statistics</h1>
+
+<h2>Address Count</h2>
+<div id='addresses'></div>
+
+<h2>Successful Sources</h2>
+<div id='sources'></div>
+
+<h2>Average Cache Time</h2>
+<div id='cache'></div>
+
+<h2>Average Process Time</h2>
+<div id='process'></div>
+
+<h1>Details for last run on <span id="lastRun"></span></h1>
+
+<h2>Address counts per source</h2>
+<div id='addressesHistogram'></div>
+
+<h2>Cache time per source</h2>
+<div id='cacheHistogram'></div>
+
+<h2>Process time per source</h2>
+<div id='processHistogram'></div>
+
+<h2>Lost sources</h2>
+<div id='lostStats'></div>
+<div id='lostTable'></div>
+
+
+<script>
+var dateFormatter = d3.time.format.utc('%Y-%m-%d');
+// var dateFormatter = d3.time.format.utc('%A, %b %d, %Y');
+
+/* Create a D3 time series graph.
+ * elem: the D3/DOM container for the graph. An SVG element will be added.
+ * data: an array. Each element is an object with a property "ts" (millseconds-since-epoch) and other data
+ * accessor: a function that given a data element, returns the value to be graphed
+ * xRange: [minX, maxX]
+ * yRange: [minY, maxY]
+ */
+function graphTimeSeries(elem, data, accessor, xRange, yRange) {
+    var margin = {top: 20, right: 20, bottom: 30, left: 100},
+        width = 960 - margin.left - margin.right,
+        height = 300 - margin.top - margin.bottom;
+
+    var x = d3.time.scale().range([0, width]);
+    var y = d3.scale.linear().range([height, 0]);
+    var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(5);
+    var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5);
+    var line = d3.svg.line()
+                   .x(function(d) { return x(d.ts); })
+                   .y(function(d) { return y(accessor(d)); });
+
+    var svg = elem.append("svg")
+        .attr("width", width + margin.left + margin.right)
+        .attr("height", height + margin.top + margin.bottom)
+      .append("g")
+        .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+    x.domain(xRange);
+    y.domain(yRange);
+
+    svg.append("g")
+      .attr("class", "x axis")
+      .attr("transform", "translate(0," + height + ")")
+      .call(xAxis);
+
+    svg.append("g")
+        .attr("class", "y axis")
+        .call(yAxis);
+
+    svg.append("path")
+        .datum(data)
+        .attr("class", "line")
+        .attr("d", line);
+}
+
+/* Create a D3 histogram bar chart.
+ * elem: the D3/DOM container for the graph. An SVG element will be added.
+ * data: an array of numbers
+ * valueExtent: histogram will be clamped to this range.
+ */
+function graphHistogram(elem, values, valueExtent) {
+    var numBins = 21;
+    var formatCount = d3.format(",.0f");
+
+    if (valueExtent == null) {
+        valueExtent = d3.extent(values);
+    }
+
+    var margin = {top: 10, right: 30, bottom: 30, left: 30},
+        width = 960 - margin.left - margin.right,
+        height = 300 - margin.top - margin.bottom;
+
+    // values = values.map(function(d) { return d > 0 ? Math.log10(d) : 0 });
+    var x = d3.scale.linear().domain(valueExtent).range([0, width]).clamp(true);
+    var barData = d3.layout.histogram().bins(x.ticks(numBins))(values);
+    var maxY = d3.max(barData, function(d) { return d.y; });
+    var y = d3.scale.linear().domain([0, maxY]).range([height, 0]);
+
+    var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(7);
+
+    var svg = elem.append("svg")
+        .attr("width", width + margin.left + margin.right)
+        .attr("height", height + margin.top + margin.bottom)
+      .append("g")
+        .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+    var bar = svg.selectAll(".bar")
+        .data(barData)
+      .enter().append("g")
+        .attr("class", "bar")
+        .attr("transform", function(d) { return "translate(" + x(d.x) + "," + y(d.y) + ")"; });
+
+    bar.append("rect")
+        .attr("x", 1)
+        .attr("width", x(barData[0].dx) - 1)
+        .attr("height", function(d) { return height - y(d.y); });
+
+    bar.append("text")
+        .attr("dy", ".75em")
+        .attr("y", 6)
+        .attr("x", x(barData[0].dx) / 2)
+        .attr("text-anchor", "middle")
+        .text(function(d) { return (height-y(d.y)) > 20 ? formatCount(d.y) : "" });
+
+    svg.append("g")
+        .attr("class", "x axis")
+        .attr("transform", "translate(0," + height + ")")
+        .call(xAxis);
+}
+
+/* Create some textual stats about lost sources */
+function addLostStats(data) {
+    var div = d3.select('#lostStats');
+    var sourceCount = data.rows.length;
+    var addressCount = d3.sum(data.rows, function(d) { return d[1]; });
+    addressCount = addressCount.toLocaleString();
+    div.html('' + sourceCount + ' previously working sources failed to process on the last run.<br>' +
+             addressCount + ' addresses in total were lost.');
+}
+
+/* Create a table display for a CSV-like data structure
+ * dataTable: object containing two fields: 'headers' and 'rows'
+ */
+function addLostTable(data) {
+    elem = d3.select('#lostTable');
+
+    var headers = data.headers;
+    var rows = data.rows.map(function(row) {
+                 return [row[0].replace('.json', ''),
+                         row[1].toLocaleString(),
+                         dateFormatter(new Date(row[2]*1000))
+                        ];});
+
+    var table = elem.append('table');
+
+    // Table headers
+    table.append('thead').append('tr').selectAll('th')
+      .data(headers).enter()
+        .append('th').text(Object)
+                     .style('text-align', function(d, i) { return i == 0 ? 'left' : 'right' });
+
+    // Table data rows
+    var tr = table.append('tbody').selectAll('tr')
+      .data(rows).enter()
+        .append('tr');
+
+    // Table data values
+    tr.selectAll('td')
+      .data(Object).enter()
+        .append('td').text(Object)
+                     .style('text-align', function(d, i) { return i == 0 ? 'left' : 'right' });
+}
+
+// Main script. Load the data and graph it.
+
+d3.json('https://s3.amazonaws.com/data.openaddresses.io/machine-stats.json', function(error, data) {
+    var timeseries = data['timeseries'];
+    var dateRange = d3.extent(timeseries, function(d) { return d.ts; });
+
+    graphTimeSeries(d3.select("#addresses"), timeseries, function(d) { return d.addresses; }, dateRange, [60000000, 300000000]);
+    graphTimeSeries(d3.select('#sources'), timeseries, function(d) { return d.successful_sources; }, dateRange, [400, 1500]);
+    graphTimeSeries(d3.select('#cache'), timeseries, function(d) { return d.average_cache_time; }, dateRange, [0, 200]);
+    graphTimeSeries(d3.select('#process'), timeseries, function(d) { return d.average_process_time; }, dateRange, [0, 400]);
+
+    graphHistogram(d3.select('#addressesHistogram'), data['last_address_counts'], [0, 200000])
+    graphHistogram(d3.select('#cacheHistogram'), data['last_cache_times'], [0, 600])
+    graphHistogram(d3.select('#processHistogram'), data['last_process_times'], [0, 600])
+
+    d3.select('#lastRun').text(dateFormatter(new Date(dateRange[1])));
+    addLostStats(data['lost_sources']);
+    addLostTable(data['lost_sources']);
+});
+</script>
+</body>
+</html>
diff --git a/openaddr/ci/webhooks.py b/openaddr/ci/webhooks.py
index 59ef4f7..9b86cf8 100644
--- a/openaddr/ci/webhooks.py
+++ b/openaddr/ci/webhooks.py
@@ -282,6 +282,11 @@ def app_get_source_history(source):
     
     return render_template('source.html', runs=runs, run=run, source_data=source_data)
 
+@webhooks.route('/dashboard', methods=['GET'])
+@log_application_errors
+def app_get_dashboard():
+    return render_template('dashboard.html')
+
 def nice_timedelta(delta):
     '''
     >>> nice_timedelta(timedelta(days=2))


Test patch:
diff --git a/openaddr/tests/dashboard_stats.py b/openaddr/tests/dashboard_stats.py
new file mode 100644
index 0000000..c97c27a
--- /dev/null
+++ b/openaddr/tests/dashboard_stats.py
@@ -0,0 +1,8 @@
+import unittest
+
+from ..ci import dashboard_stats
+
+class TestDashboardStats (unittest.TestCase):
+    
+    def test_nothing(self):
+        pass
\ No newline at end of file
diff --git a/test.py b/test.py
index 9ff2d80..7207d00 100755
--- a/test.py
+++ b/test.py
@@ -29,6 +29,7 @@ from openaddr.tests.preview import TestPreview
 from openaddr.tests.util import TestUtilities
 from openaddr.tests.summarize import TestSummarizeFunctions
 from openaddr.tests.parcels import TestParcelsUtils, TestParcelsParse
+from openaddr.tests.dashboard_stats import TestDashboardStats
 
 from openaddr.tests.ci import (
     TestHook, TestRuns, TestWorker, TestBatch, TestObjects, TestCollect,