Repository: gratipay/gratipay.com

Issue:
add data export for history
Per @esacteksab on his [blog](http://www.barrymorrison.com/2013/Mar/19/how-i-use-gittip-and-why/).



Discussion:
Jason Scott talked about this at Waza, too.

+1 from @roguelynn on [Twitter](https://twitter.com/roguelynn/status/426884542096367617) in the context of taxes.

+1

mostly interested in not having to manually tally "Gifts in", "Gifts out", and "Team take in", which seem to be the important distinctions in Canada for tax purposes -- "Gifts received (personal)", "Gifts given (business)" and "Income"

(Still need to talk to my accountant, so I might be corrected on the above.)

+1 https://gratipay.freshdesk.com/helpdesk/tickets/1749 (cf. https://github.com/gratipay/gratipay.com/issues/662#issuecomment-77820324)

> Here's what I whipped up in a spreadsheet and sent to our accountant:
> 
> ```
> Annual Gratipay summary:
> 
> -------------------------------------------------------------------------------
> {recipient} | {amount received} | {individual or organization} |
> -------------------------------------------------------------------------------
> {recipient} | {amount received} | {individual or organization} |
> 
> ```
> 
> A CSV export scoped to each calendar year would be great. :)

Me:

> What's the difference between "recipient" and "individual or organization"?

Them:

> Probably wouldn't be needed in the CSV export feature — it's how I'm distinguishing between donations to "a specific individual" vs. "a general org" like Homebrew or Bundler. (which our accountant asked for)


Resolving patch:
diff --git a/gratipay/main.py b/gratipay/main.py
--- a/gratipay/main.py
+++ b/gratipay/main.py
@@ -10,7 +10,7 @@
 from gratipay.security import authentication, csrf, x_frame_options
 from gratipay.utils import cache_static, erase_cookie, i18n, set_cookie, timer
 from gratipay.version import get_version
-from gratipay.renderers import jinja2_htmlescaped
+from gratipay.renderers import csv_dump, jinja2_htmlescaped
 
 import aspen
 from aspen.website import Website
@@ -24,6 +24,7 @@
 
 website.renderer_default = 'unspecified'  # require explicit renderer, to avoid escaping bugs
 
+website.renderer_factories['csv_dump'] = csv_dump.Factory(website)
 website.renderer_factories['jinja2_htmlescaped'] = jinja2_htmlescaped.Factory(website)
 website.default_renderers_by_media_type['text/html'] = 'jinja2_htmlescaped'
 website.default_renderers_by_media_type['text/plain'] = 'jinja2'  # unescaped is fine here
diff --git a/gratipay/renderers/csv_dump.py b/gratipay/renderers/csv_dump.py
new file mode 100644
--- /dev/null
+++ b/gratipay/renderers/csv_dump.py
@@ -0,0 +1,25 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import csv
+from io import BytesIO
+
+from aspen import renderers
+
+
+class Renderer(renderers.Renderer):
+
+    def render_content(self, context):
+        rows = eval(self.compiled, globals(), context)
+        if not rows:
+            return ''
+        f = BytesIO()
+        w = csv.writer(f)
+        if hasattr(rows[0], '_fields'):
+            w.writerow(rows[0]._fields)
+        w.writerows(rows)
+        f.seek(0)
+        return f.read()
+
+
+class Factory(renderers.Factory):
+    Renderer = Renderer
diff --git a/gratipay/utils/history.py b/gratipay/utils/history.py
--- a/gratipay/utils/history.py
+++ b/gratipay/utils/history.py
@@ -1,6 +1,7 @@
 from datetime import datetime
 from decimal import Decimal
 
+from aspen import Response
 from psycopg2 import IntegrityError
 
 
@@ -138,3 +139,66 @@ def iter_payday_events(db, participant, year=None):
         yield event
 
     yield dict(kind='day-close', balance=balance)
+
+
+def export_history(participant, year, mode, key, back_as='namedtuple', require_key=False):
+    db = participant.db
+    params = dict(username=participant.username, year=year)
+    out = {}
+    if mode == 'aggregate':
+        out['given'] = lambda: db.all("""
+            SELECT tippee, sum(amount) AS amount
+              FROM transfers
+             WHERE tipper = %(username)s
+               AND extract(year from timestamp) = %(year)s
+          GROUP BY tippee
+        """, params, back_as=back_as)
+        out['taken'] = lambda: db.all("""
+            SELECT tipper AS team, sum(amount) AS amount
+              FROM transfers
+             WHERE tippee = %(username)s
+               AND context = 'take'
+               AND extract(year from timestamp) = %(year)s
+          GROUP BY tipper
+        """, params, back_as=back_as)
+    else:
+        out['exchanges'] = lambda: db.all("""
+            SELECT timestamp, amount, fee, status, note
+              FROM exchanges
+             WHERE participant = %(username)s
+               AND extract(year from timestamp) = %(year)s
+          ORDER BY timestamp ASC
+        """, params, back_as=back_as)
+        out['given'] = lambda: db.all("""
+            SELECT timestamp, tippee, amount, context
+              FROM transfers
+             WHERE tipper = %(username)s
+               AND extract(year from timestamp) = %(year)s
+          ORDER BY timestamp ASC
+        """, params, back_as=back_as)
+        out['taken'] = lambda: db.all("""
+            SELECT timestamp, tipper AS team, amount
+              FROM transfers
+             WHERE tippee = %(username)s
+               AND context = 'take'
+               AND extract(year from timestamp) = %(year)s
+          ORDER BY timestamp ASC
+        """, params, back_as=back_as)
+        out['received'] = lambda: db.all("""
+            SELECT timestamp, amount, context
+              FROM transfers
+             WHERE tippee = %(username)s
+               AND context NOT IN ('take', 'take-over')
+               AND extract(year from timestamp) = %(year)s
+          ORDER BY timestamp ASC
+        """, params, back_as=back_as)
+
+    if key:
+        try:
+            return out[key]()
+        except KeyError:
+            raise Response(400, "bad key `%s`" % key)
+    elif require_key:
+        raise Response(400, "missing `key` parameter")
+    else:
+        return {k: v() for k, v in out.items()}


Test patch:
diff --git a/tests/py/test_history.py b/tests/py/test_history.py
--- a/tests/py/test_history.py
+++ b/tests/py/test_history.py
@@ -1,13 +1,38 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from datetime import datetime
 from decimal import Decimal
+import json
 
 from mock import patch
 
 from gratipay.billing.payday import Payday
 from gratipay.models.participant import Participant
 from gratipay.testing import Harness
-from gratipay.utils.history import iter_payday_events
+from gratipay.utils.history import get_end_of_year_balance, iter_payday_events
+
+
+def make_history(harness):
+    alice = harness.make_participant('alice', claimed_time=datetime(2001, 1, 1, 0, 0, 0))
+    harness.alice = alice
+    harness.make_exchange('bill', 50, 0, alice)
+    harness.make_exchange('bill', 12, 0, alice, status='failed')
+    harness.make_exchange('ach', -40, 0, alice)
+    harness.make_exchange('ach', -5, 0, alice, status='failed')
+    harness.db.run("""
+        UPDATE exchanges
+           SET timestamp = "timestamp" - interval '1 year'
+    """)
+    harness.past_year = int(harness.db.one("""
+        SELECT extract(year from timestamp)
+          FROM exchanges
+      ORDER BY timestamp ASC
+         LIMIT 1
+    """))
+    harness.make_exchange('bill', 35, 0, alice)
+    harness.make_exchange('bill', 49, 0, alice, status='failed')
+    harness.make_exchange('ach', -15, 0, alice)
+    harness.make_exchange('ach', -7, 0, alice, status='failed')
 
 
 class TestHistory(Harness):
@@ -84,3 +109,31 @@ def test_iter_payday_events_with_failed_exchanges(self):
         assert events[3]['balance'] == 50
         assert events[4]['kind'] == 'day-close'
         assert events[4]['balance'] == 0
+
+    def test_get_end_of_year_balance(self):
+        make_history(self)
+        balance = get_end_of_year_balance(self.db, self.alice, self.past_year, datetime.now().year)
+        assert balance == 10
+
+
+class TestExport(Harness):
+
+    def setUp(self):
+        Harness.setUp(self)
+        make_history(self)
+
+    def test_export_json(self):
+        r = self.client.GET('/alice/history/export.json', auth_as='alice')
+        assert json.loads(r.body)
+
+    def test_export_json_aggregate(self):
+        r = self.client.GET('/alice/history/export.json?mode=aggregate', auth_as='alice')
+        assert json.loads(r.body)
+
+    def test_export_json_past_year(self):
+        r = self.client.GET('/alice/history/export.json?year=%s' % self.past_year, auth_as='alice')
+        assert len(json.loads(r.body)['exchanges']) == 4
+
+    def test_export_csv(self):
+        r = self.client.GET('/alice/history/export.csv?key=exchanges', auth_as='alice')
+        assert r.body.count('\n') == 5