Repository: liberapay/liberapay.com

Issue:
Payday stats are misleading
Refunds are counted as withdrawals, and the refunded payins aren't removed from deposits because payday stats are computed during payday and never corrected after.


Discussion:

Resolving patch:
diff --git a/liberapay/billing/payday.py b/liberapay/billing/payday.py
--- a/liberapay/billing/payday.py
+++ b/liberapay/billing/payday.py
@@ -81,10 +81,11 @@ def run(self, log_dir='.', keep_log=False):
 
         self.shuffle(log_dir)
 
-        self.update_stats()
+        self.end()
+
+        self.recompute_stats(limit=10)
         self.update_cached_amounts()
 
-        self.end()
         self.notify_participants()
 
         if not keep_log:
@@ -128,43 +129,9 @@ def shuffle(self, log_dir='.'):
                 self.check_balances(cursor)
                 with open(self.transfers_filename, 'wb') as f:
                     pickle.dump(transfers, f)
-                if self.id > 1:
-                    previous_ts_start = self.db.one("""
-                        SELECT ts_start
-                          FROM paydays
-                         WHERE id = %s
-                    """, (self.id - 1,))
-                else:
-                    previous_ts_start = constants.EPOCH
-                assert previous_ts_start
-                ts_start = self.ts_start
                 cursor.run("""
-                    WITH week_exchanges AS (
-                             SELECT e.*
-                               FROM exchanges e
-                              WHERE e.timestamp < %(ts_start)s
-                                AND e.timestamp >= %(previous_ts_start)s
-                                AND status <> 'failed'
-                         )
                     UPDATE paydays
                        SET nparticipants = (SELECT count(*) FROM payday_participants)
-                         , nusers = (
-                               SELECT count(*)
-                                 FROM participants
-                                WHERE kind IN ('individual', 'organization')
-                                  AND join_time < %(ts_start)s
-                                  AND status = 'active'
-                           )
-                         , week_deposits = (
-                               SELECT COALESCE(sum(amount), 0)
-                                 FROM week_exchanges
-                                WHERE amount > 0
-                           )
-                         , week_withdrawals = (
-                               SELECT COALESCE(-sum(amount), 0)
-                                 FROM week_exchanges
-                                WHERE amount < 0
-                           )
                      WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz;
                 """, locals())
             self.clean_up()
@@ -193,6 +160,7 @@ def prepare(cursor, ts_start):
              WHERE join_time < %(ts_start)s
                AND (mangopay_user_id IS NOT NULL OR kind = 'group')
                AND is_suspended IS NOT true
+               AND status = 'active'
           ORDER BY join_time;
 
         CREATE UNIQUE INDEX ON payday_participants (id);
@@ -426,14 +394,29 @@ def clean_up(self):
             DROP FUNCTION resolve_takes(bigint);
         """)
 
-    def update_stats(self):
-        self.db.run("""\
+    @classmethod
+    def update_stats(cls, payday_id):
+        ts_start, ts_end = cls.db.one("""
+            SELECT ts_start, ts_end FROM paydays WHERE id = %s
+        """, (payday_id,))
+        if payday_id > 1:
+            previous_ts_start = cls.db.one("""
+                SELECT ts_start
+                  FROM paydays
+                 WHERE id = %s
+            """, (payday_id - 1,))
+        else:
+            previous_ts_start = constants.EPOCH
+        assert previous_ts_start
+        cls.db.run("""\
 
             WITH our_transfers AS (
                      SELECT *
                        FROM transfers
                       WHERE "timestamp" >= %(ts_start)s
+                        AND "timestamp" <= %(ts_end)s
                         AND status = 'succeeded'
+                        AND context <> 'refund'
                  )
                , our_tips AS (
                      SELECT *
@@ -445,6 +428,18 @@ def update_stats(self):
                        FROM our_transfers
                       WHERE context = 'take'
                  )
+               , week_exchanges AS (
+                     SELECT e.*
+                          , ( EXISTS (
+                                SELECT e2.id
+                                  FROM exchanges e2
+                                 WHERE e2.refund_ref = e.id
+                            )) AS refunded
+                       FROM exchanges e
+                      WHERE e.timestamp < %(ts_start)s
+                        AND e.timestamp >= %(previous_ts_start)s
+                        AND status <> 'failed'
+                 )
             UPDATE paydays
                SET nactive = (
                        SELECT DISTINCT count(*) FROM (
@@ -460,10 +455,67 @@ def update_stats(self):
                  , take_volume = (SELECT COALESCE(sum(amount), 0) FROM our_takes)
                  , ntransfers = (SELECT count(*) FROM our_transfers)
                  , transfer_volume = (SELECT COALESCE(sum(amount), 0) FROM our_transfers)
-             WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz
+                 , transfer_volume_refunded = (
+                       SELECT COALESCE(sum(amount), 0)
+                         FROM our_transfers
+                        WHERE refund_ref IS NOT NULL
+                   )
+                 , nusers = (
+                       SELECT count(*)
+                         FROM participants p
+                        WHERE p.kind IN ('individual', 'organization')
+                          AND p.join_time < %(ts_start)s
+                          AND COALESCE((
+                                SELECT payload::text
+                                  FROM events e
+                                 WHERE e.participant = p.id
+                                   AND e.type = 'set_status'
+                                   AND e.ts < %(ts_start)s
+                              ORDER BY ts DESC
+                                 LIMIT 1
+                              ), '') <> '"closed"'
+                   )
+                 , week_deposits = (
+                       SELECT COALESCE(sum(amount), 0)
+                         FROM week_exchanges
+                        WHERE amount > 0
+                          AND refund_ref IS NULL
+                          AND status = 'succeeded'
+                   )
+                 , week_deposits_refunded = (
+                       SELECT COALESCE(sum(amount), 0)
+                         FROM week_exchanges
+                        WHERE amount > 0
+                          AND refunded
+                   )
+                 , week_withdrawals = (
+                       SELECT COALESCE(-sum(amount), 0)
+                         FROM week_exchanges
+                        WHERE amount < 0
+                          AND refund_ref IS NULL
+                   )
+                 , week_withdrawals_refunded = (
+                       SELECT COALESCE(sum(amount), 0)
+                         FROM week_exchanges
+                        WHERE amount < 0
+                          AND refunded
+                   )
+             WHERE id = %(payday_id)s
 
-        """, {'ts_start': self.ts_start})
-        log("Updated payday stats.")
+        """, locals())
+        log("Updated stats of payday #%i." % payday_id)
+
+    @classmethod
+    def recompute_stats(cls, limit=None):
+        ids = cls.db.all("""
+            SELECT id
+              FROM paydays
+             WHERE ts_end > ts_start
+          ORDER BY id DESC
+             LIMIT %s
+        """, (limit,))
+        for payday_id in ids:
+            cls.update_stats(payday_id)
 
     def update_cached_amounts(self):
         now = pando.utils.utcnow()
diff --git a/liberapay/utils/__init__.py b/liberapay/utils/__init__.py
--- a/liberapay/utils/__init__.py
+++ b/liberapay/utils/__init__.py
@@ -169,7 +169,7 @@ def update_global_stats(website):
         SELECT count(*)
           FROM participants
          WHERE status = 'active'
-           AND kind <> 'community';
+           AND kind IN ('individual', 'organization');
     """)
     transfer_volume = website.db.one("""
         SELECT coalesce(sum(amount), 0)


Test patch:
diff --git a/tests/py/test_charts_json.py b/tests/py/test_charts_json.py
--- a/tests/py/test_charts_json.py
+++ b/tests/py/test_charts_json.py
@@ -135,6 +135,9 @@ def test_charts_work_for_teams(self):
         assert actual == expected
 
     def test_transfer_volume(self):
+        dana = self.make_participant('dana')
+        dana.close(None)
+
         self.run_payday()
         self.run_payday()
 
@@ -149,7 +152,10 @@ def test_transfer_volume(self):
             "xTitle": utcnow().strftime('%Y-%m-%d'),
         }
         actual = json.loads(self.client.GET('/about/charts.json').text)[0]
+        assert actual == expected
 
+        Payday.recompute_stats()
+        actual = json.loads(self.client.GET('/about/charts.json').text)[0]
         assert actual == expected
 
     def test_anonymous_receiver(self):