Repository: liberapay/liberapay.com

Issue:
Race condition in mangopay dispute callbacks
See <https://github.com/liberapay/salon/issues/223#issuecomment-382426286>.

Traceback:

```
  File "site-packages/algorithm.py", line 321, in loop
    new_state = function(**deps.as_kwargs)
  File "aspen/request_processor/algorithm.py", line 73, in render_resource
    return {'output': resource.render(state)}
  File "aspen/http/resource.py", line 88, in render
    return super(Dynamic, self).render(best_match, state)
  File "aspen/simplates/simplate.py", line 121, in render
    exec(self.pages[1], context)
  File "www/callbacks/mangopay.spt", line 91, in <module>
    recover_lost_funds(website.db, exchange, dispute_amount, dispute.RepudiationId)
  File "liberapay/billing/transactions.py", line 801, in recover_lost_funds
    return_payin_bundles_to_origin(db, exchange, LiberapayOrg, create_debts=True)
  File "liberapay/billing/transactions.py", line 774, in return_payin_bundles_to_origin
    transfer(db, payer, original_owner, amount, 'chargeback', bundles=bundles, **transfer_kw)
  File "liberapay/billing/transactions.py", line 484, in transfer
    return execute_transfer(db, t_id, tr), t_id
  File "liberapay/billing/transactions.py", line 555, in execute_transfer
    return record_transfer_result(db, t_id, tr, _raise=True)
  File "liberapay/billing/transactions.py", line 562, in record_transfer_result
    r = _record_transfer_result(db, t_id, status, error)
  File "liberapay/billing/transactions.py", line 614, in _record_transfer_result
    assert bundles_sum == amount
AssertionError: 
```


Discussion:

Resolving patch:
diff --git a/liberapay/billing/payday.py b/liberapay/billing/payday.py
--- a/liberapay/billing/payday.py
+++ b/liberapay/billing/payday.py
@@ -1045,21 +1045,15 @@ def main(override_payday_checks=False):
         assert not r, "payday has already been run this week"
 
     # Prevent a race condition, by acquiring a DB lock
-    conn = website.db.get_connection().__enter__()
-    cursor = conn.cursor()
-    lock = cursor.one("SELECT pg_try_advisory_lock(1)")
-    assert lock, "failed to acquire the payday lock"
-
-    try:
-        sync_with_mangopay(website.db)
-        Payday.start().run(website.env.log_dir, website.env.keep_payday_logs)
-    except KeyboardInterrupt:  # pragma: no cover
-        pass
-    except Exception as e:  # pragma: no cover
-        website.tell_sentry(e, {}, allow_reraise=False)
-        raise
-    finally:
-        conn.close()
+    with website.db.lock('payday', blocking=False):
+        try:
+            sync_with_mangopay(website.db)
+            Payday.start().run(website.env.log_dir, website.env.keep_payday_logs)
+        except KeyboardInterrupt:  # pragma: no cover
+            pass
+        except Exception as e:  # pragma: no cover
+            website.tell_sentry(e, {}, allow_reraise=False)
+            raise
 
 
 if __name__ == '__main__':  # pragma: no cover
diff --git a/liberapay/billing/transactions.py b/liberapay/billing/transactions.py
--- a/liberapay/billing/transactions.py
+++ b/liberapay/billing/transactions.py
@@ -381,6 +381,8 @@ def record_exchange_result(db, exchange_id, remote_id, status, error, participan
 def propagate_exchange(cursor, participant, exchange, error, amount, bundles=None):
     """Propagates an exchange's result to the participant's balance.
     """
+    cursor.run("LOCK TABLE cash_bundles IN EXCLUSIVE MODE")
+
     wallet_id = exchange.wallet_id
     new_balance = cursor.one("""
         UPDATE wallets
@@ -394,18 +396,30 @@ def propagate_exchange(cursor, participant, exchange, error, amount, bundles=Non
         raise NegativeBalance
 
     if amount < 0:
-        bundles = bundles or cursor.all("""
-            LOCK TABLE cash_bundles IN EXCLUSIVE MODE;
-            SELECT b.*
-              FROM cash_bundles b
-              JOIN exchanges e ON e.id = b.origin
-             WHERE b.owner = %s
-               AND b.ts < now() - INTERVAL %s
-               AND b.disputed IS NOT TRUE
-               AND b.locked_for IS NULL
-               AND b.amount::currency = %s
-          ORDER BY b.owner = e.participant DESC, b.ts
-        """, (participant.id, QUARANTINE, amount.currency))
+        if bundles:
+            # Refetch the bundles to ensure validity and prevent race conditions
+            bundles = cursor.all("""
+                SELECT b.*
+                  FROM cash_bundles b
+                  JOIN exchanges e ON e.id = b.origin
+                 WHERE b.owner = %s
+                   AND b.locked_for IS NULL
+                   AND b.amount::currency = %s
+                   AND b.id IN %s
+              ORDER BY b.owner = e.participant DESC, b.ts
+            """, (participant.id, amount.currency, tuple(bundles)))
+        else:
+            bundles = cursor.all("""
+                SELECT b.*
+                  FROM cash_bundles b
+                  JOIN exchanges e ON e.id = b.origin
+                 WHERE b.owner = %s
+                   AND b.ts < now() - INTERVAL %s
+                   AND b.disputed IS NOT TRUE
+                   AND b.locked_for IS NULL
+                   AND b.amount::currency = %s
+              ORDER BY b.owner = e.participant DESC, b.ts
+            """, (participant.id, QUARANTINE, amount.currency))
         withdrawable = sum(b.amount for b in bundles)
         x = -amount
         if x > withdrawable:
@@ -487,6 +501,7 @@ def transfer(db, tipper, tippee, amount, context, **kw):
 def prepare_transfer(db, tipper, tippee, amount, context, wallet_from, wallet_to,
                      team=None, invoice=None, counterpart=None, refund_ref=None, **kw):
     with db.get_cursor() as cursor:
+        cursor.run("LOCK TABLE cash_bundles IN EXCLUSIVE MODE")
         transfer = cursor.one("""
             INSERT INTO transfers
                         (tipper, tippee, amount, context, team, invoice, status,
@@ -502,21 +517,34 @@ def prepare_transfer(db, tipper, tippee, amount, context, wallet_from, wallet_to
 
 def lock_bundles(cursor, transfer, bundles=None, prefer_bundles_from=-1):
     assert transfer.status == 'pre'
-    cursor.run("LOCK TABLE cash_bundles IN EXCLUSIVE MODE")
     tipper, tippee = transfer.tipper, transfer.tippee
     currency = transfer.amount.currency
-    bundles = bundles or cursor.all("""
-        SELECT b.*
-          FROM cash_bundles b
-          JOIN exchanges e ON e.id = b.origin
-         WHERE b.owner = %(tipper)s
-           AND b.withdrawal IS NULL
-           AND b.locked_for IS NULL
-           AND b.amount::currency = %(currency)s
-      ORDER BY b.origin = %(prefer_bundles_from)s DESC
-             , e.participant = %(tippee)s DESC
-             , b.ts
-    """, locals())
+    if bundles:
+        # Refetch the bundles to ensure validity and prevent race conditions
+        bundles = tuple(bundles)
+        bundles = cursor.all("""
+            SELECT b.*
+              FROM cash_bundles b
+             WHERE b.owner = %(tipper)s
+               AND b.withdrawal IS NULL
+               AND b.locked_for IS NULL
+               AND b.amount::currency = %(currency)s
+               AND b.id IN %(bundles)s
+          ORDER BY b.ts
+        """, locals())
+    else:
+        bundles = cursor.all("""
+            SELECT b.*
+              FROM cash_bundles b
+              JOIN exchanges e ON e.id = b.origin
+             WHERE b.owner = %(tipper)s
+               AND b.withdrawal IS NULL
+               AND b.locked_for IS NULL
+               AND b.amount::currency = %(currency)s
+          ORDER BY b.origin = %(prefer_bundles_from)s DESC
+                 , e.participant = %(tippee)s DESC
+                 , b.ts
+        """, locals())
     transferable = sum(b.amount for b in bundles)
     x = transfer.amount
     if x > transferable:
@@ -774,13 +802,13 @@ def refund_payin(db, exchange, create_debts=False, refund_fee=False, dry_run=Fal
     if not (e_refund and e_refund.status == 'pre'):
         with db.get_cursor() as cursor:
             cursor.run("LOCK TABLE cash_bundles IN EXCLUSIVE MODE")
-            bundles = [NS(d._asdict()) for d in cursor.all("""
-                SELECT *
+            bundles = cursor.all("""
+                SELECT id
                   FROM cash_bundles
                  WHERE origin = %s
                    AND wallet_id = %s
                    AND disputed = true
-            """, (exchange.id, exchange.wallet_id))]
+            """, (exchange.id, exchange.wallet_id))
             e_refund = cursor.one("""
                 INSERT INTO exchanges
                             (participant, amount, fee, vat, route, status, refund_ref, wallet_id)
@@ -844,6 +872,7 @@ def return_payin_bundles_to_origin(db, exchange, last_resort_payer, create_debts
             create_debt(db, withdrawer, payer, amount, exchange.id)
             create_debt(db, original_owner, withdrawer, amount, exchange.id)
         else:
+            bundles = [b.id for b in bundles]
             payer = current_owner
             if create_debts:
                 create_debt(db, original_owner, payer, amount, exchange.id)
diff --git a/liberapay/models/__init__.py b/liberapay/models/__init__.py
--- a/liberapay/models/__init__.py
+++ b/liberapay/models/__init__.py
@@ -311,6 +311,31 @@ def clean_up_counters(db):
 DB.clean_up_counters = clean_up_counters
 
 
+DB_LOCKS = {
+    'payday': 1,
+    'dispute_callback': 2,
+}
+
+@contextmanager
+def acquire_db_lock(db, lock_name, blocking=True):
+    lock_id = DB_LOCKS[lock_name]
+    with db.get_cursor() as cursor:
+        if blocking:
+            cursor.run("SELECT pg_advisory_lock(%s)", (lock_id,))
+        else:
+            locked = cursor.one("SELECT pg_try_advisory_lock(%s)", (lock_id,))
+            assert locked, "failed to acquire the %s lock" % lock_name
+        try:
+            yield cursor
+        finally:
+            try:
+                cursor.run("SELECT pg_advisory_unlock(%s)", (lock_id,))
+            except:
+                pass
+
+DB.lock = acquire_db_lock
+
+
 if __name__ == '__main__':
     from liberapay import wireup
     db = wireup.minimal_algorithm.run()['db']


Test patch: