Repository: liberapay/liberapay.com

Issue:
Duplicate emails
The `Participant.dequeue_emails()` function needs to be modified to prevent the accidental sending of duplicate emails when the application has trouble communicating with the database.


Discussion:

Resolving patch:
diff --git a/liberapay/cron.py b/liberapay/cron.py
--- a/liberapay/cron.py
+++ b/liberapay/cron.py
@@ -21,19 +21,61 @@ class Cron(object):
     def __init__(self, website):
         self.website = website
         self.conn = None
+        self._wait_for_lock_thread = None
         self.has_lock = False
         self.jobs = []
 
     def __call__(self, period, func, exclusive=False):
-        self.jobs.append((period, func, exclusive))
+        job = Job(self, period, func, exclusive)
+        self.jobs.append(job)
         if not self.website.env.run_cron_jobs or not period:
             return
         if exclusive and not self.has_lock:
             self._wait_for_lock()
             return
+        job.start()
+
+    def _wait_for_lock(self):
+        if self.conn:
+            return  # Already waiting
+        self.conn = self.website.db.get_connection().__enter__()
         def f():
-            if isinstance(period, Weekly):
-                while True:
+            cursor = self.conn.cursor()
+            while True:
+                if cursor.one("SELECT pg_try_advisory_lock(0)"):
+                    self.has_lock = True
+                    break
+                sleep(300)
+            for job in self.jobs:
+                if job.exclusive:
+                    job.start()
+        t = self._wait_for_lock_thread = threading.Thread(target=f)
+        t.daemon = True
+        t.start()
+
+
+class Job:
+
+    __slots__ = ('cron', 'period', 'func', 'exclusive', 'thread')
+
+    def __init__(self, cron, period, func, exclusive=False):
+        self.cron = cron
+        self.period = period
+        self.func = func
+        self.exclusive = exclusive
+        self.thread = None
+
+    def __repr__(self):
+        return f"Job(func={self.func!r}, period={self.period!r}, exclusive={self.exclusive!r}, thread={self.thread!r})"
+
+    def start(self):
+        if self.thread and self.thread.is_alive():
+            return
+
+        def f():
+            while True:
+                period = self.period
+                if isinstance(period, Weekly):
                     now = datetime.utcnow()
                     then = now.replace(hour=period.hour, minute=10, second=0)
                     days = (period.weekday - now.isoweekday()) % 7
@@ -45,13 +87,7 @@ def f():
                     elif seconds < -60:
                         sleep(86400 * 6)
                         continue
-                    try:
-                        func()
-                    except Exception as e:
-                        self.website.tell_sentry(e, {})
-                    sleep(86400 * 6)
-            elif isinstance(period, Daily):
-                while True:
+                elif isinstance(period, Daily):
                     now = datetime.utcnow()
                     then = now.replace(hour=period.hour, minute=5, second=0)
                     seconds = (then - now).total_seconds()
@@ -61,57 +97,32 @@ def f():
                     elif seconds < -60:
                         # tomorrow
                         sleep(3600 * 24 + seconds)
-                    try:
-                        func()
-                    except Exception as e:
-                        self.website.tell_sentry(e, {})
-                        # retry in 5 minutes
-                        sleep(300)
-                    else:
-                        # success, sleep until tomorrow
-                        sleep(3600 * 23)
-            elif period == 'once':
-                while True:
-                    try:
-                        r = func()
-                    except Exception as e:
-                        self.website.tell_sentry(e, {})
-                    else:
-                        if r is CRON_ENCORE:
-                            sleep(2)
-                            continue
+                try:
+                    test_hook()
+                    r = self.func()
+                except Exception as e:
+                    self.cron.website.tell_sentry(e, {})
+                    # retry in 5 minutes
+                    sleep(300)
+                    continue
+                else:
+                    if r is CRON_ENCORE:
+                        sleep(2)
+                        continue
+                    if r is CRON_STOP:
+                        return
+                if period == 'once':
                     return
-            else:
-                while True:
-                    try:
-                        r = func()
-                    except Exception as e:
-                        self.website.tell_sentry(e, {})
-                    else:
-                        if r is CRON_ENCORE:
-                            sleep(2)
-                            continue
-                        if r is CRON_STOP:
-                            return
+                elif isinstance(period, (float, int)):
                     sleep(period)
-        t = threading.Thread(target=f, name=func.__name__)
-        t.daemon = True
-        t.start()
+                else:
+                    sleep(3600 * 23)
 
-    def _wait_for_lock(self):
-        if self.conn:
-            return  # Already waiting
-        self.conn = self.website.db.get_connection().__enter__()
-        def f():
-            cursor = self.conn.cursor()
-            while True:
-                if cursor.one("SELECT pg_try_advisory_lock(0)"):
-                    self.has_lock = True
-                    break
-                sleep(300)
-            for period, func, exclusive in self.jobs:
-                if exclusive:
-                    self(period, func, exclusive=True)
-        t = threading.Thread(target=f)
+        t = self.thread = threading.Thread(target=f, name=self.func.__name__)
         t.daemon = True
         t.start()
+        return t
+
+
+def test_hook():
+    pass
diff --git a/liberapay/models/participant.py b/liberapay/models/participant.py
--- a/liberapay/models/participant.py
+++ b/liberapay/models/participant.py
@@ -21,7 +21,7 @@
 from pando import json, Response
 from pando.utils import utcnow
 from postgres.orm import Model
-from psycopg2 import IntegrityError
+from psycopg2.errors import IntegrityError, ReadOnlySqlTransaction
 from psycopg2.extras import execute_batch, execute_values
 import requests
 
@@ -1298,36 +1298,57 @@ def dequeue_emails(cls):
             SELECT *
               FROM notifications
              WHERE id > %s
-               AND email AND email_sent IS NULL
+               AND email AND email_status = 'queued'
           ORDER BY id ASC
              LIMIT 60
         """, (last_id,))
-        dequeue = lambda m, sent: cls.db.run(
-            "UPDATE notifications SET email_sent = %(sent)s WHERE id = %(id)s",
-            dict(id=m.id, sent=sent)
-        )
+        def dequeue(msg, status):
+            try:
+                return cls.db.run(
+                    "UPDATE notifications SET email_status = %(status)s WHERE id = %(id)s",
+                    dict(id=msg.id, status=status)
+                )
+            except Exception as e:
+                website.tell_sentry(e, {})
+                sleep(5)
+                return dequeue(msg, status)
         last_id = 0
         while True:
             messages = fetch_messages(last_id)
             if not messages:
                 break
             for msg in messages:
+                try:
+                    r = cls.db.one("""
+                        UPDATE notifications
+                           SET email_status = 'sending'
+                         WHERE id = %s
+                           AND email_status = 'queued'
+                     RETURNING email_status
+                    """, (msg.id,))
+                except ReadOnlySqlTransaction:
+                    # The database is in read-only mode, give up for now
+                    return
+                if not r:
+                    # Message already (being) sent by another thread
+                    continue
                 d = deserialize(msg.context)
                 d['notification_ts'] = msg.ts
                 p = cls.from_id(msg.participant)
                 email = d.get('email') or p.email
                 if not email:
-                    dequeue(msg, False)
+                    dequeue(msg, 'skipped')
                     continue
                 email_row = p.get_email(email)
                 try:
                     p.send_email(msg.event, email_row, **d)
                 except EmailAddressIsBlacklisted:
-                    dequeue(msg, False)
+                    dequeue(msg, 'skipped')
                 except Exception as e:
                     website.tell_sentry(e, {})
+                    dequeue(msg, 'failed')
                 else:
-                    dequeue(msg, True)
+                    dequeue(msg, 'sent')
                 sleep(1)
             last_id = messages[-1].id
         # Delete old email-only notifications
@@ -1376,10 +1397,11 @@ def notify(self, event, force_email=False, email=True, web=True, idem_key=None,
             )
             email = False
         # Okay, add the notification to the queue
+        email_status = 'queued' if email else None
         n_id = self.db.one("""
             INSERT INTO notifications
-                        (participant, event, context, web, email, idem_key)
-                 VALUES (%(p_id)s, %(event)s, %(context)s, %(web)s, %(email)s, %(idem_key)s)
+                        (participant, event, context, web, email, email_status, idem_key)
+                 VALUES (%(p_id)s, %(event)s, %(context)s, %(web)s, %(email)s, %(email_status)s, %(idem_key)s)
               RETURNING id;
         """, locals())
         if not web:
@@ -1744,8 +1766,8 @@ def send_newsletters(cls):
                         context = dict(msg.context, unsubscribe_url=s.unsubscribe_url)
                         count += cursor.one("""
                             INSERT INTO notifications
-                                        (participant, event, context, web, email)
-                                 SELECT p.id, 'newsletter', %s, false, true
+                                        (participant, event, context, web, email, email_status)
+                                 SELECT p.id, 'newsletter', %s, false, true, 'queued'
                                    FROM participants p
                                   WHERE p.id = %s
                                     AND p.email IS NOT NULL


Test patch:
diff --git a/liberapay/testing/__init__.py b/liberapay/testing/__init__.py
--- a/liberapay/testing/__init__.py
+++ b/liberapay/testing/__init__.py
@@ -4,8 +4,10 @@
 from contextlib import contextmanager
 import itertools
 import json
-import unittest
 from os.path import dirname, join, realpath
+import threading
+import unittest
+from unittest.mock import patch
 
 import html5lib
 from pando.utils import utcnow
@@ -495,6 +497,26 @@ def make_invoice(self, sender, addressee, amount, status):
 class Foobar(Exception): pass
 
 
+@contextmanager
+def fake_sleep(target='time.sleep', raise_after=None, exception=StopIteration()):
+
+    def no_sleep(seconds):
+        thread = threading.current_thread()
+        sleep_count = getattr(thread, '_liberapay_test_sleep_count', 0)
+        if raise_after is not None and sleep_count >= raise_after:
+            raise exception
+        thread._liberapay_test_sleep_count = sleep_count + 1
+
+    with patch(target) as sleep:
+        sleep.side_effect = no_sleep
+        try:
+            yield sleep
+        finally:
+            thread = threading.current_thread()
+            if hasattr(thread, '_liberapay_test_sleep_count'):
+                del thread._liberapay_test_sleep_count
+
+
 @contextmanager
 def postgres_readonly(db):
     db.readonly = True
diff --git a/tests/py/test_cron.py b/tests/py/test_cron.py
--- a/tests/py/test_cron.py
+++ b/tests/py/test_cron.py
@@ -1,16 +1,88 @@
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
 from psycopg2.extras import execute_values
 
-from liberapay.cron import Weekly
+from liberapay.cron import Daily, Weekly
 from liberapay.i18n.currencies import fetch_currency_exchange_rates
-from liberapay.testing import Harness
+from liberapay.testing import fake_sleep, Harness
+
+
+utcnow = datetime.utcnow
 
 
 class TestCronJobs(Harness):
 
-    def test_cron_jobs_with_empty_db(self):
-        for period, func, exclusive in self.website.cron.jobs:
-            if not isinstance(period, Weekly) and func is not fetch_currency_exchange_rates:
-                func()
+    @fake_sleep(target='liberapay.cron.sleep', raise_after=0)
+    @patch('liberapay.cron.datetime', autospec=True)
+    def test_cron_jobs_with_empty_db(self, datetime):
+        now = utcnow()
+        for job in self.website.cron.jobs:
+            print(job)
+            with patch.object(job, 'func', autospec=True) as mock_func:
+                if isinstance(job.period, Weekly):
+                    datetime.utcnow.return_value = (
+                        now.replace(hour=job.period.hour, minute=10, second=0) +
+                        timedelta(days=(job.period.weekday - now.isoweekday()) % 7)
+                    )
+                elif isinstance(job.period, Daily):
+                    datetime.utcnow.return_value = now.replace(
+                        hour=job.period.hour, minute=5, second=0
+                    )
+                if isinstance(job.period, Weekly) or job.func is fetch_currency_exchange_rates:
+                    mock_func.return_value = None
+                job.start()
+                job.thread.join()
+                assert mock_func.call_count == 1
+
+    @fake_sleep(target='liberapay.cron.sleep', raise_after=0)
+    @patch('liberapay.cron.datetime', autospec=True)
+    def test_weekly_and_daily_cron_jobs_at_the_wrong_time(self, datetime):
+        now = utcnow()
+        for job in self.website.cron.jobs:
+            with patch.object(job, 'func', autospec=True) as mock_func:
+                mock_func.return_value = None
+                if isinstance(job.period, Weekly):
+                    days_delta = timedelta(days=(job.period.weekday - now.isoweekday()) % 7)
+                    # Wrong day
+                    datetime.utcnow.return_value = (
+                        now.replace(hour=job.period.hour, minute=10, second=0) +
+                        days_delta + timedelta(days=1)
+                    )
+                    job.start()
+                    job.thread.join()
+                    assert mock_func.call_count == 0
+                    # Not yet time
+                    datetime.utcnow.return_value = (
+                        now.replace(hour=job.period.hour, minute=0, second=0) +
+                        days_delta
+                    )
+                    job.start()
+                    job.thread.join()
+                    assert mock_func.call_count == 0
+                    # Too late
+                    datetime.utcnow.return_value = (
+                        now.replace(hour=job.period.hour, minute=20, second=0) +
+                        days_delta
+                    )
+                    job.start()
+                    job.thread.join()
+                    assert mock_func.call_count == 0
+                elif isinstance(job.period, Daily):
+                    # Not yet time
+                    datetime.utcnow.return_value = utcnow().replace(
+                        hour=job.period.hour, minute=0, second=0
+                    )
+                    job.start()
+                    job.thread.join()
+                    assert mock_func.call_count == 0
+                    # Too late
+                    datetime.utcnow.return_value = utcnow().replace(
+                        hour=job.period.hour, minute=10, second=0
+                    )
+                    job.start()
+                    job.thread.join()
+                    assert mock_func.call_count == 0
 
     def test_fetch_currency_exchange_rates(self):
         fake_rates = self.db.all("SELECT * FROM currency_exchange_rates")
diff --git a/tests/py/test_email.py b/tests/py/test_email.py
--- a/tests/py/test_email.py
+++ b/tests/py/test_email.py
@@ -11,6 +11,7 @@
 )
 from liberapay.models.participant import Participant
 from liberapay.security.authentication import ANON
+from liberapay.testing import postgres_readonly
 from liberapay.testing.emails import EmailHarness
 from liberapay.utils.emails import EmailVerificationResult, check_email_blacklist
 
@@ -404,7 +405,7 @@ def test_can_dequeue_an_email(self):
         assert self.mailer.call_count == 1
         last_email = self.get_last_email()
         assert last_email['to'][0] == 'larry <larry@example.com>'
-        assert self.db.one("SELECT email_sent FROM notifications") is True
+        assert self.db.one("SELECT email_status FROM notifications") == 'sent'
 
     def test_dequeueing_an_email_without_address_just_skips_it(self):
         larry = self.make_participant('larry')
@@ -412,7 +413,15 @@ def test_dequeueing_an_email_without_address_just_skips_it(self):
 
         Participant.dequeue_emails()
         assert self.mailer.call_count == 0
-        assert self.db.one("SELECT email_sent FROM notifications") is False
+        assert self.db.one("SELECT email_status FROM notifications") == 'skipped'
+
+    def test_emails_are_not_sent_went_database_is_read_only(self):
+        larry = self.make_participant('larry')
+        self.queue_email(larry, 'team_invite', team='team', team_url='fake_url', inviter='bob')
+        with postgres_readonly(self.db):
+            Participant.dequeue_emails()
+            assert self.mailer.call_count == 0
+            assert self.db.one("SELECT email_status FROM notifications") == 'queued'
 
     def test_blacklisting_email_domain(self):
         admin = self.make_participant('admin', privileges=1)
diff --git a/tests/py/test_wireup.py b/tests/py/test_wireup.py
--- a/tests/py/test_wireup.py
+++ b/tests/py/test_wireup.py
@@ -1,7 +1,8 @@
+from datetime import timedelta
 from importlib import reload
+import sys
 import unittest
 from unittest.mock import patch
-import sys
 
 
 class Test(unittest.TestCase):
@@ -27,6 +28,28 @@ def test_01_main_doesnt_fail_when_db_is_down(self):
             # Check that the website.platforms attribute exists
             assert website.platforms is website.db
 
+    @patch.dict('os.environ', {'RUN_CRON_JOBS': 'yes'})
+    def test_98_main_starts_cron_jobs(self):
+        import liberapay.cron
+        now = liberapay.cron.datetime.utcnow()
+        datetime_patch = patch.object(liberapay.cron, 'datetime', autospec=True)
+        sleep_patch = patch.object(liberapay.cron, 'sleep')
+        test_hook_patch = patch.object(liberapay.cron, 'test_hook')
+        with datetime_patch as datetime, sleep_patch as sleep, test_hook_patch as test_hook:
+            datetime.utcnow.return_value = now
+            def forward_time(seconds):
+                datetime.utcnow.return_value += timedelta(seconds=seconds)
+            sleep.side_effect = forward_time
+            test_hook.side_effect = StopIteration()
+            from liberapay.main import website
+            assert website.cron
+            assert website.cron.jobs
+            website.cron._wait_for_lock_thread.join()
+            for job in website.cron.jobs:
+                print(job)
+                if job.period:
+                    job.thread.join()
+
     def test_99_main_can_be_reloaded(self):
         import liberapay.main
         reload(liberapay.main)