Repository: liberapay/liberapay.com

Issue:
Unify notification and email queues
For historical reasons we currently have two tables, `email_queue` and `notification_queue`. They should be merged now that web notifications and emails are both rendered using the "simplates" of the `emails/` directory.
- [ ] add a `branch.sql` to merge `email_queue` and `notification_queue` into `notifications`
- [ ] add two boolean columns: `email` and `web`, signalling where the notification is supposed to go
- [ ] add a boolean `email_sent` column
- [ ] modify the email cron to use the `email` and `email_sent` columns
- [ ] modify the notifications code to only render the ones who have the `web` flag on
- [ ] add tests



Discussion:

Resolving patch:
diff --git a/liberapay/models/participant.py b/liberapay/models/participant.py
--- a/liberapay/models/participant.py
+++ b/liberapay/models/participant.py
@@ -798,14 +798,6 @@ def render(t, context):
         website.log_email(message)
         return n
 
-    def queue_email(self, spt_name, **context):
-        context = serialize(context)
-        self.db.run("""
-            INSERT INTO email_queue
-                        (participant, spt_name, context)
-                 VALUES (%s, %s, %s)
-        """, (self.id, spt_name, context))
-
     @classmethod
     def dequeue_emails(cls):
         with cls.db.get_cursor() as cursor:
@@ -815,13 +807,16 @@ def dequeue_emails(cls):
     def _dequeue_emails(cls, cursor):
         fetch_messages = lambda last_id: cursor.all("""
             SELECT *
-              FROM email_queue
+              FROM notifications
              WHERE id > %s
+               AND email AND email_sent IS NOT true
           ORDER BY id ASC
              LIMIT 60
         """, (last_id,))
-        delete = lambda m: cls.db.run(
-            "DELETE FROM email_queue WHERE id = %s", (m.id,)
+        dequeue = lambda m: cls.db.run(
+            "DELETE FROM notifications WHERE id = %s" if not m.web else
+            "UPDATE notifications SET email_sent = true WHERE id = %s",
+            (m.id,)
         )
         last_id = 0
         while True:
@@ -833,15 +828,15 @@ def _dequeue_emails(cls, cursor):
                 p = cls.from_id(msg.participant)
                 email = d.get('email') or p.email
                 if not email:
-                    delete(msg)
+                    dequeue(msg)
                     continue
                 try:
-                    r = p.send_email(msg.spt_name, email, **d)
+                    r = p.send_email(msg.event, email, **d)
                     assert r == 1
                 except Exception as e:
                     website.tell_sentry(e, {})
                 else:
-                    delete(msg)
+                    dequeue(msg)
                 sleep(1)
             last_id = messages[-1].id
 
@@ -860,21 +855,18 @@ def set_email_lang(self, accept_lang, cursor=None):
     # Notifications
     # =============
 
-    def notify(self, event, force_email=False, web=True, **context):
-        if force_email or self.email_notif_bits & EVENTS.get(event).bit:
-            self.queue_email(event, **context)
-        if web:
-            return self.add_notification(event, **context)
-
-    def add_notification(self, event, **context):
+    def notify(self, event, force_email=False, email=True, web=True, **context):
+        email = email and (force_email or self.email_notif_bits & EVENTS.get(event).bit > 0)
         p_id = self.id
         context = serialize(context)
         n_id = self.db.one("""
-            INSERT INTO notification_queue
-                        (participant, event, context)
-                 VALUES (%(p_id)s, %(event)s, %(context)s)
+            INSERT INTO notifications
+                        (participant, event, context, web, email)
+                 VALUES (%(p_id)s, %(event)s, %(context)s, %(web)s, %(email)s)
               RETURNING id;
         """, locals())
+        if not web:
+            return n_id
         pending_notifs = self.db.one("""
             UPDATE participants
                SET pending_notifs = pending_notifs + 1
@@ -888,11 +880,12 @@ def mark_notification_as_read(self, n_id):
         p_id = self.id
         r = self.db.one("""
             WITH updated AS (
-                UPDATE notification_queue
+                UPDATE notifications
                    SET is_new = false
                  WHERE participant = %(p_id)s
                    AND id = %(n_id)s
                    AND is_new
+                   AND web
              RETURNING id
             )
             UPDATE participants
@@ -913,10 +906,11 @@ def mark_notifications_as_read(self, event=None, until=None):
 
         r = self.db.one("""
             WITH updated AS (
-                UPDATE notification_queue
+                UPDATE notifications
                    SET is_new = false
                  WHERE participant = %(p_id)s
                    AND is_new
+                   AND web
                    {0}
              RETURNING id
             )
@@ -931,9 +925,10 @@ def remove_notification(self, n_id):
         p_id = self.id
         r = self.db.one("""
             WITH deleted AS (
-                DELETE FROM notification_queue
+                DELETE FROM notifications
                  WHERE id = %(n_id)s
                    AND participant = %(p_id)s
+                   AND web
              RETURNING is_new
             )
             UPDATE participants
@@ -963,8 +958,9 @@ def fill_notification_context(self, context):
     def get_notifs(self):
         return self.db.all("""
             SELECT id, event, context, is_new, ts
-              FROM notification_queue
+              FROM notifications
              WHERE participant = %s
+               AND web
           ORDER BY is_new DESC, id DESC
         """, (self.id,))
 
@@ -1128,9 +1124,9 @@ def send_newsletters(cls):
                     for s in cls.get_subscriptions(msg.sender):
                         context = dict(msg.context, unsubscribe_url=s.unsubscribe_url)
                         count += cursor.one("""
-                            INSERT INTO email_queue
-                                        (participant, spt_name, context)
-                                 SELECT p.id, 'newsletter', %s
+                            INSERT INTO notifications
+                                        (participant, event, context, web, email)
+                                 SELECT p.id, 'newsletter', %s, false, true
                                    FROM participants p
                                   WHERE p.id = %s
                                     AND p.email IS NOT NULL


Test patch:
diff --git a/tests/py/test_email.py b/tests/py/test_email.py
--- a/tests/py/test_email.py
+++ b/tests/py/test_email.py
@@ -241,23 +241,26 @@ def test_html_escaping(self):
         assert 'foo&#39;bar' in last_email['html']
         assert '&#39;' not in last_email['text']
 
+    def queue_email(self, participant, event, **context):
+        return participant.notify(event, force_email=True, web=False, **context)
+
     def test_can_dequeue_an_email(self):
         larry = self.make_participant('larry', email='larry@example.com')
-        larry.queue_email("verification", link='https://example.com/larry')
+        self.queue_email(larry, "verification", link='https://example.com/larry')
 
-        assert self.db.one("SELECT spt_name FROM email_queue") == "verification"
+        assert self.db.one("SELECT event FROM notifications") == "verification"
         Participant.dequeue_emails()
         assert self.mailer.call_count == 1
         last_email = self.get_last_email()
         assert last_email['to'][0] == 'larry <larry@example.com>'
         assert last_email['subject'] == "Email address verification - Liberapay"
-        assert self.db.one("SELECT spt_name FROM email_queue") is None
+        assert self.db.one("SELECT event FROM notifications") is None
 
     def test_dequeueing_an_email_without_address_just_skips_it(self):
         larry = self.make_participant('larry')
-        larry.queue_email("verification")
+        self.queue_email(larry, "verification")
 
-        assert self.db.one("SELECT spt_name FROM email_queue") == "verification"
+        assert self.db.one("SELECT event FROM notifications") == "verification"
         Participant.dequeue_emails()
         assert self.mailer.call_count == 0
-        assert self.db.one("SELECT spt_name FROM email_queue") is None
+        assert self.db.one("SELECT event FROM notifications") is None
diff --git a/tests/py/test_notifications.py b/tests/py/test_notifications.py
--- a/tests/py/test_notifications.py
+++ b/tests/py/test_notifications.py
@@ -6,16 +6,16 @@ class TestNotifications(Harness):
 
     def test_add_notifications(self):
         alice = self.make_participant('alice')
-        alice.add_notification('abcd')
-        alice.add_notification('1234')
+        alice.notify('abcd', email=False)
+        alice.notify('1234', email=False)
         assert alice.pending_notifs == 2
 
     def test_remove_notification(self):
         alice = self.make_participant('alice')
         bob = self.make_participant('bob')
-        alice.add_notification('abcd')
-        id = alice.add_notification('1234')
-        alice.add_notification('bcde')
+        alice.notify('abcd', email=False)
+        id = alice.notify('1234', email=False)
+        alice.notify('bcde', email=False)
 
         # check that bob can't remove alice's notification
         bob.remove_notification(id)
@@ -36,9 +36,10 @@ def test_render_notifications(self):
             """).load(jinja_env_html, None)
         }
         alice = self.make_participant('alice')
-        alice.add_notification('test_event')
-        alice.add_notification(
+        alice.notify('test_event', email=False)
+        alice.notify(
             'team_invite',
+            email=False,
             team='team',
             team_url='fake_url',
             inviter='bob',
@@ -54,15 +55,15 @@ def test_render_notifications(self):
 
     def test_render_unknown_notification(self):
         alice = self.make_participant('alice')
-        alice.add_notification('fake_event_name')
+        alice.notify('fake_event_name', email=False)
         r = self.client.GET('/alice/notifications.html', auth_as=alice,
                             sentry_reraise=False).text
         assert 'fake_event_name' not in r
 
     def test_marking_notifications_as_read_avoids_race_condition(self):
         alice = self.make_participant('alice')
-        n1 = alice.add_notification('low_balance')
-        n2 = alice.add_notification('low_balance')
+        n1 = alice.notify('low_balance', email=False)
+        n2 = alice.notify('low_balance', email=False)
         assert alice.pending_notifs == 2
 
         data = {'mark_all_as_read': 'true', 'until': str(n1)}
@@ -71,9 +72,10 @@ def test_marking_notifications_as_read_avoids_race_condition(self):
 
         notifications = self.db.all("""
             SELECT id, event, context, is_new
-              FROM notification_queue
+              FROM notifications
              WHERE participant = %s
                AND is_new = true
+               AND web
           ORDER BY id DESC
         """, (alice.id,))
         assert len(notifications) == 1