Repository: zulip/zulip

Issue:
Fix emails when disabling "Include message content in missed message notifications"
I got a report from a user in the Recurse Center who was confused as to why they were getting missed-message emails that didn't contain any content.  The emails also incorrectly claimed one couldn't reply because missed-message emails weren't configured on the server.  There are 2 things we should do to fix (mostly centered on this block of code:
```
    if not message_content_allowed_in_missedmessage_emails(user_profile):                            
        context.update({                                                                             
            'reply_to_zulip': False,                                                                 
            'messages': [],                                                                          
            'sender_str': "",                                                                        
            'realm_str': user_profile.realm.name,                                                    
            'huddle_display_name': "",                                                               
```

* [x] In this case, rather than the email displaying `Reply in Zulip\nDo not reply to this message. This Zulip server is not configured to accept incoming emails`, we should say "."Reply in Zulip\nThis email does not include message content because you have disabled [message content appearing in email notifications](https://zulipchat.com/help/pm-mention-alert-notifications)."
* [x] We should potentially move that text from the footer to the message body, rather than having no body?
* [x] We also should change s/Do not reply to this message/Do not reply to this email/ for better clarity.

@hackerkid can you take charge of fixing this?


Discussion:
Hello @zulip/server-development members, this issue was labeled with the "area: emails" label, so you may want to check it out!

<!-- areaLabelAddition -->
@zulipbot claim
Hello @hackerkid, you claimed this issue to work on it, but this issue and any referenced pull requests haven't been updated for 10 days. Are you still working on this issue?

If so, please update this issue by leaving a comment on this issue to let me know that you're still working on it. Otherwise, I'll automatically remove you from this issue in 4 days.

If you've decided to work on something else, simply comment `@zulipbot abandon` so that someone else can claim it and continue from where you left off.

Thank you for your valuable contributions to Zulip!

<!-- inactiveWarning -->

Resolving patch:
diff --git a/version.py b/version.py
--- a/version.py
+++ b/version.py
@@ -26,4 +26,4 @@
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '67.2'
+PROVISION_VERSION = '67.3'
diff --git a/zerver/lib/email_notifications.py b/zerver/lib/email_notifications.py
--- a/zerver/lib/email_notifications.py
+++ b/zerver/lib/email_notifications.py
@@ -323,7 +323,6 @@ def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
         'message_count': message_count,
         'unsubscribe_link': unsubscribe_link,
         'realm_name_in_notifications': user_profile.realm_name_in_notifications,
-        'show_message_content': message_content_allowed_in_missedmessage_emails(user_profile)
     })
 
     triggers = list(message['trigger'] for message in missed_messages)
@@ -396,18 +395,23 @@ def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
 
     # If message content is disabled, then flush all information we pass to email.
     if not message_content_allowed_in_missedmessage_emails(user_profile):
+        realm = user_profile.realm
         context.update({
             'reply_to_zulip': False,
             'messages': [],
             'sender_str': "",
-            'realm_str': user_profile.realm.name,
+            'realm_str': realm.name,
             'huddle_display_name': "",
+            'show_message_content': False,
+            'message_content_disabled_by_user': not user_profile.message_content_in_email_notifications,
+            'message_content_disabled_by_realm': not realm.message_content_allowed_in_email_notifications,
         })
     else:
         context.update({
             'messages': build_message_list(user_profile, list(m['message'] for m in missed_messages)),
             'sender_str': ", ".join(sender.full_name for sender in senders),
             'realm_str': user_profile.realm.name,
+            'show_message_content': True,
         })
 
     from_name = "Zulip missed messages"  # type: str


Test patch:
diff --git a/zerver/tests/test_email_notifications.py b/zerver/tests/test_email_notifications.py
--- a/zerver/tests/test_email_notifications.py
+++ b/zerver/tests/test_email_notifications.py
@@ -136,7 +136,7 @@ def normalize_string(self, s: str) -> str:
     def _get_tokens(self) -> List[str]:
         return ['mm' + str(random.getrandbits(32)) for _ in range(30)]
 
-    def _test_cases(self, msg_id: int, body: str, email_subject: str,
+    def _test_cases(self, msg_id: int, verify_body_include: List[str], email_subject: str,
                     send_as_user: bool, verify_html_body: bool=False,
                     show_message_content: bool=True,
                     verify_body_does_not_include: Optional[List[str]]=None,
@@ -161,9 +161,11 @@ def _test_cases(self, msg_id: int, body: str, email_subject: str,
         self.assertEqual(len(msg.reply_to), 1)
         self.assertIn(msg.reply_to[0], reply_to_emails)
         if verify_html_body:
-            self.assertIn(body, self.normalize_string(msg.alternatives[0][0]))
+            for text in verify_body_include:
+                self.assertIn(text, self.normalize_string(msg.alternatives[0][0]))
         else:
-            self.assertIn(body, self.normalize_string(msg.body))
+            for text in verify_body_include:
+                self.assertIn(text, self.normalize_string(msg.body))
         if verify_body_does_not_include is not None:
             for text in verify_body_does_not_include:
                 self.assertNotIn(text, self.normalize_string(msg.body))
@@ -174,12 +176,12 @@ def _realm_name_in_missed_message_email_subject(self, realm_name_in_notification
             self.example_email('hamlet'),
             'Extremely personal message!',
         )
-        body = 'Extremely personal message!'
+        verify_body_include = ['Extremely personal message!']
         email_subject = 'PMs with Othello, the Moor of Venice'
 
         if realm_name_in_notifications:
             email_subject = 'PMs with Othello, the Moor of Venice [Zulip Dev]'
-        self._test_cases(msg_id, body, email_subject, False)
+        self._test_cases(msg_id, verify_body_include, email_subject, False)
 
     def _extra_context_in_missed_stream_messages_mention(self, send_as_user: bool,
                                                          show_message_content: bool=True) -> None:
@@ -193,18 +195,26 @@ def _extra_context_in_missed_stream_messages_mention(self, send_as_user: bool,
             '@**King Hamlet**')
 
         if show_message_content:
-            body = ("Othello, the Moor of Venice: 1 2 3 4 5 6 7 8 9 10 @**King Hamlet** -- "
-                    "You are receiving this because you were mentioned in Zulip Dev.")
+            verify_body_include = [
+                "Othello, the Moor of Venice: 1 2 3 4 5 6 7 8 9 10 @**King Hamlet** -- ",
+                "You are receiving this because you were mentioned in Zulip Dev."
+            ]
             email_subject = '#Denmark > test'
             verify_body_does_not_include = []  # type: List[str]
         else:
             # Test in case if message content in missed email message are disabled.
-            body = 'Manage email preferences: http://zulip.testserver/#settings/notifications'
+            verify_body_include = [
+                "This email does not include message content because you have disabled message ",
+                "http://zulip.testserver/help/pm-mention-alert-notifications ",
+                "View or reply in Zulip",
+                " Manage email preferences: http://zulip.testserver/#settings/notifications"
+            ]
+
             email_subject = 'New missed messages'
             verify_body_does_not_include = ['Denmark > test', 'Othello, the Moor of Venice',
                                             '1 2 3 4 5 6 7 8 9 10 @**King Hamlet**', 'private', 'group',
                                             'Reply to this email directly, or view it in Zulip']
-        self._test_cases(msg_id, body, email_subject, send_as_user,
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user,
                          show_message_content=show_message_content,
                          verify_body_does_not_include=verify_body_does_not_include,
                          trigger='mentioned')
@@ -221,18 +231,25 @@ def _extra_context_in_missed_stream_messages_wildcard_mention(self, send_as_user
             '@**all**')
 
         if show_message_content:
-            body = ("Othello, the Moor of Venice: 1 2 3 4 5 @**all** -- "
-                    "You are receiving this because you were mentioned in Zulip Dev.")
+            verify_body_include = [
+                "Othello, the Moor of Venice: 1 2 3 4 5 @**all** -- ",
+                "You are receiving this because you were mentioned in Zulip Dev."
+            ]
             email_subject = '#Denmark > test'
             verify_body_does_not_include = []  # type: List[str]
         else:
             # Test in case if message content in missed email message are disabled.
-            body = 'Manage email preferences: http://zulip.testserver/#settings/notifications'
+            verify_body_include = [
+                "This email does not include message content because you have disabled message ",
+                "http://zulip.testserver/help/pm-mention-alert-notifications ",
+                "View or reply in Zulip",
+                " Manage email preferences: http://zulip.testserver/#settings/notifications"
+            ]
             email_subject = 'New missed messages'
             verify_body_does_not_include = ['Denmark > test', 'Othello, the Moor of Venice',
                                             '1 2 3 4 5 @**all**', 'private', 'group',
                                             'Reply to this email directly, or view it in Zulip']
-        self._test_cases(msg_id, body, email_subject, send_as_user,
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user,
                          show_message_content=show_message_content,
                          verify_body_does_not_include=verify_body_does_not_include,
                          trigger='wildcard_mentioned')
@@ -246,10 +263,12 @@ def _extra_context_in_missed_stream_messages_email_notify(self, send_as_user: bo
         msg_id = self.send_stream_message(
             self.example_email('othello'), "denmark",
             '12')
-        body = ("Othello, the Moor of Venice: 1 2 3 4 5 6 7 8 9 10 12 -- "
-                "You are receiving this because you have email notifications enabled for this stream.")
+        verify_body_include = [
+            "Othello, the Moor of Venice: 1 2 3 4 5 6 7 8 9 10 12 -- ",
+            "You are receiving this because you have email notifications enabled for this stream."
+        ]
         email_subject = '#Denmark > test'
-        self._test_cases(msg_id, body, email_subject, send_as_user, trigger='stream_email_notify')
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user, trigger='stream_email_notify')
 
     def _extra_context_in_missed_stream_messages_mention_two_senders(self, send_as_user: bool) -> None:
         for i in range(0, 3):
@@ -257,13 +276,17 @@ def _extra_context_in_missed_stream_messages_mention_two_senders(self, send_as_u
         msg_id = self.send_stream_message(
             self.example_email('othello'), "Denmark",
             '@**King Hamlet**')
-        body = ("Cordelia Lear: 0 1 2 Othello, the Moor of Venice: @**King Hamlet** -- "
-                "You are receiving this because you were mentioned in Zulip Dev.")
+        verify_body_include = [
+            "Cordelia Lear: 0 1 2 Othello, the Moor of Venice: @**King Hamlet** -- ",
+            "You are receiving this because you were mentioned in Zulip Dev."
+        ]
         email_subject = '#Denmark > test'
-        self._test_cases(msg_id, body, email_subject, send_as_user, trigger='mentioned')
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user, trigger='mentioned')
 
     def _extra_context_in_personal_missed_stream_messages(self, send_as_user: bool,
-                                                          show_message_content: bool=True) -> None:
+                                                          show_message_content: bool=True,
+                                                          message_content_disabled_by_user: bool=False,
+                                                          message_content_disabled_by_realm: bool=False) -> None:
         msg_id = self.send_personal_message(
             self.example_email('othello'),
             self.example_email('hamlet'),
@@ -271,15 +294,28 @@ def _extra_context_in_personal_missed_stream_messages(self, send_as_user: bool,
         )
 
         if show_message_content:
-            body = 'Extremely personal message!'
+            verify_body_include = ['Extremely personal message!']
             email_subject = 'PMs with Othello, the Moor of Venice'
             verify_body_does_not_include = []  # type: List[str]
         else:
-            body = 'Manage email preferences: http://zulip.testserver/#settings/notifications'
+            if message_content_disabled_by_realm:
+                verify_body_include = [
+                    "This email does not include message content because your organization has disabled",
+                    "http://zulip.testserver/help/hide-message-content-in-emails",
+                    "View or reply in Zulip",
+                    " Manage email preferences: http://zulip.testserver/#settings/notifications"
+                ]
+            elif message_content_disabled_by_user:
+                verify_body_include = [
+                    "This email does not include message content because you have disabled message ",
+                    "http://zulip.testserver/help/pm-mention-alert-notifications ",
+                    "View or reply in Zulip",
+                    " Manage email preferences: http://zulip.testserver/#settings/notifications"
+                ]
             email_subject = 'New missed messages'
             verify_body_does_not_include = ['Othello, the Moor of Venice', 'Extremely personal message!',
                                             'mentioned', 'group', 'Reply to this email directly, or view it in Zulip']
-        self._test_cases(msg_id, body, email_subject, send_as_user,
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user,
                          show_message_content=show_message_content,
                          verify_body_does_not_include=verify_body_does_not_include)
 
@@ -289,9 +325,9 @@ def _reply_to_email_in_personal_missed_stream_messages(self, send_as_user: bool)
             self.example_email('hamlet'),
             'Extremely personal message!',
         )
-        body = 'Reply to this email directly, or view it in Zulip'
+        verify_body_include = ['Reply to this email directly, or view it in Zulip']
         email_subject = 'PMs with Othello, the Moor of Venice'
-        self._test_cases(msg_id, body, email_subject, send_as_user)
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user)
 
     def _reply_warning_in_personal_missed_stream_messages(self, send_as_user: bool) -> None:
         msg_id = self.send_personal_message(
@@ -299,9 +335,9 @@ def _reply_warning_in_personal_missed_stream_messages(self, send_as_user: bool)
             self.example_email('hamlet'),
             'Extremely personal message!',
         )
-        body = 'Do not reply to this message.'
+        verify_body_include = ['Do not reply to this email.']
         email_subject = 'PMs with Othello, the Moor of Venice'
-        self._test_cases(msg_id, body, email_subject, send_as_user)
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user)
 
     def _extra_context_in_huddle_missed_stream_messages_two_others(self, send_as_user: bool,
                                                                    show_message_content: bool=True) -> None:
@@ -315,16 +351,21 @@ def _extra_context_in_huddle_missed_stream_messages_two_others(self, send_as_use
         )
 
         if show_message_content:
-            body = 'Othello, the Moor of Venice: Group personal message! -- Reply'
+            verify_body_include = ['Othello, the Moor of Venice: Group personal message! -- Reply']
             email_subject = 'Group PMs with Iago and Othello, the Moor of Venice'
             verify_body_does_not_include = []  # type: List[str]
         else:
-            body = 'Manage email preferences: http://zulip.testserver/#settings/notifications'
+            verify_body_include = [
+                "This email does not include message content because you have disabled message ",
+                "http://zulip.testserver/help/pm-mention-alert-notifications ",
+                "View or reply in Zulip",
+                " Manage email preferences: http://zulip.testserver/#settings/notifications"
+            ]
             email_subject = 'New missed messages'
             verify_body_does_not_include = ['Iago', 'Othello, the Moor of Venice Othello, the Moor of Venice',
                                             'Group personal message!', 'mentioned',
                                             'Reply to this email directly, or view it in Zulip']
-        self._test_cases(msg_id, body, email_subject, send_as_user,
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user,
                          show_message_content=show_message_content,
                          verify_body_does_not_include=verify_body_does_not_include)
 
@@ -339,9 +380,9 @@ def _extra_context_in_huddle_missed_stream_messages_three_others(self, send_as_u
             'Group personal message!',
         )
 
-        body = 'Othello, the Moor of Venice: Group personal message! -- Reply'
+        verify_body_include = ['Othello, the Moor of Venice: Group personal message! -- Reply']
         email_subject = 'Group PMs with Cordelia Lear, Iago, and Othello, the Moor of Venice'
-        self._test_cases(msg_id, body, email_subject, send_as_user)
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user)
 
     def _extra_context_in_huddle_missed_stream_messages_many_others(self, send_as_user: bool) -> None:
         msg_id = self.send_huddle_message(self.example_email('othello'),
@@ -351,9 +392,9 @@ def _extra_context_in_huddle_missed_stream_messages_many_others(self, send_as_us
                                            self.example_email('prospero')],
                                           'Group personal message!')
 
-        body = 'Othello, the Moor of Venice: Group personal message! -- Reply'
+        verify_body_include = ['Othello, the Moor of Venice: Group personal message! -- Reply']
         email_subject = 'Group PMs with Cordelia Lear, Iago, and 2 others'
-        self._test_cases(msg_id, body, email_subject, send_as_user)
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user)
 
     def _deleted_message_in_missed_stream_messages(self, send_as_user: bool) -> None:
         msg_id = self.send_stream_message(
@@ -426,7 +467,8 @@ def test_message_content_disabled_in_missed_message_notifications(self) -> None:
         mail.outbox = []
         self._extra_context_in_missed_stream_messages_wildcard_mention(False, show_message_content=False)
         mail.outbox = []
-        self._extra_context_in_personal_missed_stream_messages(False, show_message_content=False)
+        self._extra_context_in_personal_missed_stream_messages(False, show_message_content=False,
+                                                               message_content_disabled_by_user=True)
         mail.outbox = []
         self._extra_context_in_huddle_missed_stream_messages_two_others(False, show_message_content=False)
 
@@ -530,7 +572,8 @@ def test_realm_message_content_allowed_in_email_notifications(self) -> None:
         # Emails don't have missed message content when message content is disabled by the user
         do_change_notification_settings(user, "message_content_in_email_notifications", False)
         mail.outbox = []
-        self._extra_context_in_personal_missed_stream_messages(False, show_message_content=False)
+        self._extra_context_in_personal_missed_stream_messages(False, show_message_content=False,
+                                                               message_content_disabled_by_user=True)
 
         # When message content is not allowed at realm level
         # Emails don't have missed message irrespective of message content setting of the user
@@ -540,11 +583,14 @@ def test_realm_message_content_allowed_in_email_notifications(self) -> None:
 
         do_change_notification_settings(user, "message_content_in_email_notifications", True)
         mail.outbox = []
-        self._extra_context_in_personal_missed_stream_messages(False, show_message_content=False)
+        self._extra_context_in_personal_missed_stream_messages(False, show_message_content=False,
+                                                               message_content_disabled_by_realm=True)
 
         do_change_notification_settings(user, "message_content_in_email_notifications", False)
         mail.outbox = []
-        self._extra_context_in_personal_missed_stream_messages(False, show_message_content=False)
+        self._extra_context_in_personal_missed_stream_messages(False, show_message_content=False,
+                                                               message_content_disabled_by_user=True,
+                                                               message_content_disabled_by_realm=True)
 
     def test_realm_emoji_in_missed_message(self) -> None:
         realm = get_realm("zulip")
@@ -555,9 +601,9 @@ def test_realm_emoji_in_missed_message(self) -> None:
         realm_emoji_id = realm.get_active_emoji()['green_tick']['id']
         realm_emoji_url = "http://zulip.testserver/user_avatars/%s/emoji/images/%s.png" % (
             realm.id, realm_emoji_id,)
-        body = '<img alt=":green_tick:" src="%s" title="green tick" style="height: 20px;">' % (realm_emoji_url,)
+        verify_body_include = ['<img alt=":green_tick:" src="%s" title="green tick" style="height: 20px;">' % (realm_emoji_url,)]
         email_subject = 'PMs with Othello, the Moor of Venice'
-        self._test_cases(msg_id, body, email_subject, send_as_user=False, verify_html_body=True)
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user=False, verify_html_body=True)
 
     def test_emojiset_in_missed_message(self) -> None:
         hamlet = self.example_user('hamlet')
@@ -566,9 +612,9 @@ def test_emojiset_in_missed_message(self) -> None:
         msg_id = self.send_personal_message(
             self.example_email('othello'), self.example_email('hamlet'),
             'Extremely personal message with a hamburger :hamburger:!')
-        body = '<img alt=":hamburger:" src="http://zulip.testserver/static/generated/emoji/images-twitter-64/1f354.png" title="hamburger" style="height: 20px;">'
+        verify_body_include = ['<img alt=":hamburger:" src="http://zulip.testserver/static/generated/emoji/images-twitter-64/1f354.png" title="hamburger" style="height: 20px;">']
         email_subject = 'PMs with Othello, the Moor of Venice'
-        self._test_cases(msg_id, body, email_subject, send_as_user=False, verify_html_body=True)
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user=False, verify_html_body=True)
 
     def test_stream_link_in_missed_message(self) -> None:
         msg_id = self.send_personal_message(
@@ -576,9 +622,9 @@ def test_stream_link_in_missed_message(self) -> None:
             'Come and join us in #**Verona**.')
         stream_id = get_stream('Verona', get_realm('zulip')).id
         href = "http://zulip.testserver/#narrow/stream/{stream_id}-Verona".format(stream_id=stream_id)
-        body = '<a class="stream" data-stream-id="5" href="{href}">#Verona</a'.format(href=href)
+        verify_body_include = ['<a class="stream" data-stream-id="5" href="{href}">#Verona</a'.format(href=href)]
         email_subject = 'PMs with Othello, the Moor of Venice'
-        self._test_cases(msg_id, body, email_subject, send_as_user=False, verify_html_body=True)
+        self._test_cases(msg_id, verify_body_include, email_subject, send_as_user=False, verify_html_body=True)
 
     def test_sender_name_in_missed_message(self) -> None:
         hamlet = self.example_user('hamlet')