Repository: zulip/zulip

Issue:
outgoing bots: Prevent bugs with multiple services (sender_realm_id)
Thanks to @Udit107710 we discovered that if an outgoing webhook bot supported multiple services, we would have errors when we tried to deliver messages to multiple services.

This was probably never a real world problem, since our current UI only makes it easy to set up one service per bot, even though the DB theoretically supports multiple services.  Also, we never heard of folks in the outside world trying to do this.

The root cause of the problem was that `finalize_payload` was mutating a dictionary for the first service we called, and then a field called `sender_realm_id` would be missing.


Discussion:

Resolving patch:
diff --git a/zerver/lib/message.py b/zerver/lib/message.py
--- a/zerver/lib/message.py
+++ b/zerver/lib/message.py
@@ -2,6 +2,7 @@
 import ujson
 import zlib
 import ahocorasick
+import copy
 
 from django.utils.translation import ugettext as _
 from django.utils.timezone import now as timezone_now
@@ -190,17 +191,46 @@ def wide_dict(message: Message) -> Dict[str, Any]:
 
     @staticmethod
     def post_process_dicts(objs: List[Dict[str, Any]], apply_markdown: bool, client_gravatar: bool) -> None:
+        '''
+        NOTE: This function mutates the objects in
+              the `objs` list, rather than making
+              shallow copies.  It might be safer to
+              make shallow copies here, but performance
+              is somewhat important here, as we are
+              often fetching several messages.
+        '''
         MessageDict.bulk_hydrate_sender_info(objs)
         MessageDict.bulk_hydrate_recipient_info(objs)
 
         for obj in objs:
-            MessageDict.finalize_payload(obj, apply_markdown, client_gravatar)
+            MessageDict._finalize_payload(obj, apply_markdown, client_gravatar)
 
     @staticmethod
     def finalize_payload(obj: Dict[str, Any],
                          apply_markdown: bool,
                          client_gravatar: bool,
-                         keep_rendered_content: bool=False) -> None:
+                         keep_rendered_content: bool=False) -> Dict[str, Any]:
+        '''
+        Make a shallow copy of the incoming dict to avoid
+        mutation-related bugs.  This function is often
+        called when we're sending out message events to
+        multiple clients, who often want the final dictionary
+        to have different shapes here based on the parameters.
+        '''
+        new_obj = copy.copy(obj)
+
+        # Next call our worker, which mutates the record in place.
+        MessageDict._finalize_payload(
+            new_obj,
+            apply_markdown=apply_markdown,
+            client_gravatar=client_gravatar,
+            keep_rendered_content=keep_rendered_content
+        )
+        return new_obj
+
+    @staticmethod
+    def _finalize_payload(obj: Dict[str, Any], apply_markdown: bool, client_gravatar: bool,
+                          keep_rendered_content: bool=False) -> None:
         MessageDict.set_sender_avatar(obj, client_gravatar)
         if apply_markdown:
             obj['content_type'] = 'text/html'
diff --git a/zerver/lib/outgoing_webhook.py b/zerver/lib/outgoing_webhook.py
--- a/zerver/lib/outgoing_webhook.py
+++ b/zerver/lib/outgoing_webhook.py
@@ -28,15 +28,24 @@ def __init__(self, token: str, user_profile: UserProfile, service_name: str) ->
 class GenericOutgoingWebhookService(OutgoingWebhookServiceInterface):
 
     def build_bot_request(self, event: Dict[str, Any]) -> Optional[Any]:
-        # Because we don't have a place for the recipient of an
-        # outgoing webhook to indicate whether it wants the raw
-        # Markdown or the rendered HTML, we leave both the content and
-        # rendered_content fields in the message payload.
-        MessageDict.finalize_payload(event['message'], False, False,
-                                     keep_rendered_content=True)
+        '''
+        We send a simple version of the message to outgoing
+        webhooks, since most of them really only need
+        `content` and a few other fields.  We may eventually
+        allow certain bots to get more information, but
+        that's not a high priority.  We do send the gravatar
+        info to the clients (so they don't have to compute
+        it themselves).
+        '''
+        message_dict = MessageDict.finalize_payload(
+            event['message'],
+            apply_markdown=False,
+            client_gravatar=False,
+            keep_rendered_content=True
+        )
 
         request_data = {"data": event['command'],
-                        "message": event['message'],
+                        "message": message_dict,
                         "bot_email": self.user_profile.email,
                         "token": self.token,
                         "trigger": event['trigger']}
@@ -53,8 +62,7 @@ def send_data_to_server(self,
         response = requests.request('POST', base_url, data=request_data, headers=headers)
         return response
 
-    def process_success(self, response_json: Dict[str, Any],
-                        event: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    def process_success(self, response_json: Dict[str, Any]) -> Optional[Dict[str, Any]]:
         if "response_not_required" in response_json and response_json['response_not_required']:
             return None
 
@@ -102,8 +110,7 @@ def send_data_to_server(self,
         response = requests.request('POST', base_url, data=request_data)
         return response
 
-    def process_success(self, response_json: Dict[str, Any],
-                        event: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    def process_success(self, response_json: Dict[str, Any]) -> Optional[Dict[str, Any]]:
         if "text" in response_json:
             content = response_json['text']
             success_data = dict(content=content)
@@ -251,7 +258,7 @@ def process_success_response(event: Dict[str, Any],
         fail_with_message(event, "Invalid JSON in response")
         return
 
-    success_data = service_handler.process_success(response_json, event)
+    success_data = service_handler.process_success(response_json)
 
     if success_data is None:
         return
diff --git a/zerver/tornado/event_queue.py b/zerver/tornado/event_queue.py
--- a/zerver/tornado/event_queue.py
+++ b/zerver/tornado/event_queue.py
@@ -824,6 +824,15 @@ def process_message_event(event_template: Mapping[str, Any], users: Iterable[Map
     presence_idle_user_ids = set(event_template.get('presence_idle_user_ids', []))
     wide_dict = event_template['message_dict']  # type: Dict[str, Any]
 
+    # Temporary transitional code: Zulip servers that have message
+    # events in their event queues and upgrade to the new version
+    # that expects sender_delivery_email in these events will
+    # throw errors processing events.  We can remove this block
+    # once we don't expect anyone to be directly upgrading from
+    # 2.0.x to the latest Zulip.
+    if 'sender_delivery_email' not in wide_dict:  # nocoverage
+        wide_dict['sender_delivery_email'] = wide_dict['sender_email']
+
     sender_id = wide_dict['sender_id']  # type: int
     message_id = wide_dict['id']  # type: int
     message_type = wide_dict['type']  # type: str
@@ -831,19 +840,11 @@ def process_message_event(event_template: Mapping[str, Any], users: Iterable[Map
 
     @cachify
     def get_client_payload(apply_markdown: bool, client_gravatar: bool) -> Dict[str, Any]:
-        dct = copy.deepcopy(wide_dict)
-
-        # Temporary transitional code: Zulip servers that have message
-        # events in their event queues and upgrade to the new version
-        # that expects sender_delivery_email in these events will
-        # throw errors processing events.  We can remove this block
-        # once we don't expect anyone to be directly upgrading from
-        # 2.0.x to the latest Zulip.
-        if 'sender_delivery_email' not in dct:  # nocoverage
-            dct['sender_delivery_email'] = dct['sender_email']
-
-        MessageDict.finalize_payload(dct, apply_markdown, client_gravatar)
-        return dct
+        return MessageDict.finalize_payload(
+            wide_dict,
+            apply_markdown=apply_markdown,
+            client_gravatar=client_gravatar
+        )
 
     # Extra user-specific data to include
     extra_user_data = {}  # type: Dict[int, Any]


Test patch:
diff --git a/zerver/tests/test_messages.py b/zerver/tests/test_messages.py
--- a/zerver/tests/test_messages.py
+++ b/zerver/tests/test_messages.py
@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
-import copy
-
 from django.db import IntegrityError
 from django.db.models import Q
 from django.conf import settings
@@ -1302,12 +1300,8 @@ def get_send_message_payload(
             msg = reload_message(msg_id)
             wide_dict = MessageDict.wide_dict(msg)
 
-            # TODO: Have finalize_payload make this
-            #       copy for us, rather than mutating
-            #       in place.
-            narrow_dict = copy.copy(wide_dict)
-            MessageDict.finalize_payload(
-                narrow_dict,
+            narrow_dict = MessageDict.finalize_payload(
+                wide_dict,
                 apply_markdown=apply_markdown,
                 client_gravatar=client_gravatar,
             )
diff --git a/zerver/tests/test_narrow.py b/zerver/tests/test_narrow.py
--- a/zerver/tests/test_narrow.py
+++ b/zerver/tests/test_narrow.py
@@ -2416,9 +2416,13 @@ def test_message_without_rendered_content(self) -> None:
         m = self.get_last_message()
         m.rendered_content = m.rendered_content_version = None
         m.content = 'test content'
-        d = MessageDict.wide_dict(m)
-        MessageDict.finalize_payload(d, apply_markdown=True, client_gravatar=False)
-        self.assertEqual(d['content'], '<p>test content</p>')
+        wide_dict = MessageDict.wide_dict(m)
+        final_dict = MessageDict.finalize_payload(
+            wide_dict,
+            apply_markdown=True,
+            client_gravatar=False,
+        )
+        self.assertEqual(final_dict['content'], '<p>test content</p>')
 
     def common_check_get_messages_query(self, query_params: Dict[str, object], expected: str) -> None:
         user_profile = self.example_user('hamlet')
diff --git a/zerver/tests/test_outgoing_webhook_interfaces.py b/zerver/tests/test_outgoing_webhook_interfaces.py
--- a/zerver/tests/test_outgoing_webhook_interfaces.py
+++ b/zerver/tests/test_outgoing_webhook_interfaces.py
@@ -5,37 +5,34 @@
 import json
 import requests
 
+from zerver.lib.avatar import get_gravatar_url
 from zerver.lib.message import MessageDict
 from zerver.lib.outgoing_webhook import (
     get_service_interface_class,
     process_success_response,
 )
 from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.timestamp import datetime_to_timestamp
 from zerver.lib.topic import TOPIC_NAME
-from zerver.models import get_realm, get_user, SLACK_INTERFACE, Message
+from zerver.models import (
+    get_realm,
+    get_stream,
+    get_user,
+    Message,
+    SLACK_INTERFACE,
+)
+
 
 class TestGenericOutgoingWebhookService(ZulipTestCase):
 
     def setUp(self) -> None:
         super().setUp()
 
-        # TODO: Ideally, this test would use the full flow, rather
-        # than making a mock message like this.
-        message_id = self.send_stream_message(self.example_user('othello'),
-                                              "Denmark", content="@**test**")
-        message = Message.objects.get(id=message_id)
-        wide_message_dict = MessageDict.wide_dict(message)
-
-        self.event = {
-            u'command': '@**test**',
-            u'message': wide_message_dict,
-            u'trigger': 'mention',
-        }
-        self.bot_user = get_user("outgoing-webhook@zulip.com", get_realm("zulip"))
+        bot_user = get_user("outgoing-webhook@zulip.com", get_realm("zulip"))
         service_class = get_service_interface_class('whatever')  # GenericOutgoingWebhookService
         self.handler = service_class(service_name='test-service',
                                      token='abcdef',
-                                     user_profile=self.bot_user)
+                                     user_profile=bot_user)
 
     def test_process_success_response(self) -> None:
         class Stub:
@@ -72,19 +69,69 @@ def make_response(text: str) -> requests.Response:
         self.assertTrue(m.called)
 
     def test_build_bot_request(self) -> None:
-        request_data = self.handler.build_bot_request(self.event)
+        othello = self.example_user('othello')
+        stream = get_stream('Denmark', othello.realm)
+        message_id = self.send_stream_message(
+            othello,
+            stream.name,
+            content="@**test**"
+        )
+
+        message = Message.objects.get(id=message_id)
+
+        gravatar_url = get_gravatar_url(
+            othello.delivery_email,
+            othello.avatar_version,
+        )
+
+        expected_message_data = {
+            'avatar_url': gravatar_url,
+            'client': 'test suite',
+            'content': '@**test**',
+            'content_type': 'text/x-markdown',
+            'display_recipient': 'Denmark',
+            'id': message.id,
+            'is_me_message': False,
+            'reactions': [],
+            'recipient_id': message.recipient_id,
+            'rendered_content': '<p>@<strong>test</strong></p>',
+            'sender_email': othello.email,
+            'sender_full_name': 'Othello, the Moor of Venice',
+            'sender_id': othello.id,
+            'sender_realm_str': 'zulip',
+            'sender_short_name': 'othello',
+            'stream_id': stream.id,
+            TOPIC_NAME: 'test',
+            'submessages': [],
+            'timestamp': datetime_to_timestamp(message.date_sent),
+            'topic_links': [],
+            'type': 'stream',
+        }
+
+        wide_message_dict = MessageDict.wide_dict(message)
+
+        event = {
+            u'command': '@**test**',
+            u'message': wide_message_dict,
+            u'trigger': 'mention',
+        }
+
+        request_data = self.handler.build_bot_request(event)
         request_data = json.loads(request_data)
         self.assertEqual(request_data['data'], "@**test**")
         self.assertEqual(request_data['token'], "abcdef")
-        self.assertEqual(request_data['message'], self.event['message'])
+        self.assertEqual(request_data['message'], expected_message_data)
+
+        # Make sure we didn't accidentally mutate wide_message_dict.
+        self.assertEqual(wide_message_dict['sender_realm_id'], othello.realm_id)
 
     def test_process_success(self) -> None:
         response = dict(response_not_required=True)  # type: Dict[str, Any]
-        success_response = self.handler.process_success(response, self.event)
+        success_response = self.handler.process_success(response)
         self.assertEqual(success_response, None)
 
         response = dict(response_string='test_content')
-        success_response = self.handler.process_success(response, self.event)
+        success_response = self.handler.process_success(response)
         self.assertEqual(success_response, dict(content='test_content'))
 
         response = dict(
@@ -92,7 +139,7 @@ def test_process_success(self) -> None:
             widget_content='test_widget_content',
             red_herring='whatever',
         )
-        success_response = self.handler.process_success(response, self.event)
+        success_response = self.handler.process_success(response)
         expected_response = dict(
             content='test_content',
             widget_content='test_widget_content',
@@ -100,7 +147,7 @@ def test_process_success(self) -> None:
         self.assertEqual(success_response, expected_response)
 
         response = dict()
-        success_response = self.handler.process_success(response, self.event)
+        success_response = self.handler.process_success(response)
         self.assertEqual(success_response, None)
 
 class TestSlackOutgoingWebhookService(ZulipTestCase):
@@ -172,9 +219,9 @@ def test_build_bot_request_private_message(self, mock_fail_with_message: mock.Mo
 
     def test_process_success(self) -> None:
         response = dict(response_not_required=True)  # type: Dict[str, Any]
-        success_response = self.handler.process_success(response, self.stream_message_event)
+        success_response = self.handler.process_success(response)
         self.assertEqual(success_response, None)
 
         response = dict(text='test_content')
-        success_response = self.handler.process_success(response, self.stream_message_event)
+        success_response = self.handler.process_success(response)
         self.assertEqual(success_response, dict(content='test_content'))
diff --git a/zerver/tests/test_outgoing_webhook_system.py b/zerver/tests/test_outgoing_webhook_system.py
--- a/zerver/tests/test_outgoing_webhook_system.py
+++ b/zerver/tests/test_outgoing_webhook_system.py
@@ -7,6 +7,10 @@
 
 from typing import Any, Optional
 
+from zerver.lib.actions import (
+    do_create_user,
+)
+
 from zerver.lib.outgoing_webhook import (
     do_rest_call,
     GenericOutgoingWebhookService,
@@ -15,7 +19,15 @@
 
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.topic import TOPIC_NAME
-from zerver.models import get_realm, get_user, UserProfile, get_display_recipient
+from zerver.lib.users import add_service
+from zerver.models import (
+    get_display_recipient,
+    get_realm,
+    get_user,
+    Recipient,
+    Service,
+    UserProfile,
+)
 
 from version import ZULIP_VERSION
 
@@ -139,36 +151,103 @@ def test_request_exception(self, mock_fail_with_message: mock.Mock,
         self.assertEqual(bot_owner_notification.recipient_id, self.bot_user.bot_owner.id)
 
 class TestOutgoingWebhookMessaging(ZulipTestCase):
-    def setUp(self) -> None:
-        super().setUp()
-        self.user_profile = self.example_user("othello")
-        self.bot_profile = self.create_test_bot('outgoing-webhook', self.user_profile,
-                                                full_name='Outgoing Webhook bot',
-                                                bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
-                                                service_name='foo-service')
+    def create_outgoing_bot(self, bot_owner: UserProfile) -> UserProfile:
+        return self.create_test_bot(
+            'outgoing-webhook',
+            bot_owner,
+            full_name='Outgoing Webhook bot',
+            bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
+            service_name='foo-service'
+        )
+
+    def test_multiple_services(self) -> None:
+        bot_owner = self.example_user("othello")
+
+        bot = do_create_user(
+            bot_owner=bot_owner,
+            bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
+            full_name='Outgoing Webhook Bot',
+            email='whatever',
+            realm=bot_owner.realm,
+            short_name='',
+            password=None,
+        )
+
+        add_service(
+            'weather',
+            user_profile=bot,
+            interface=Service.GENERIC,
+            base_url='weather_url',
+            token='weather_token',
+        )
+
+        add_service(
+            'qotd',
+            user_profile=bot,
+            interface=Service.GENERIC,
+            base_url='qotd_url',
+            token='qotd_token',
+        )
+
+        sender = self.example_user("hamlet")
+
+        with mock.patch('zerver.worker.queue_processors.do_rest_call') as m:
+            self.send_personal_message(
+                sender,
+                bot,
+                content="some content"
+            )
+
+        url_token_tups = set()
+        for item in m.call_args_list:
+            args = item[0]
+            base_url = args[0]
+            request_data = ujson.loads(args[1])
+            tup = (base_url, request_data['token'])
+            url_token_tups.add(tup)
+            message_data = request_data['message']
+            self.assertEqual(message_data['content'], 'some content')
+            self.assertEqual(message_data['sender_id'], sender.id)
+
+        self.assertEqual(
+            url_token_tups,
+            {
+                ('weather_url', 'weather_token'),
+                ('qotd_url', 'qotd_token'),
+            }
+        )
 
     @mock.patch('requests.request', return_value=ResponseMock(200, {"response_string": "Hidley ho, I'm a webhook responding!"}))
     def test_pm_to_outgoing_webhook_bot(self, mock_requests_request: mock.Mock) -> None:
-        self.send_personal_message(self.user_profile, self.bot_profile,
+        bot_owner = self.example_user("othello")
+        bot = self.create_outgoing_bot(bot_owner)
+        sender = self.example_user("hamlet")
+
+        self.send_personal_message(sender, bot,
                                    content="foo")
         last_message = self.get_last_message()
         self.assertEqual(last_message.content, "Hidley ho, I'm a webhook responding!")
-        self.assertEqual(last_message.sender_id, self.bot_profile.id)
-        display_recipient = get_display_recipient(last_message.recipient)
-        # The next two lines error on mypy because the display_recipient is of type Union[str, List[Dict[str, Any]]].
-        # In this case, we know that display_recipient will be of type List[Dict[str, Any]].
-        # Otherwise this test will error, which is wanted behavior anyway.
-        self.assert_length(display_recipient, 1)  # type: ignore
-        self.assertEqual(display_recipient[0]['email'], self.user_profile.email)   # type: ignore
+        self.assertEqual(last_message.sender_id, bot.id)
+        self.assertEqual(
+            last_message.recipient.type_id,
+            sender.id
+        )
+        self.assertEqual(
+            last_message.recipient.type,
+            Recipient.PERSONAL
+        )
 
     @mock.patch('requests.request', return_value=ResponseMock(200, {"response_string": "Hidley ho, I'm a webhook responding!"}))
     def test_stream_message_to_outgoing_webhook_bot(self, mock_requests_request: mock.Mock) -> None:
-        self.send_stream_message(self.user_profile, "Denmark",
-                                 content="@**{}** foo".format(self.bot_profile.full_name),
+        bot_owner = self.example_user("othello")
+        bot = self.create_outgoing_bot(bot_owner)
+
+        self.send_stream_message(bot_owner, "Denmark",
+                                 content="@**{}** foo".format(bot.full_name),
                                  topic_name="bar")
         last_message = self.get_last_message()
         self.assertEqual(last_message.content, "Hidley ho, I'm a webhook responding!")
-        self.assertEqual(last_message.sender_id, self.bot_profile.id)
+        self.assertEqual(last_message.sender_id, bot.id)
         self.assertEqual(last_message.topic_name(), "bar")
         display_recipient = get_display_recipient(last_message.recipient)
         self.assertEqual(display_recipient, "Denmark")