Repository: zulip/zulip

Issue:
Include attachments with emails
### Current and desired situation

When I forwarded an email that contained a PDF attachment I was surprised that Zulip did not include this PDF, so I had to manually download and upload the attachment to the thread.

### Proposed implementation

Show emails with their attachments, in the same fashion as attaching a file to a message works currently.



Discussion:
I had thought we did include attachments.  @mateuszmandera can you investigate?
Hello @zulip/server-development, @zulip/server-integrations members, this issue was labeled with the "area: emails", "area: integrations" labels, so you may want to check it out!

<!-- areaLabelAddition -->
@zulipbot claim


Resolving patch:
diff --git a/zerver/lib/email_mirror.py b/zerver/lib/email_mirror.py
--- a/zerver/lib/email_mirror.py
+++ b/zerver/lib/email_mirror.py
@@ -141,6 +141,8 @@ def construct_zulip_body(message: message.Message, realm: Realm, show_sender: bo
     if not include_footer:
         body = filter_footer(body)
 
+    if not body.endswith('\n'):
+        body += '\n'
     body += extract_and_upload_attachments(message, realm)
     body = body.strip()
     if not body:
@@ -226,14 +228,9 @@ def filter_footer(text: str) -> str:
 
 def extract_and_upload_attachments(message: message.Message, realm: Realm) -> str:
     user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT)
-    attachment_links = []
-
-    payload = message.get_payload()
-    if not isinstance(payload, list):
-        # This is not a multipart message, so it can't contain attachments.
-        return ""
 
-    for part in payload:
+    attachment_links = []
+    for part in message.walk():
         content_type = part.get_content_type()
         filename = part.get_filename()
         if filename:
@@ -249,7 +246,7 @@ def extract_and_upload_attachments(message: message.Message, realm: Realm) -> st
                 logger.warning("Payload is not bytes (invalid attachment %s in message from %s)." %
                                (filename, message.get("From")))
 
-    return "\n".join(attachment_links)
+    return '\n'.join(attachment_links)
 
 def decode_stream_email_address(email: str) -> Tuple[Stream, Dict[str, bool]]:
     token, options = decode_email_address(email)
diff --git a/zerver/management/commands/send_to_email_mirror.py b/zerver/management/commands/send_to_email_mirror.py
--- a/zerver/management/commands/send_to_email_mirror.py
+++ b/zerver/management/commands/send_to_email_mirror.py
@@ -105,12 +105,15 @@ def _parse_email_fixture(self, fixture_path: str) -> Message:
     def _prepare_message(self, message: Message, realm: Realm, stream_name: str) -> None:
         stream = get_stream(stream_name, realm)
 
+        # The block below ensures that the imported email message doesn't have any recipient-like
+        # headers that are inconsistent with the recipient we want (the stream address).
         recipient_headers = ["X-Gm-Original-To", "Delivered-To",
-                             "Resent-To", "Resent-CC", "To", "CC"]
+                             "Resent-To", "Resent-CC", "CC"]
         for header in recipient_headers:
             if header in message:
                 del message[header]
                 message[header] = encode_email_address(stream)
-                return
 
+        if 'To' in message:
+            del message['To']
         message['To'] = encode_email_address(stream)


Test patch:
diff --git a/zerver/tests/test_email_mirror.py b/zerver/tests/test_email_mirror.py
--- a/zerver/tests/test_email_mirror.py
+++ b/zerver/tests/test_email_mirror.py
@@ -422,7 +422,45 @@ def test_message_with_valid_attachment(self) -> None:
                                                    target_realm=user_profile.realm)
 
         message = most_recent_message(user_profile)
-        self.assertEqual(message.content, "Test body[image.png](https://test_url)")
+        self.assertEqual(message.content, "Test body\n[image.png](https://test_url)")
+
+    def test_message_with_valid_nested_attachment(self) -> None:
+        user_profile = self.example_user('hamlet')
+        self.login(user_profile.email)
+        self.subscribe(user_profile, "Denmark")
+        stream = get_stream("Denmark", user_profile.realm)
+        stream_to_address = encode_email_address(stream)
+
+        incoming_valid_message = MIMEMultipart()
+        text_msg = MIMEText("Test body")
+        incoming_valid_message.attach(text_msg)
+
+        nested_multipart = MIMEMultipart()
+        nested_text_message = MIMEText("Nested text that should get skipped.")
+        nested_multipart.attach(nested_text_message)
+        with open(os.path.join(settings.DEPLOY_ROOT, "static/images/default-avatar.png"), 'rb') as f:
+            image_bytes = f.read()
+
+        attachment_msg = MIMEImage(image_bytes)
+        attachment_msg.add_header('Content-Disposition', 'attachment', filename="image.png")
+        nested_multipart.attach(attachment_msg)
+        incoming_valid_message.attach(nested_multipart)
+
+        incoming_valid_message['Subject'] = 'Subject'
+        incoming_valid_message['From'] = self.example_email('hamlet')
+        incoming_valid_message['To'] = stream_to_address
+        incoming_valid_message['Reply-to'] = self.example_email('othello')
+
+        with mock.patch('zerver.lib.email_mirror.upload_message_file',
+                        return_value='https://test_url') as upload_message_file:
+            process_message(incoming_valid_message)
+            upload_message_file.assert_called_with('image.png', len(image_bytes),
+                                                   'image/png', image_bytes,
+                                                   get_system_bot(settings.EMAIL_GATEWAY_BOT),
+                                                   target_realm=user_profile.realm)
+
+        message = most_recent_message(user_profile)
+        self.assertEqual(message.content, "Test body\n[image.png](https://test_url)")
 
     def test_message_with_invalid_attachment(self) -> None:
         user_profile = self.example_user('hamlet')