Repository: zulip/zulip

Issue:
boto3: Fix support for using non-S3 providers
In #14378, we made the migration from boto to boto3, which generally makes life easier, but it breaks the pattern we'd been using of using `self.connection.DefaultHost` to get the "S3" URL for the bucket in use based on the boto configuration.  See 7e0ea61b004ed12e653ed085d75c2508882862e2 for context; we'll want to re-apply that with the right boto protocol for doing so.

Relatedly, we'll want to figure out what (if anything) should replace `/etc/zulip/boto.cfg`.  

Tagging as a release blocker since it's a regression, but it's not so important to block moving #14378 to master as we can fix forward.

We'll also want to look at:
* Updating the discussion in f1f60bc9bb916276ca5643f21971b40cba6a0b0d; I bet with boto3 the signature part is different / can be deleted?
* See if we can remove the block in `zproject/settings.py` related to `/etc/zulip/boto.cfg` that exists as a GCE workaround.



Discussion:
Hello @zulip/server-misc members, this issue was labeled with the "area: uploads" label, so you may want to check it out!

<!-- areaLabelAddition -->

Resolving patch:
diff --git a/zerver/lib/transfer.py b/zerver/lib/transfer.py
--- a/zerver/lib/transfer.py
+++ b/zerver/lib/transfer.py
@@ -45,9 +45,8 @@ def _transfer_message_files_to_s3(attachment: Attachment) -> int:
         file_path = os.path.join(settings.LOCAL_UPLOADS_DIR, "files", attachment.path_id)
         try:
             with open(file_path, 'rb') as f:
-                bucket_name = settings.S3_AUTH_UPLOADS_BUCKET
                 guessed_type = guess_type(attachment.file_name)[0]
-                upload_image_to_s3(bucket_name, attachment.path_id, guessed_type, attachment.owner, f.read())
+                upload_image_to_s3(s3backend.uploads_bucket, attachment.path_id, guessed_type, attachment.owner, f.read())
                 logging.info("Uploaded message file in path %s", file_path)
         except FileNotFoundError:  # nocoverage
             pass
diff --git a/zerver/lib/upload.py b/zerver/lib/upload.py
--- a/zerver/lib/upload.py
+++ b/zerver/lib/upload.py
@@ -32,14 +32,7 @@
 from zerver.lib.avatar_hash import user_avatar_path
 from zerver.lib.exceptions import ErrorCode, JsonableError
 from zerver.lib.utils import generate_random_token
-from zerver.models import (
-    Attachment,
-    Message,
-    Realm,
-    RealmEmoji,
-    UserProfile,
-    get_user_profile_by_id,
-)
+from zerver.models import Attachment, Message, Realm, RealmEmoji, UserProfile
 
 DEFAULT_AVATAR_SIZE = 100
 MEDIUM_AVATAR_SIZE = 500
@@ -281,14 +274,12 @@ def get_bucket(session: Session, bucket_name: str) -> ServiceResource:
     return bucket
 
 def upload_image_to_s3(
-        bucket_name: str,
+        # See https://github.com/python/typeshed/issues/2706
+        bucket: ServiceResource,
         file_name: str,
         content_type: Optional[str],
         user_profile: UserProfile,
         contents: bytes) -> None:
-
-    session = boto3.Session(settings.S3_KEY, settings.S3_SECRET_KEY)
-    bucket = get_bucket(session, bucket_name)
     key = bucket.Object(file_name)
     metadata = {
         "user_profile_id": str(user_profile.id),
@@ -341,23 +332,18 @@ def get_signed_upload_url(path: str) -> str:
                                          ExpiresIn=SIGNED_UPLOAD_URL_DURATION,
                                          HttpMethod='GET')
 
-def get_realm_for_filename(path: str) -> Optional[int]:
-    session = boto3.Session(settings.S3_KEY, settings.S3_SECRET_KEY)
-    bucket = get_bucket(session, settings.S3_AUTH_UPLOADS_BUCKET)
-    key = bucket.Object(path)
-
-    try:
-        user_profile_id = key.metadata['user_profile_id']
-    except botocore.exceptions.ClientError:
-        return None
-    return get_user_profile_by_id(user_profile_id).realm_id
-
 class S3UploadBackend(ZulipUploadBackend):
     def __init__(self) -> None:
         self.session = boto3.Session(settings.S3_KEY, settings.S3_SECRET_KEY)
 
-    def delete_file_from_s3(self, path_id: str, bucket_name: str) -> bool:
-        bucket = get_bucket(self.session, bucket_name)
+        self.avatar_bucket = get_bucket(self.session, settings.S3_AVATAR_BUCKET)
+        network_location = urllib.parse.urlparse(
+            self.avatar_bucket.meta.client.meta.endpoint_url).netloc
+        self.avatar_bucket_url = f"https://{self.avatar_bucket.name}.{network_location}"
+
+        self.uploads_bucket = get_bucket(self.session, settings.S3_AUTH_UPLOADS_BUCKET)
+
+    def delete_file_from_s3(self, path_id: str, bucket: ServiceResource) -> bool:
         key = bucket.Object(path_id)
 
         try:
@@ -372,7 +358,6 @@ def delete_file_from_s3(self, path_id: str, bucket_name: str) -> bool:
     def upload_message_file(self, uploaded_file_name: str, uploaded_file_size: int,
                             content_type: Optional[str], file_data: bytes,
                             user_profile: UserProfile, target_realm: Optional[Realm]=None) -> str:
-        bucket_name = settings.S3_AUTH_UPLOADS_BUCKET
         if target_realm is None:
             target_realm = user_profile.realm
         s3_file_name = "/".join([
@@ -383,7 +368,7 @@ def upload_message_file(self, uploaded_file_name: str, uploaded_file_size: int,
         url = f"/user_uploads/{s3_file_name}"
 
         upload_image_to_s3(
-            bucket_name,
+            self.uploads_bucket,
             s3_file_name,
             content_type,
             user_profile,
@@ -394,14 +379,12 @@ def upload_message_file(self, uploaded_file_name: str, uploaded_file_size: int,
         return url
 
     def delete_message_image(self, path_id: str) -> bool:
-        return self.delete_file_from_s3(path_id, settings.S3_AUTH_UPLOADS_BUCKET)
+        return self.delete_file_from_s3(path_id, self.uploads_bucket)
 
     def write_avatar_images(self, s3_file_name: str, target_user_profile: UserProfile,
                             image_data: bytes, content_type: Optional[str]) -> None:
-        bucket_name = settings.S3_AVATAR_BUCKET
-
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name + ".original",
             content_type,
             target_user_profile,
@@ -411,7 +394,7 @@ def write_avatar_images(self, s3_file_name: str, target_user_profile: UserProfil
         # custom 500px wide version
         resized_medium = resize_avatar(image_data, MEDIUM_AVATAR_SIZE)
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name + "-medium.png",
             "image/png",
             target_user_profile,
@@ -420,7 +403,7 @@ def write_avatar_images(self, s3_file_name: str, target_user_profile: UserProfil
 
         resized_data = resize_avatar(image_data)
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name,
             'image/png',
             target_user_profile,
@@ -443,18 +426,15 @@ def upload_avatar_image(self, user_file: File,
 
     def delete_avatar_image(self, user: UserProfile) -> None:
         path_id = user_avatar_path(user)
-        bucket_name = settings.S3_AVATAR_BUCKET
 
-        self.delete_file_from_s3(path_id + ".original", bucket_name)
-        self.delete_file_from_s3(path_id + "-medium.png", bucket_name)
-        self.delete_file_from_s3(path_id, bucket_name)
+        self.delete_file_from_s3(path_id + ".original", self.avatar_bucket)
+        self.delete_file_from_s3(path_id + "-medium.png", self.avatar_bucket)
+        self.delete_file_from_s3(path_id, self.avatar_bucket)
 
     def get_avatar_key(self, file_name: str) -> ServiceResource:
         # See https://github.com/python/typeshed/issues/2706
         # for why this return type is a `ServiceResource`.
-        bucket = get_bucket(self.session, settings.S3_AVATAR_BUCKET)
-
-        key = bucket.Object(file_name)
+        key = self.avatar_bucket.Object(file_name)
         return key
 
     def copy_avatar(self, source_profile: UserProfile, target_profile: UserProfile) -> None:
@@ -468,27 +448,24 @@ def copy_avatar(self, source_profile: UserProfile, target_profile: UserProfile)
         self.write_avatar_images(s3_target_file_name, target_profile, image_data, content_type)
 
     def get_avatar_url(self, hash_key: str, medium: bool=False) -> str:
-        bucket = settings.S3_AVATAR_BUCKET
         medium_suffix = "-medium.png" if medium else ""
         # ?x=x allows templates to append additional parameters with &s
-        return f"https://{bucket}.s3.amazonaws.com/{hash_key}{medium_suffix}?x=x"
+        return f"{self.avatar_bucket_url}/{hash_key}{medium_suffix}?x=x"
 
     def get_export_tarball_url(self, realm: Realm, export_path: str) -> str:
-        bucket = settings.S3_AVATAR_BUCKET
         # export_path has a leading /
-        return f"https://{bucket}.s3.amazonaws.com{export_path}"
+        return f"{self.avatar_bucket_url}{export_path}"
 
     def realm_avatar_and_logo_path(self, realm: Realm) -> str:
         return os.path.join(str(realm.id), 'realm')
 
     def upload_realm_icon_image(self, icon_file: File, user_profile: UserProfile) -> None:
         content_type = guess_type(icon_file.name)[0]
-        bucket_name = settings.S3_AVATAR_BUCKET
         s3_file_name = os.path.join(self.realm_avatar_and_logo_path(user_profile.realm), 'icon')
 
         image_data = icon_file.read()
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name + ".original",
             content_type,
             user_profile,
@@ -497,7 +474,7 @@ def upload_realm_icon_image(self, icon_file: File, user_profile: UserProfile) ->
 
         resized_data = resize_avatar(image_data)
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name + ".png",
             'image/png',
             user_profile,
@@ -507,14 +484,12 @@ def upload_realm_icon_image(self, icon_file: File, user_profile: UserProfile) ->
         # that users use gravatar.)
 
     def get_realm_icon_url(self, realm_id: int, version: int) -> str:
-        bucket = settings.S3_AVATAR_BUCKET
         # ?x=x allows templates to append additional parameters with &s
-        return f"https://{bucket}.s3.amazonaws.com/{realm_id}/realm/icon.png?version={version}"
+        return f"{self.avatar_bucket_url}/{realm_id}/realm/icon.png?version={version}"
 
     def upload_realm_logo_image(self, logo_file: File, user_profile: UserProfile,
                                 night: bool) -> None:
         content_type = guess_type(logo_file.name)[0]
-        bucket_name = settings.S3_AVATAR_BUCKET
         if night:
             basename = 'night_logo'
         else:
@@ -523,7 +498,7 @@ def upload_realm_logo_image(self, logo_file: File, user_profile: UserProfile,
 
         image_data = logo_file.read()
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name + ".original",
             content_type,
             user_profile,
@@ -532,7 +507,7 @@ def upload_realm_logo_image(self, logo_file: File, user_profile: UserProfile,
 
         resized_data = resize_logo(image_data)
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name + ".png",
             'image/png',
             user_profile,
@@ -542,26 +517,23 @@ def upload_realm_logo_image(self, logo_file: File, user_profile: UserProfile,
         # that users use gravatar.)
 
     def get_realm_logo_url(self, realm_id: int, version: int, night: bool) -> str:
-        bucket = settings.S3_AVATAR_BUCKET
         # ?x=x allows templates to append additional parameters with &s
         if not night:
             file_name = 'logo.png'
         else:
             file_name = 'night_logo.png'
-        return f"https://{bucket}.s3.amazonaws.com/{realm_id}/realm/{file_name}?version={version}"
+        return f"{self.avatar_bucket_url}/{realm_id}/realm/{file_name}?version={version}"
 
     def ensure_medium_avatar_image(self, user_profile: UserProfile) -> None:
         file_path = user_avatar_path(user_profile)
         s3_file_name = file_path
 
-        bucket_name = settings.S3_AVATAR_BUCKET
-        bucket = get_bucket(self.session, bucket_name)
-        key = bucket.Object(file_path + ".original")
+        key = self.avatar_bucket.Object(file_path + ".original")
         image_data = key.get()['Body'].read()
 
         resized_medium = resize_avatar(image_data, MEDIUM_AVATAR_SIZE)
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name + "-medium.png",
             "image/png",
             user_profile,
@@ -574,14 +546,12 @@ def ensure_basic_avatar_image(self, user_profile: UserProfile) -> None:  # nocov
         # Also TODO: Migrate to user_avatar_path(user_profile) + ".png".
         s3_file_name = file_path
 
-        bucket_name = settings.S3_AVATAR_BUCKET
-        bucket = get_bucket(self.session, bucket_name)
-        key = bucket.Object(file_path + ".original")
+        key = self.avatar_bucket.Object(file_path + ".original")
         image_data = key.get()['Body'].read()
 
         resized_avatar = resize_avatar(image_data)
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             s3_file_name,
             "image/png",
             user_profile,
@@ -591,7 +561,6 @@ def ensure_basic_avatar_image(self, user_profile: UserProfile) -> None:  # nocov
     def upload_emoji_image(self, emoji_file: File, emoji_file_name: str,
                            user_profile: UserProfile) -> None:
         content_type = guess_type(emoji_file.name)[0]
-        bucket_name = settings.S3_AVATAR_BUCKET
         emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
             realm_id=user_profile.realm_id,
             emoji_file_name=emoji_file_name,
@@ -600,14 +569,14 @@ def upload_emoji_image(self, emoji_file: File, emoji_file_name: str,
         image_data = emoji_file.read()
         resized_image_data = resize_emoji(image_data)
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             ".".join((emoji_path, "original")),
             content_type,
             user_profile,
             image_data,
         )
         upload_image_to_s3(
-            bucket_name,
+            self.avatar_bucket,
             emoji_path,
             content_type,
             user_profile,
@@ -615,21 +584,18 @@ def upload_emoji_image(self, emoji_file: File, emoji_file_name: str,
         )
 
     def get_emoji_url(self, emoji_file_name: str, realm_id: int) -> str:
-        bucket = settings.S3_AVATAR_BUCKET
         emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(realm_id=realm_id,
                                                         emoji_file_name=emoji_file_name)
-        return f"https://{bucket}.s3.amazonaws.com/{emoji_path}"
+        return f"{self.avatar_bucket_url}/{emoji_path}"
 
     def upload_export_tarball(self, realm: Optional[Realm], tarball_path: str) -> str:
         def percent_callback(bytes_transferred: Any) -> None:
             sys.stdout.write('.')
             sys.stdout.flush()
 
-        session = boto3.Session(settings.S3_KEY, settings.S3_SECRET_KEY)
         # We use the avatar bucket, because it's world-readable.
-        bucket = get_bucket(session, settings.S3_AVATAR_BUCKET)
-        key = bucket.Object(os.path.join("exports", generate_random_token(32),
-                                         os.path.basename(tarball_path)))
+        key = self.avatar_bucket.Object(os.path.join("exports", generate_random_token(32),
+                                                     os.path.basename(tarball_path)))
 
         key.upload_file(tarball_path, Callback=percent_callback)
 
@@ -639,7 +605,7 @@ def percent_callback(bytes_transferred: Any) -> None:
         public_url = session.create_client('s3', config=config).generate_presigned_url(
             'get_object',
             Params={
-                'Bucket': bucket.name,
+                'Bucket': self.avatar_bucket.name,
                 'Key': key.key,
             },
             ExpiresIn=0,
@@ -647,7 +613,7 @@ def percent_callback(bytes_transferred: Any) -> None:
         return public_url
 
     def delete_export_tarball(self, path_id: str) -> Optional[str]:
-        if self.delete_file_from_s3(path_id, settings.S3_AVATAR_BUCKET):
+        if self.delete_file_from_s3(path_id, self.avatar_bucket):
             return path_id
         return None
 


Test patch:
diff --git a/zerver/tests/test_upload.py b/zerver/tests/test_upload.py
--- a/zerver/tests/test_upload.py
+++ b/zerver/tests/test_upload.py
@@ -53,7 +53,6 @@
     delete_export_tarball,
     delete_message_image,
     exif_rotate,
-    get_realm_for_filename,
     resize_avatar,
     resize_emoji,
     sanitize_name,
@@ -1771,19 +1770,6 @@ def test_delete_avatar_image(self) -> None:
         with self.assertRaises(botocore.exceptions.ClientError):
             bucket.Object(avatar_medium_path_id).load()
 
-    @use_s3_backend
-    def test_get_realm_for_filename(self) -> None:
-        create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)
-
-        user_profile = self.example_user('hamlet')
-        uri = upload_message_file('dummy.txt', len(b'zulip!'), 'text/plain', b'zulip!', user_profile)
-        path_id = re.sub('/user_uploads/', '', uri)
-        self.assertEqual(user_profile.realm_id, get_realm_for_filename(path_id))
-
-    @use_s3_backend
-    def test_get_realm_for_filename_when_key_doesnt_exist(self) -> None:
-        self.assertIsNone(get_realm_for_filename('non-existent-file-path'))
-
     @use_s3_backend
     def test_upload_realm_icon_image(self) -> None:
         bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]