Repository: zulip/zulip

Issue:
Deduplicate `create_internal_realm` between populate_db and initialize_voyager_db
Once https://github.com/zulip/zulip/pull/13735 is merged, I believe that we can now do a commit series like this:

* Move `create_internal_realm` and `create_users` from `populate_db` to some reasonable place, like `zerver/lib/server_initialization.py`
* Move the `get_client("website")` common setup code block into `create_internal_realm`
* Change `tos_version` to be an option to `create_users`
* Port in the `bot_owner` logic in `initialize_voyager_db` into `create_internal_realm`
* Verify that `initialize_voyager_db` has code identical to what's in `server_initialization.py`.
* Delete the duplicate code in `initialize_voyager_db`.

Once we've done this, I'd like to add a function in `zerver/lib/server_initialization.py` for checking if the server has been initialized (it can use the existing `Realm` check); we'll use this in `initialize_voyager_db` and then will be able to take advantage of it for checking whether `docker-zulip` needs to run initialization code.

@mateuszmandera this is follow-up to work you did on cleaning up the system bot realm some number of months ago; maybe you should take this issue?  
Deduplicate `create_internal_realm` between populate_db and initialize_voyager_db
Once https://github.com/zulip/zulip/pull/13735 is merged, I believe that we can now do a commit series like this:

* Move `create_internal_realm` and `create_users` from `populate_db` to some reasonable place, like `zerver/lib/server_initialization.py`
* Move the `get_client("website")` common setup code block into `create_internal_realm`
* Change `tos_version` to be an option to `create_users`
* Port in the `bot_owner` logic in `initialize_voyager_db` into `create_internal_realm`
* Verify that `initialize_voyager_db` has code identical to what's in `server_initialization.py`.
* Delete the duplicate code in `initialize_voyager_db`.

Once we've done this, I'd like to add a function in `zerver/lib/server_initialization.py` for checking if the server has been initialized (it can use the existing `Realm` check); we'll use this in `initialize_voyager_db` and then will be able to take advantage of it for checking whether `docker-zulip` needs to run initialization code.

@mateuszmandera this is follow-up to work you did on cleaning up the system bot realm some number of months ago; maybe you should take this issue?  


Discussion:
Hello @zulip/server-bots, @zulip/server-production, @zulip/server-refactoring members, this issue was labeled with the "area: bots", "area: production installer", "area: refactoring" labels, so you may want to check it out!

<!-- areaLabelAddition -->
OK, #13735 is merged and this is now ready to work on.
Hello @zulip/server-bots, @zulip/server-production, @zulip/server-refactoring members, this issue was labeled with the "area: bots", "area: production installer", "area: refactoring" labels, so you may want to check it out!

<!-- areaLabelAddition -->
OK, #13735 is merged and this is now ready to work on.

Resolving patch:
diff --git a/zerver/lib/server_initialization.py b/zerver/lib/server_initialization.py
new file mode 100644
--- /dev/null
+++ b/zerver/lib/server_initialization.py
@@ -0,0 +1,38 @@
+from django.conf import settings
+
+from zerver.lib.actions import do_change_is_admin
+from zerver.lib.bulk_create import bulk_create_users
+from zerver.models import Realm, UserProfile, email_to_username, get_client, \
+    get_system_bot
+
+from typing import Iterable, Optional, Tuple
+
+def create_internal_realm() -> None:
+    realm = Realm.objects.create(string_id=settings.SYSTEM_BOT_REALM)
+
+    # Create the "website" and "API" clients:
+    get_client("website")
+    get_client("API")
+
+    internal_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
+                     for bot in settings.INTERNAL_BOTS]
+    create_users(realm, internal_bots, bot_type=UserProfile.DEFAULT_BOT)
+    # Set the owners for these bots to the bots themselves
+    bots = UserProfile.objects.filter(email__in=[bot_info[1] for bot_info in internal_bots])
+    for bot in bots:
+        bot.bot_owner = bot
+        bot.save()
+
+    # Initialize the email gateway bot as an API Super User
+    email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
+    do_change_is_admin(email_gateway_bot, True, permission="api_super_user")
+
+def create_users(realm: Realm, name_list: Iterable[Tuple[str, str]],
+                 tos_version: Optional[str]=None,
+                 bot_type: Optional[int]=None,
+                 bot_owner: Optional[UserProfile]=None) -> None:
+    user_set = set()
+    for full_name, email in name_list:
+        short_name = email_to_username(email)
+        user_set.add((email, full_name, short_name, True))
+    bulk_create_users(realm, user_set, bot_type=bot_type, bot_owner=bot_owner, tos_version=tos_version)
diff --git a/zerver/management/commands/initialize_voyager_db.py b/zerver/management/commands/initialize_voyager_db.py
--- a/zerver/management/commands/initialize_voyager_db.py
+++ b/zerver/management/commands/initialize_voyager_db.py
@@ -1,23 +1,14 @@
 from argparse import ArgumentParser
-from typing import Any, Iterable, Optional, Tuple
+from typing import Any
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
 
-from zerver.lib.actions import do_change_is_admin
-from zerver.lib.bulk_create import bulk_create_users
-from zerver.models import Realm, UserProfile, email_to_username, get_client, \
-    get_system_bot
+from zerver.lib.server_initialization import create_internal_realm
+from zerver.models import Realm
 
 settings.TORNADO_SERVER = None
 
-def create_users(realm: Realm, name_list: Iterable[Tuple[str, str]], bot_type: Optional[int]=None) -> None:
-    user_set = set()
-    for full_name, email in name_list:
-        short_name = email_to_username(email)
-        user_set.add((email, full_name, short_name, True))
-    bulk_create_users(realm, user_set, bot_type)
-
 class Command(BaseCommand):
     help = "Populate an initial database for Zulip Voyager"
 
@@ -32,23 +23,7 @@ def handle(self, *args: Any, **options: Any) -> None:
         if Realm.objects.count() > 0:
             print("Database already initialized; doing nothing.")
             return
-        realm = Realm.objects.create(string_id=settings.SYSTEM_BOT_REALM)
-
-        get_client("website")
-        get_client("API")
-
-        internal_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
-                         for bot in settings.INTERNAL_BOTS]
-        create_users(realm, internal_bots, bot_type=UserProfile.DEFAULT_BOT)
-        # Set the owners for these bots to the bots themselves
-        bots = UserProfile.objects.filter(email__in=[bot_info[1] for bot_info in internal_bots])
-        for bot in bots:
-            bot.bot_owner = bot
-            bot.save()
-
-        # Initialize the email gateway bot as an API Super User
-        email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
-        do_change_is_admin(email_gateway_bot, True, permission="api_super_user")
+        create_internal_realm()
 
         self.stdout.write("Successfully populated database with initial data.\n")
         self.stdout.write("Please run ./manage.py generate_realm_creation_link "
diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py
--- a/zilencer/management/commands/populate_db.py
+++ b/zilencer/management/commands/populate_db.py
@@ -2,7 +2,7 @@
 import os
 import random
 from typing import Any, Callable, Dict, Iterable, List, \
-    Mapping, Optional, Sequence, Set, Tuple
+    Mapping, Sequence, Tuple
 
 import ujson
 from datetime import datetime
@@ -18,11 +18,12 @@
 from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, check_add_realm_emoji, \
     do_change_is_admin, do_send_messages, do_update_user_custom_profile_data_if_changed, \
     try_add_realm_custom_profile_field, try_add_realm_default_custom_profile_field
-from zerver.lib.bulk_create import bulk_create_streams, bulk_create_users
+from zerver.lib.bulk_create import bulk_create_streams
 from zerver.lib.cache import cache_set
 from zerver.lib.generate_test_data import create_test_data
 from zerver.lib.onboarding import create_if_missing_realm_internal_bots
 from zerver.lib.push_notifications import logger as push_notifications_logger
+from zerver.lib.server_initialization import create_internal_realm, create_users
 from zerver.lib.storage import static_path
 from zerver.lib.users import add_service
 from zerver.lib.url_preview.preview import CACHE_NAME as PREVIEW_CACHE_NAME
@@ -31,8 +32,8 @@
 from zerver.models import CustomProfileField, DefaultStream, Message, Realm, RealmAuditLog, \
     RealmDomain, Recipient, Service, Stream, Subscription, \
     UserMessage, UserPresence, UserProfile, Huddle, Client, \
-    email_to_username, get_client, get_huddle, get_realm, get_stream, \
-    get_system_bot, get_user, get_user_profile_by_id
+    get_client, get_huddle, get_realm, get_stream, \
+    get_user, get_user_profile_by_id
 from zerver.lib.types import ProfileFieldData
 
 from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path
@@ -73,16 +74,6 @@ def clear_database() -> None:
 # Suppress spammy output from the push notifications logger
 push_notifications_logger.disabled = True
 
-def create_users(realm: Realm, name_list: Iterable[Tuple[str, str]],
-                 bot_type: Optional[int]=None,
-                 bot_owner: Optional[UserProfile]=None) -> None:
-    user_set = set()  # type: Set[Tuple[str, str, str, bool]]
-    for full_name, email in name_list:
-        short_name = email_to_username(email)
-        user_set.add((email, full_name, short_name, True))
-    tos_version = settings.TOS_VERSION if bot_type is None else None
-    bulk_create_users(realm, user_set, bot_type=bot_type, bot_owner=bot_owner, tos_version=tos_version)
-
 def subscribe_users_to_streams(realm: Realm, stream_dict: Dict[str, Dict[str, Any]]) -> None:
     subscriptions_to_add = []
     event_time = timezone_now()
@@ -271,7 +262,7 @@ def handle(self, **options: Any) -> None:
                 email = fname.lower() + '@zulip.com'
                 names.append((full_name, email))
 
-            create_users(zulip_realm, names)
+            create_users(zulip_realm, names, tos_version=settings.TOS_VERSION)
 
             iago = get_user("iago@zulip.com", zulip_realm)
             do_change_is_admin(iago, True)
@@ -511,12 +502,6 @@ def handle(self, **options: Any) -> None:
             generate_and_send_messages(job)
 
         if options["delete"]:
-            # Create the "website" and "API" clients; if we don't, the
-            # default values in zerver/decorators.py will not work
-            # with the Django test suite.
-            get_client("website")
-            get_client("API")
-
             if options["test_suite"]:
                 # Create test users; the MIT ones are needed to test
                 # the Zephyr mirroring codepaths.
@@ -525,13 +510,13 @@ def handle(self, **options: Any) -> None:
                     ("Athena Consulting Exchange User (MIT)", "starnine@mit.edu"),
                     ("Esp Classroom (MIT)", "espuser@mit.edu"),
                 ]
-                create_users(mit_realm, testsuite_mit_users)
+                create_users(mit_realm, testsuite_mit_users, tos_version=settings.TOS_VERSION)
 
                 testsuite_lear_users = [
                     ("King Lear", "king@lear.org"),
                     ("Cordelia Lear", "cordelia@zulip.com"),
                 ]
-                create_users(lear_realm, testsuite_lear_users)
+                create_users(lear_realm, testsuite_lear_users, tos_version=settings.TOS_VERSION)
 
             if not options["test_suite"]:
                 # To keep the messages.json fixtures file for the test
@@ -608,17 +593,6 @@ def handle(self, **options: Any) -> None:
                 call_command('populate_analytics_db')
             self.stdout.write("Successfully populated test database.\n")
 
-def create_internal_realm() -> None:
-    internal_realm = Realm.objects.create(string_id=settings.SYSTEM_BOT_REALM)
-
-    internal_realm_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
-                           for bot in settings.INTERNAL_BOTS]
-    create_users(internal_realm, internal_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
-
-    # Initialize the email gateway bot as an API Super User
-    email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
-    do_change_is_admin(email_gateway_bot, True, permission="api_super_user")
-
 recipient_hash = {}  # type: Dict[int, Recipient]
 def get_recipient_by_id(rid: int) -> Recipient:
     if rid in recipient_hash:


Test patch:
diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.py
--- a/zerver/tests/test_subs.py
+++ b/zerver/tests/test_subs.py
@@ -3183,7 +3183,7 @@ def test_subscriptions_query_count(self) -> None:
                 dict(principals=ujson.dumps([user1.email, user2.email])),
                 invite_only=True,
             )
-        self.assert_length(queries, 38)
+        self.assert_length(queries, 40)
 
         # Test creating a public stream with announce when realm has a notification stream.
         notifications_stream = get_stream(self.streams[0], self.test_realm)