Repository: zulip/zulip

Issue:
Split /production/deployment.html into multiple pages
To make https://zulip.readthedocs.io/en/stable/production/deployment.html easier to navigate, we should split two sections out into dedicated pages:

- https://zulip.readthedocs.io/en/stable/production/deployment.html#putting-the-zulip-application-behind-a-reverse-proxy section, to be labeled "Reverse proxies", placed right after Deployment options in the left sidebar.
- https://zulip.readthedocs.io/en/stable/production/deployment.html#system-and-deployment-configuration, to be labeled "System configuration", and placed after "Server configuration" in the left sidebar.

[CZO discussion](https://chat.zulip.org/#narrow/stream/19-documentation/topic/splitting.20.2Fproduction.2Fdeployment.2Ehtml.20.2328928/near/1737746)


Discussion:
Hello @zulip/server-production members, this issue was labeled with the "area: documentation (production)" label, so you may want to check it out!

<!-- areaLabelAddition -->

@drrosa Would you be up for taking care of this one in the next few days?

Resolving patch:
diff --git a/analytics/lib/counts.py b/analytics/lib/counts.py
--- a/analytics/lib/counts.py
+++ b/analytics/lib/counts.py
@@ -797,6 +797,7 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
         # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
         # sum sequence of 'active_users_log:is_bot:day', for any realm that
         # started after the latter stat was introduced.
+        # Included in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER.
         LoggingCountStat("active_users_log:is_bot:day", RealmCount, CountStat.DAY),
         # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
         # approximation, e.g. if a user is deactivated between the end of the
@@ -843,13 +844,15 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
             "minutes_active::day", DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY
         ),
         # Tracks the number of push notifications requested by the server.
+        # Included in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER.
         LoggingCountStat(
             "mobile_pushes_sent::day",
             RealmCount,
             CountStat.DAY,
         ),
         # Rate limiting stats
-        # Used to limit the number of invitation emails sent by a realm
+        # Used to limit the number of invitation emails sent by a realm.
+        # Included in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER.
         LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY),
         # Dependent stats
         # Must come after their dependencies.
@@ -890,6 +893,19 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
     "mobile_pushes_forwarded::day",
 ]
 
+# LoggingCountStats with a daily duration and that are directly stored on
+# the RealmCount table (instead of via aggregation in process_count_stat),
+# can be in a state, after the hourly cron job to update analytics counts,
+# where the logged value will be live-updated later (as the end time for
+# the stat is still in the future). As these logging counts are designed
+# to be used on the self-hosted installation for either debugging or rate
+# limiting, sending these incomplete counts to the bouncer has low value.
+LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER = [
+    "invites_sent::day",
+    "mobile_pushes_sent::day",
+    "active_users_log:is_bot:day",
+]
+
 # To avoid refactoring for now COUNT_STATS can be used as before
 COUNT_STATS = get_count_stats()
 
diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py
--- a/zerver/lib/remote_server.py
+++ b/zerver/lib/remote_server.py
@@ -9,6 +9,7 @@
 from django.utils.translation import gettext as _
 from pydantic import UUID4, BaseModel, ConfigDict, Field, Json, field_validator
 
+from analytics.lib.counts import LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER
 from analytics.models import InstallationCount, RealmCount
 from version import API_FEATURE_LEVEL, ZULIP_VERSION
 from zerver.actions.realm_settings import (
@@ -364,8 +365,10 @@ def send_server_data_to_push_bouncer(consider_usage_statistics: bool = True) ->
         # uploading such statistics enabled.
         installation_count_query = InstallationCount.objects.filter(
             id__gt=last_acked_installation_count_id
+        ).exclude(property__in=LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER)
+        realm_count_query = RealmCount.objects.filter(id__gt=last_acked_realm_count_id).exclude(
+            property__in=LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER
         )
-        realm_count_query = RealmCount.objects.filter(id__gt=last_acked_realm_count_id)
     else:
         installation_count_query = InstallationCount.objects.none()
         realm_count_query = RealmCount.objects.none()
diff --git a/zerver/management/commands/export_search.py b/zerver/management/commands/export_search.py
--- a/zerver/management/commands/export_search.py
+++ b/zerver/management/commands/export_search.py
@@ -17,6 +17,7 @@
 from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated
 from zerver.lib.upload import save_attachment_contents
 from zerver.models import Attachment, Message, Recipient, Stream, UserProfile
+from zerver.models.recipients import get_or_create_huddle
 from zerver.models.users import get_user_by_delivery_email
 
 
@@ -89,6 +90,12 @@ def add_arguments(self, parser: ArgumentParser) -> None:
             metavar="<email>",
             help="Limit to messages received by users with any of these emails (may be specified more than once).  This is a superset of --sender, since senders receive every message they send.",
         )
+        users.add_argument(
+            "--dm",
+            action="append",
+            metavar="<email>",
+            help="Limit to messages in a DM between all of the users provided.  This option must be given two or more times.",
+        )
 
     @override
     def handle(self, *args: Any, **options: Any) -> None:
@@ -104,6 +111,7 @@ def handle(self, *args: Any, **options: Any) -> None:
             and not options["after"]
             and not options["sender"]
             and not options["recipient"]
+            and not options["dm"]
         ):
             raise CommandError("One or more limits are required!")
 
@@ -153,6 +161,18 @@ def handle(self, *args: Any, **options: Any) -> None:
                 or_,
                 [Q(sender__delivery_email__iexact=e) for e in options["sender"]],
             )
+        elif options["dm"]:
+            if len(options["dm"]) == 1:
+                raise CommandError("Must pass two or more --dm arguments, not just one")
+            user_profiles = [get_user_by_delivery_email(e, realm) for e in options["dm"]]
+            if len(user_profiles) == 2:
+                user_a, user_b = user_profiles
+                limits &= Q(recipient=user_a.recipient, sender=user_b) | Q(
+                    recipient=user_b.recipient, sender=user_a
+                )
+            else:
+                huddle = get_or_create_huddle([user.id for user in user_profiles])
+                limits &= Q(recipient=huddle.recipient)
 
         attachments_written: Set[str] = set()
         messages_query = Message.objects.filter(limits, realm=realm).order_by("date_sent")


Test patch:
diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py
--- a/zerver/tests/test_push_notifications.py
+++ b/zerver/tests/test_push_notifications.py
@@ -23,7 +23,7 @@
 from typing_extensions import override
 
 from analytics.lib.counts import CountStat, LoggingCountStat
-from analytics.models import InstallationCount, RealmCount
+from analytics.models import InstallationCount, RealmCount, UserCount
 from corporate.models import CustomerPlan
 from version import ZULIP_VERSION
 from zerver.actions.message_delete import do_delete_messages
@@ -1478,17 +1478,35 @@ def check_counts(
             )
 
         # Create some rows we'll send to remote server
-        realm_stat = LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY)
+        # LoggingCountStat that should be included;
+        # i.e. not in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER
+        messages_read_logging_stat = LoggingCountStat(
+            "messages_read::hour", UserCount, CountStat.HOUR
+        )
         RealmCount.objects.create(
-            realm=user.realm, property=realm_stat.property, end_time=end_time, value=5
+            realm=user.realm,
+            property=messages_read_logging_stat.property,
+            end_time=end_time,
+            value=5,
+        )
+        InstallationCount.objects.create(
+            property=messages_read_logging_stat.property,
+            end_time=end_time,
+            value=5,
+        )
+        # LoggingCountStat that should not be included;
+        # i.e. in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER
+        invites_sent_logging_stat = LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY)
+        RealmCount.objects.create(
+            realm=user.realm,
+            property=invites_sent_logging_stat.property,
+            end_time=end_time,
+            value=5,
         )
         InstallationCount.objects.create(
-            property=realm_stat.property,
+            property=invites_sent_logging_stat.property,
             end_time=end_time,
             value=5,
-            # We set a subgroup here to work around:
-            # https://github.com/zulip/zulip/issues/12362
-            subgroup="test_subgroup",
         )
         # Event type in SYNCED_BILLING_EVENTS -- should be included
         RealmAuditLog.objects.create(
@@ -1510,8 +1528,8 @@ def check_counts(
             event_time=end_time,
             extra_data=orjson.dumps({"foo": "bar"}).decode(),
         )
-        self.assertEqual(RealmCount.objects.count(), 1)
-        self.assertEqual(InstallationCount.objects.count(), 1)
+        self.assertEqual(RealmCount.objects.count(), 2)
+        self.assertEqual(InstallationCount.objects.count(), 2)
         self.assertEqual(RealmAuditLog.objects.filter(id__gt=audit_log_max_id).count(), 2)
 
         with self.settings(SUBMIT_USAGE_STATISTICS=False):
@@ -1686,13 +1704,13 @@ def check_counts(
         # Test only having new RealmCount rows
         RealmCount.objects.create(
             realm=user.realm,
-            property=realm_stat.property,
+            property=messages_read_logging_stat.property,
             end_time=end_time + timedelta(days=1),
             value=6,
         )
         RealmCount.objects.create(
             realm=user.realm,
-            property=realm_stat.property,
+            property=messages_read_logging_stat.property,
             end_time=end_time + timedelta(days=2),
             value=9,
         )
@@ -1701,7 +1719,9 @@ def check_counts(
 
         # Test only having new InstallationCount rows
         InstallationCount.objects.create(
-            property=realm_stat.property, end_time=end_time + timedelta(days=1), value=6
+            property=messages_read_logging_stat.property,
+            end_time=end_time + timedelta(days=1),
+            value=6,
         )
         send_server_data_to_push_bouncer()
         check_counts(8, 8, 3, 2, 7)
@@ -1820,12 +1840,17 @@ def test_analytics_api_foreign_keys_to_remote_realm(self) -> None:
         end_time = self.TIME_ZERO
 
         # Create some rows we'll send to remote server
-        realm_stat = LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY)
+        messages_read_logging_stat = LoggingCountStat(
+            "messages_read::hour", UserCount, CountStat.HOUR
+        )
         realm_count = RealmCount.objects.create(
-            realm=user.realm, property=realm_stat.property, end_time=end_time, value=5
+            realm=user.realm,
+            property=messages_read_logging_stat.property,
+            end_time=end_time,
+            value=5,
         )
         installation_count = InstallationCount.objects.create(
-            property=realm_stat.property,
+            property=messages_read_logging_stat.property,
             end_time=end_time,
             value=5,
         )
@@ -1917,12 +1942,14 @@ def test_analytics_api_foreign_keys_to_remote_realm(self) -> None:
         # set as it should.
         RealmCount.objects.create(
             realm=user.realm,
-            property=realm_stat.property,
+            property=messages_read_logging_stat.property,
             end_time=end_time + timedelta(days=1),
             value=6,
         )
         InstallationCount.objects.create(
-            property=realm_stat.property, end_time=end_time + timedelta(days=1), value=6
+            property=messages_read_logging_stat.property,
+            end_time=end_time + timedelta(days=1),
+            value=6,
         )
         RealmAuditLog.objects.create(
             realm=user.realm,