Repository: zulip/zulip

Issue:
provision: Avoid spending 15s every no-op provision rebuilding the database
Currently, the no-op provision time is about 30s, and more than half of that is rebuilding the "manual testing" database.  

This is unnecessary in most cases; we should only need to do that when there are new migrations (or perhaps old migrations being removed).

We've already done something similar to the work required with the `is_template_database_current` section in `tools/lib/provision.py`, where we only regenerate the unit testing database if the set of migrations present have changed (or `populate_db` has changed, etc.).

I think it might be possible to just call `is_template_database_current` with different options (we'll want to use different `migration_status` path files, at least, for test vs. dev), but we should test :).

This is, conceptually, the last specific item from #1461, which I'm going to close at this point in favor of these more specific issues.


Discussion:
Hello @zulip/server-development members, this issue was labeled with the **area: provision** label, so you may want to check it out!

Resolving patch:
diff --git a/tools/lib/provision.py b/tools/lib/provision.py
--- a/tools/lib/provision.py
+++ b/tools/lib/provision.py
@@ -240,18 +240,28 @@ def main(options):
     if not options.is_production_travis:
         # These won't be used anyway
         run(["scripts/setup/configure-rabbitmq"])
-        run(["tools/setup/postgres-init-dev-db"])
-        run(["tools/do-destroy-rebuild-database"])
         # Need to set up Django before using is_template_database_current.
         os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
         import django
         django.setup()
         from zerver.lib.test_fixtures import is_template_database_current
+
+        if options.is_force or not is_template_database_current(
+                migration_status="var/migration_status_dev",
+                settings="zproject.settings",
+                database_name="zulip",
+        ):
+            run(["tools/setup/postgres-init-dev-db"])
+            run(["tools/do-destroy-rebuild-database"])
+        else:
+            print("No need to regenerate the dev DB.")
+
         if options.is_force or not is_template_database_current():
             run(["tools/setup/postgres-init-test-db"])
             run(["tools/do-destroy-rebuild-test-database"])
         else:
             print("No need to regenerate the test DB.")
+
         run(["./manage.py", "compilemessages"])
 
     # Here we install nvm, node, and npm.
diff --git a/zproject/dev_urls.py b/zproject/dev_urls.py
--- a/zproject/dev_urls.py
+++ b/zproject/dev_urls.py
@@ -11,6 +11,10 @@
 static_root = os.path.join(settings.DEPLOY_ROOT, 'prod-static/serve' if use_prod_static else 'static')
 
 urls = [
+    # Serve static assets via the Django server
+    url(r'^static/(?P<path>.*)$', serve, {'document_root': static_root}),
+
+    # Serve useful development environment resources (docs, coverage reports, etc.)
     url(r'^coverage/(?P<path>.*)$',
         serve, {'document_root':
                 os.path.join(settings.DEPLOY_ROOT, 'var/coverage'),
@@ -18,7 +22,8 @@
     url(r'^docs/(?P<path>.*)$',
         serve, {'document_root':
                 os.path.join(settings.DEPLOY_ROOT, 'docs/_build/html')}),
-    url(r'^static/(?P<path>.*)$', serve, {'document_root': static_root}),
+
+    # The special no-password login endpoint for development
     url(r'^devlogin/$', zerver.views.auth.login_page,
         {'template_name': 'zerver/dev_login.html'}, name='zerver.views.auth.login_page'),
 ]


Test patch:
diff --git a/zerver/lib/test_fixtures.py b/zerver/lib/test_fixtures.py
--- a/zerver/lib/test_fixtures.py
+++ b/zerver/lib/test_fixtures.py
@@ -14,7 +14,6 @@
 from django.utils.module_loading import module_has_submodule
 
 FILENAME_SPLITTER = re.compile('[\W\-_]')
-TEST_DB_STATUS_DIR = 'var/test_db_status'
 
 def database_exists(database_name, **options):
     # type: (Text, **Any) -> bool
@@ -66,19 +65,19 @@ def are_migrations_the_same(migration_file, **options):
         migration_content = f.read()
     return migration_content == get_migration_status(**options)
 
-def _get_hash_file_path(source_file_path):
-    # type: (str) -> str
+def _get_hash_file_path(source_file_path, status_dir):
+    # type: (str, str) -> str
     basename = os.path.basename(source_file_path)
     filename = '_'.join(FILENAME_SPLITTER.split(basename)).lower()
-    return os.path.join(TEST_DB_STATUS_DIR, filename)
+    return os.path.join(status_dir, filename)
 
-def _check_hash(target_hash_file, **options):
-    # type: (str, **Any) -> bool
+def _check_hash(target_hash_file, status_dir):
+    # type: (str, str) -> bool
     """
     This function has a side effect of creating a new hash file or
     updating the old hash file.
     """
-    source_hash_file = _get_hash_file_path(target_hash_file)
+    source_hash_file = _get_hash_file_path(target_hash_file, status_dir)
 
     with open(target_hash_file) as f:
         target_hash_content = hashlib.sha1(f.read().encode('utf8')).hexdigest()
@@ -96,10 +95,11 @@ def _check_hash(target_hash_file, **options):
 
 def is_template_database_current(
         database_name='zulip_test_template',
-        migration_status='var/migration-status',
+        migration_status='var/migration_status_test',
         settings='zproject.test_settings',
+        status_dir='var/test_db_status',
         check_files=None):
-    # type: (Text, Text, Text, Optional[List[str]]) -> bool
+    # type: (str, str, str, str, Optional[List[str]]) -> bool
     # Using str type for check_files because re.split doesn't accept unicode
     if check_files is None:
         check_files = [
@@ -108,14 +108,14 @@ def is_template_database_current(
             'tools/setup/postgres-init-dev-db',
         ]
 
-    if not os.path.exists(TEST_DB_STATUS_DIR):
-        os.mkdir(TEST_DB_STATUS_DIR)
+    if not os.path.exists(status_dir):
+        os.mkdir(status_dir)
 
     if database_exists(database_name):
         # To ensure Python evaluates all the hash tests (and thus creates the
         # hash files about the current state), we evaluate them in a
         # list and then process the result
-        hash_status = all([_check_hash(fn) for fn in check_files])
+        hash_status = all([_check_hash(fn, status_dir) for fn in check_files])
         return are_migrations_the_same(migration_status, settings=settings) and hash_status
 
     return False