Repository: edgedb/edgedb

Issue:
Generate test dumps for a4 and a5 releases
We need to generate dumps for a4 and a5: https://github.com/edgedb/edgedb/tree/master/tests/dumps/dump01

Would be great if we could automate that with some subcommand under the `edb` tool. Also need a checklist for releases.


Discussion:

Resolving patch:
diff --git a/edb/testbase/server.py b/edb/testbase/server.py
--- a/edb/testbase/server.py
+++ b/edb/testbase/server.py
@@ -52,6 +52,11 @@
 from edb.testbase import serutils
 
 
+if TYPE_CHECKING:
+    DatabaseName = str
+    SetupScript = str
+
+
 def get_test_cases(tests):
     result = collections.OrderedDict()
 
@@ -577,24 +582,29 @@ def run_cli(self, *args, input: Optional[str]=None):
 
 class CLITestCaseMixin:
 
-    def run_cli(self, *args, input: Optional[str]=None):
+    def run_cli(self, *args, input: Optional[str] = None) -> None:
         conn_args = self.get_connect_args()
+        self.run_cli_on_connection(conn_args, *args, input=input)
 
-        cmd_args = (
+    @classmethod
+    def run_cli_on_connection(
+        cls, conn_args: Dict[str, Any], *args, input: Optional[str] = None
+    ) -> None:
+        cmd_args = [
             '--host', conn_args['host'],
             '--port', str(conn_args['port']),
-            '--user', conn_args['user'],
-        ) + args
-
-        if conn_args['password']:
-            cmd_args = ('--password-from-stdin',) + cmd_args
+        ]
+        if conn_args.get('user'):
+            cmd_args += ['--user', conn_args['user']]
+        if conn_args.get('password'):
+            cmd_args += ['--password-from-stdin']
             if input is not None:
                 input = f"{conn_args['password']}\n{input}"
             else:
                 input = f"{conn_args['password']}\n"
-
+        cmd_args += args
         subprocess.run(
-            ('edgedb',) + cmd_args,
+            ['edgedb'] + cmd_args,
             input=input.encode() if input else None,
             check=True,
             capture_output=True,
@@ -879,8 +889,10 @@ class QueryTestCase(BaseQueryTestCase):
     BASE_TEST_CLASS = True
 
 
-def get_test_cases_setup(cases):
-    result = []
+def get_test_cases_setup(
+    cases: Iterable[unittest.TestCase]
+) -> List[Tuple[unittest.TestCase, DatabaseName, SetupScript]]:
+    result: List[Tuple[unittest.TestCase, DatabaseName, SetupScript]] = []
 
     for case in cases:
         if not hasattr(case, 'get_setup_script'):
diff --git a/edb/tools/edb.py b/edb/tools/edb.py
--- a/edb/tools/edb.py
+++ b/edb/tools/edb.py
@@ -61,4 +61,5 @@ def server(version=False, **kwargs):
 from . import inittestdb  # noqa
 from . import test  # noqa
 from . import wipe  # noqa
+from . import gen_test_dumps  # noqa
 from .profiling import cli  # noqa


Test patch:
diff --git a/edb/tools/gen_test_dumps.py b/edb/tools/gen_test_dumps.py
new file mode 100644
--- /dev/null
+++ b/edb/tools/gen_test_dumps.py
@@ -0,0 +1,140 @@
+#
+# This source file is part of the EdgeDB open source project.
+#
+# Copyright 2020-present MagicStack Inc. and the EdgeDB authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+from __future__ import annotations
+from typing import *
+
+import os
+import pathlib
+import shutil
+import sys
+import tempfile
+import unittest
+
+import click
+import setuptools_scm
+
+from edb.server import buildmeta
+from edb.server import cluster as edgedb_cluster
+from edb.testbase import server as tb
+from edb.tools.edb import edbcommands
+
+
+class TestResult:
+    def wasSuccessful(self):
+        return True
+
+
+class TestRunner:
+    def __init__(self):
+        self.cases = set()
+
+    def run(self, test):
+        self.cases.update(tb.get_test_cases([test]))
+        return TestResult()
+
+
+def execute(
+    tests_dir: str, conn: Dict[str, Any], num_workers: int, version: str
+) -> None:
+    runner = TestRunner()
+    unittest.main(
+        module=None,
+        argv=["unittest", "discover", "-s", tests_dir],
+        testRunner=runner,
+        exit=False,
+    )
+
+    setup_scripts = tb.get_test_cases_setup(runner.cases)
+    dump_cases = {
+        db_name: case
+        for case, db_name, _ss in setup_scripts
+        if getattr(case, "STABLE_DUMP", False)
+    }
+    tb.setup_test_cases(list(dump_cases.values()), conn, num_workers)
+
+    dumps_dir = pathlib.Path(tests_dir) / "dumps"
+    db_friendly_version = version.split("+", 1)[0]
+    db_friendly_version = db_friendly_version.replace("-", "_")
+    db_friendly_version = db_friendly_version.replace(".", "_")
+    for db_name in dump_cases:
+        with tempfile.NamedTemporaryFile() as f:
+            tb.CLITestCaseMixin.run_cli_on_connection(
+                conn, "-d", db_name, "dump", f.name
+            )
+            db_dumps_dir = dumps_dir / db_name
+            db_dumps_dir.mkdir(exist_ok=True)
+            dump_p = (db_dumps_dir / db_friendly_version).with_suffix(".dump")
+            shutil.copy(f.name, dump_p)
+            print(f"Dumped {dump_p}")
+
+
+def die(msg):
+    print(f"FATAL: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+@edbcommands.command("gen-test-dumps")
+@click.option(
+    "-t",
+    "--tests-dir",
+    type=str,
+    default=str(
+        pathlib.Path(__file__).parent.parent.parent.resolve() / "tests"
+    ),
+    help="directory to start dump test discovery from",
+)
+@click.option(
+    "-j",
+    "--jobs",
+    type=int,
+    default=lambda: round(os.cpu_count() * 0.75),
+    help="number of parallel processes to use",
+)
+def gen_test_dumps(*, jobs, tests_dir):
+    version = setuptools_scm.get_version(
+        version_scheme=buildmeta.scm_version_scheme
+    )
+    if not jobs:
+        jobs = os.cpu_count()
+
+    with tempfile.TemporaryDirectory(
+        dir="/tmp/", prefix="edb_gen-test-dumps_"
+    ) as data_dir:
+        cluster = edgedb_cluster.Cluster(data_dir, testmode=True)
+        print(
+            f"Generating test dumps for version {version}"
+            f" with a temporary EdgeDB instance in {data_dir}..."
+        )
+
+        try:
+            cluster.init()
+            cluster.start(port="dynamic")
+            cluster.trust_local_connections()
+        except BaseException:
+            raise
+
+        conn = cluster.get_connect_args()
+        try:
+            execute(tests_dir, conn, num_workers=jobs, version=version)
+        except BaseException:
+            raise
+        finally:
+            cluster.stop()
+            cluster.destroy()
diff --git a/tests/dumps/dump01/1_0_a2.dump b/tests/dumps/dump01/1_0a2.dump
similarity index 100%
rename from tests/dumps/dump01/1_0_a2.dump
rename to tests/dumps/dump01/1_0a2.dump
diff --git a/tests/dumps/dump01/1_0_a3.dump b/tests/dumps/dump01/1_0a3.dump
similarity index 100%
rename from tests/dumps/dump01/1_0_a3.dump
rename to tests/dumps/dump01/1_0a3.dump
diff --git a/tests/dumps/dump01/1_0a4.dump b/tests/dumps/dump01/1_0a4.dump
new file mode 100644
Binary files /dev/null and b/tests/dumps/dump01/1_0a4.dump differ
diff --git a/tests/dumps/dump01/1_0a5.dump b/tests/dumps/dump01/1_0a5.dump
new file mode 100644
Binary files /dev/null and b/tests/dumps/dump01/1_0a5.dump differ
diff --git a/tests/test_dump01.py b/tests/test_dump01.py
--- a/tests/test_dump01.py
+++ b/tests/test_dump01.py
@@ -25,7 +25,7 @@
 from edb.testbase import server as tb
 
 
-class TestDump01_CLI(tb.QueryTestCase, tb.CLITestCaseMixin):
+class TestDump01(tb.QueryTestCase, tb.CLITestCaseMixin):
 
     SCHEMA = os.path.join(os.path.dirname(__file__), 'schemas',
                           'dump01_test.esdl')
@@ -37,6 +37,7 @@ class TestDump01_CLI(tb.QueryTestCase, tb.CLITestCaseMixin):
 
     ISOLATED_METHODS = False
     SERIALIZED = True
+    STABLE_DUMP = True
 
     async def test_dump01_dump_restore(self):
         assert type(self).__name__.startswith('Test')
@@ -65,13 +66,14 @@ async def test_dump01_dump_restore(self):
             await self.con.execute(f'DROP DATABASE {dbname}_restored')
 
     async def test_dump01_restore_compatibility(self):
-        dumpsdir = os.path.join(os.path.dirname(__file__), 'dumps', 'dump01')
-        for entry in os.scandir(dumpsdir):
-            if not entry.is_file():
+        current_dir = os.path.dirname(__file__)
+        dumps_dir = os.path.join(current_dir, 'dumps', 'dump01')
+        for entry in os.scandir(dumps_dir):
+            if not entry.is_file() or not entry.name.endswith(".dump"):
                 continue
 
-            dbname = f'{type(self).__name__}_{entry.name}'
-            dumpfn = os.path.join(dumpsdir, entry.name)
+            dbname = type(self).__name__ + "_" + entry.name[:-len(".dump")]
+            dumpfn = os.path.join(dumps_dir, entry.name)
             await self.con.execute(f'CREATE DATABASE `{dbname}`')
             try:
                 self.run_cli('-d', dbname, 'restore', dumpfn)