Repository: mathesar-foundation/mathesar

Issue:
Automatic import of tables when new database is connected.
**Problem**
<!-- Please provide a clear and concise description of the problem that this feature request is designed to solve.-->
When a user sets up an existing database in Mathesar, they'll expect to see their tables. We don't currently do anything to sync existing databases with Mathesar.

**Proposed solution**
<!-- A clear and concise description of your proposed solution or feature. -->
We should create instances of the appropriate Django models for tables in existing databases.

**Additional context**
<!-- Add any other context or screenshots about the feature request here.-->
Blocked by #61


Discussion:
This might already be handled by #61, if it is, we need to verify that it works.
Currently, if an existing DB is added to the `DATABASES` constant in the settings, the schemata and tables from the DB should be returned in the API once the cache expires.  The tricky parts I can think of will be:
- the custom functions and types associated with Mathesar won't be installed on the DB without the user running the `install.py` script.
- I think the way we're configuring the `mathesar_tables` database with an environment variable means the user will have to add a new environment variable and read it in to have more than one DB for their tables at a time.

Resolving patch:
diff --git a/mathesar/models.py b/mathesar/models.py
--- a/mathesar/models.py
+++ b/mathesar/models.py
@@ -30,7 +30,7 @@ def __str__(self):
 
 # TODO: Replace with a proper form of caching
 # See: https://github.com/centerofci/mathesar/issues/280
-_engine = None
+_engines = {}
 
 
 class Schema(DatabaseObject):
@@ -38,11 +38,11 @@ class Schema(DatabaseObject):
 
     @property
     def _sa_engine(self):
-        global _engine
+        global _engines
         # We're caching this since the engine is used frequently.
-        if _engine is None:
-            _engine = create_mathesar_engine(self.database)
-        return _engine
+        if self.database not in _engines:
+            _engines[self.database] = create_mathesar_engine(self.database)
+        return _engines[self.database]
 
     @cached_property
     def name(self):
diff --git a/mathesar/utils/schemas.py b/mathesar/utils/schemas.py
--- a/mathesar/utils/schemas.py
+++ b/mathesar/utils/schemas.py
@@ -31,6 +31,6 @@ def reflect_schemas_from_database(database):
         for oid in db_schema_oids
     ]
     for schema in Schema.objects.all():
-        if schema.oid not in db_schema_oids:
+        if schema.database == database and schema.oid not in db_schema_oids:
             schema.delete()
     return schemas


Test patch:
diff --git a/mathesar/tests/test_multi_db.py b/mathesar/tests/test_multi_db.py
new file mode 100644
--- /dev/null
+++ b/mathesar/tests/test_multi_db.py
@@ -0,0 +1,119 @@
+import pytest
+from sqlalchemy import create_engine, text
+from sqlalchemy.schema import DropSchema
+from config.settings import DATABASES
+from django.core.cache import cache
+from django.conf import settings
+from dj_database_url import parse as db_url
+
+from db import schemas, tables
+
+MULTI_DB_TEST_DB = "mathesar_multi_db_test"
+
+
+def _get_connection_string(username, password, hostname, database, port=5432):
+    return f"postgresql://{username}:{password}@{hostname}:{port}/{database}"
+
+
+def _get_superuser_engine():
+    return create_engine(
+        _get_connection_string(
+            username=DATABASES["default"]["USER"],
+            password=DATABASES["default"]["PASSWORD"],
+            hostname=DATABASES["default"]["HOST"],
+            database=DATABASES["default"]["NAME"],
+        ),
+        future=True,
+    )
+
+
+@pytest.fixture(scope="module")
+def multi_db_test_db_name():
+    return MULTI_DB_TEST_DB
+
+
+@pytest.fixture(scope="module")
+def multi_db_test_db_connection_string(multi_db_test_db_name):
+    return _get_connection_string(
+        DATABASES["default"]["USER"],
+        DATABASES["default"]["PASSWORD"],
+        DATABASES["default"]["HOST"],
+        multi_db_test_db_name,
+    )
+
+
+@pytest.fixture(scope="module")
+def multi_db_test_db(multi_db_test_db_connection_string):
+    superuser_engine = _get_superuser_engine()
+    with superuser_engine.connect() as conn:
+        conn.execution_options(isolation_level="AUTOCOMMIT")
+        conn.execute(text(f"DROP DATABASE IF EXISTS {MULTI_DB_TEST_DB} WITH (FORCE)"))
+        conn.execute(text(f"CREATE DATABASE {MULTI_DB_TEST_DB}"))
+
+    settings.DATABASES[MULTI_DB_TEST_DB] = db_url(
+        multi_db_test_db_connection_string
+    )
+
+    yield MULTI_DB_TEST_DB
+
+    with superuser_engine.connect() as conn:
+        conn.execution_options(isolation_level="AUTOCOMMIT")
+        conn.execute(text(f"DROP DATABASE {MULTI_DB_TEST_DB} WITH (FORCE)"))
+
+    del settings.DATABASES[MULTI_DB_TEST_DB]
+
+
+@pytest.fixture(scope="module")
+def multi_db_engine(multi_db_test_db, multi_db_test_db_connection_string):
+    return create_engine(
+        multi_db_test_db_connection_string,
+        future=True,
+    )
+
+
+def test_multi_db_schema(engine, multi_db_engine, client):
+    test_schemas = ["test_schema_1", "test_schema_2"]
+    for schema in test_schemas:
+        schemas.create_schema(schema, engine)
+        schemas.create_schema("multi_db_" + schema, multi_db_engine)
+
+    cache.clear()
+    response = client.get('/api/v0/schemas/')
+    response_data = response.json()
+    response_schemas = [
+        s['name'] for s in response_data['results'] if s['name'] != 'public'
+    ]
+
+    assert response.status_code == 200
+    assert len(response_schemas) == 4
+
+    expected_schemas = test_schemas + ["multi_db_" + s for s in test_schemas]
+    assert set(response_schemas) == set(expected_schemas)
+
+    # We have to delete the schemas to not break later tests
+    with engine.begin() as conn:
+        for schema in test_schemas:
+            conn.execute(DropSchema(schema))
+
+
+def test_multi_db_tables(engine, multi_db_engine, client):
+    schema_name = "test_multi_db_tables_schema"
+    test_tables = ["test_table_1", "test_table_2"]
+    for table_name in test_tables:
+        tables.create_mathesar_table(table_name, schema_name, [], engine)
+        tables.create_mathesar_table(
+            "multi_db_" + table_name, schema_name, [], multi_db_engine
+        )
+
+    cache.clear()
+    response = client.get('/api/v0/tables/')
+    response_tables = [s['name'] for s in response.json()['results']]
+
+    assert response.status_code == 200
+    expected_tables = test_tables + ["multi_db_" + s for s in test_tables]
+    for table_name in expected_tables:
+        assert table_name in response_tables
+
+    # We have to delete the schema to not break later tests
+    with engine.begin() as conn:
+        conn.execute(DropSchema(schema_name, cascade=True))