Repository: mathesar-foundation/mathesar

Issue:
Attempting to add connection with both Library Management and Movie Collection schema results in a 502
## Description
* I'm consistently able to reproduce this on my test setup on GCP, not locally.
* When I try to add a new connection to a DB and check both "Library Management" and "Movie Collection" schemas in the "Schemas to Install' field, it results in a 502 Bad Gateway.
* Endpoint: https://pavish-testing.mathesar.dev/api/ui/v0/connections/create_from_scratch/
* This subsequently results in a followup issue: https://github.com/mathesar-foundation/mathesar/issues/3420

## Hunch
* Could it be that caddy is killing the request if it takes too long to complete?

## Expected behavior
* The request should succeed.



Discussion:
I tried to troubleshoot this issue, here are the findings:

- Gunicorn has a default timeout for requests which is 30s.
- Adding both the datasets at the same time took about **25s** locally but takes **3.1 mins** for a remote db.
- This time could vary depending on where the remote db exists i.e. latency between the server hosting mathesar and the remote db server.

Possible solutions:
- Increase the timeout for requests to something large enough.
- Remove 'Movie Collection' and introduce a smaller dataset.
- Make the data loading pipeline efficient for large enough datasets like the 'Movie Collection'.
@Anish9901 thanks for the update!

> Make the data loading pipeline efficient for large enough datasets like the 'Movie Collection'.

Could you provide more detail on what this would entail?
> Could you provide more detail on what this would entail?

One way I could think of is to use both .sql and .csv, we use .sql to set up the schema, tables, and FKs and use csv to bulk load data using `COPY` as it is supposed to be much more efficient than regular `INSERTS`.

If this works, we should do it for all our datasets as it would also make it efficient to start a new demo instance for users. 

Resolving patch:
diff --git a/demo/install/base.py b/demo/install/base.py
--- a/demo/install/base.py
+++ b/demo/install/base.py
@@ -13,7 +13,9 @@
 LIBRARY_ONE = os.path.join(RESOURCES, "library_without_checkouts.sql")
 LIBRARY_TWO = os.path.join(RESOURCES, "library_add_checkouts.sql")
 DEVCON_DATASET = os.path.join(RESOURCES, "devcon_dataset.sql")
-MOVIES_SQL_BZ2 = os.path.join(RESOURCES, "movie_collection.sql.bz2")
+MOVIES_SQL_TABLES = os.path.join(RESOURCES, "movie_collection_tables.sql")
+MOVIES_SQL_FKS = os.path.join(RESOURCES, "movie_collection_fks.sql")
+MOVIES_CSV = os.path.join(RESOURCES, 'movies_csv')
 ARXIV_SETUP_SQL = os.path.join(RESOURCES, 'arxiv_dataset_setup.sql')
 ARXIV_PAPERS_PICKLE = os.path.join(RESOURCES, 'arxiv_papers.pickle')
 LIBRARY_MANAGEMENT = 'Library Management'
diff --git a/demo/install/dumpcsvs.py b/demo/install/dumpcsvs.py
new file mode 100644
--- /dev/null
+++ b/demo/install/dumpcsvs.py
@@ -0,0 +1,40 @@
+"""
+Dump data for all the tables of a provided schema to separate {table_name}.csv files
+with header as column names.
+
+Usage: python dumpcsvs.py
+"""
+import psycopg
+import csv
+
+DB_NAME = "mathesar"
+DB_USER = "mathesar"
+DB_PASSWORD = "mathesar"
+DB_HOST = "mathesar_dev_db"
+SCHEMA_NAME = "Movie Collection"
+
+conn = psycopg.connect(
+    dbname=DB_NAME,
+    user=DB_USER,
+    password=DB_PASSWORD,
+    host=DB_HOST,
+    port=5432
+)
+
+# get names of tables.
+tables = conn.execute(
+    f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{SCHEMA_NAME}'"
+).fetchall()
+
+for table in tables:
+    table_name = table[0]
+    with open(f'{table_name}.csv', 'w', newline="") as csv_file:
+        csv_writer = csv.writer(csv_file)
+        columns = conn.execute(
+            f"""SELECT column_name FROM information_schema.columns WHERE
+            table_schema = '{SCHEMA_NAME}' AND table_name = '{table_name}';"""
+        ).fetchall()
+        columns = [column[0] for column in columns]
+        csv_writer.writerow(columns)
+        with conn.cursor().copy(f"""COPY "{SCHEMA_NAME}"."{table_name}" TO STDOUT""") as copy:
+            csv_writer.writerows(copy.rows())
diff --git a/demo/install/movies_dataset.py b/demo/install/movies_dataset.py
--- a/demo/install/movies_dataset.py
+++ b/demo/install/movies_dataset.py
@@ -1,9 +1,8 @@
 """This module contains functions to load the Movie Collection dataset."""
-import bz2
-
+import os
 from sqlalchemy import text
 
-from demo.install.base import MOVIE_COLLECTION, MOVIES_SQL_BZ2
+from demo.install.base import MOVIE_COLLECTION, MOVIES_SQL_TABLES, MOVIES_CSV, MOVIES_SQL_FKS
 
 
 def load_movies_dataset(engine, safe_mode=False):
@@ -16,11 +15,15 @@ def load_movies_dataset(engine, safe_mode=False):
                    schema already exists instead of dropping it.
     """
     drop_schema_query = text(f"""DROP SCHEMA IF EXISTS "{MOVIE_COLLECTION}" CASCADE;""")
-    create_schema_query = text(f"""CREATE SCHEMA "{MOVIE_COLLECTION}";""")
-    set_search_path = text(f"""SET search_path="{MOVIE_COLLECTION}";""")
-    with engine.begin() as conn, bz2.open(MOVIES_SQL_BZ2, 'rt') as f:
+    with engine.begin() as conn, open(MOVIES_SQL_TABLES) as f, open(MOVIES_SQL_FKS) as f2:
         if safe_mode is False:
             conn.execute(drop_schema_query)
-        conn.execute(create_schema_query)
-        conn.execute(set_search_path)
         conn.execute(text(f.read()))
+        for file in os.scandir(MOVIES_CSV):
+            table_name = file.name.split('.csv')[0]
+            with open(file, 'r') as csv_file:
+                conn.connection.cursor().copy_expert(
+                    f"""COPY "{MOVIE_COLLECTION}"."{table_name}" FROM STDIN DELIMITER ',' CSV HEADER""",
+                    csv_file
+                )
+        conn.execute(text(f2.read()))


Test patch: