Repository: WordPress/openverse-api

Issue:
Indices are not renamed after go-live swap
## Description
<!-- Concisely describe the bug. -->
During data refresh, a new table `temp_import_<media>` is made for the tables being imported. The indices of the matching table (`<media>`) are then applied to the table using a similar prefix convention. [(Source)](https://github.com/WordPress/openverse-api/blob/b760ca1c403265bafffda5b831033186b3d0c50f/ingestion_server/ingestion_server/ingest.py#L103).

When the final go-live update is made at the end of the refresh, the table names are updated but the indices are not [(Source)](https://github.com/WordPress/openverse-api/blob/b760ca1c403265bafffda5b831033186b3d0c50f/ingestion_server/ingestion_server/queries.py#L159). This results in `temp_import_` prefixed indices which conflict with the indices Django is expecting during migrations.

## Expectation
<!-- Concisely describe what you expected to happen. -->
The table that is re-created after data refresh mirrors exactly the current media table in production.

## Additional context
<!-- Add any other context about the problem here; or delete the section entirely. -->

## Resolution
<!-- Replace the [ ] with [x] to check the box. -->
- [ ] 🙋 I would be interested in resolving this bug.



Discussion:

Resolving patch:
diff --git a/ingestion_server/ingestion_server/ingest.py b/ingestion_server/ingestion_server/ingest.py
--- a/ingestion_server/ingestion_server/ingest.py
+++ b/ingestion_server/ingestion_server/ingest.py
@@ -75,45 +75,58 @@ def _get_shared_cols(downstream, upstream, table: str):
     return shared
 
 
-def _generate_indices(conn, table: str):
+def _generate_indices(conn, table: str) -> tuple[list[str], dict[str, str]]:
     """
     Using the existing table as a template, generate CREATE INDEX statements for
     the new table.
 
     :param conn: A connection to the API database.
     :param table: The table to be updated.
-    :return: A list of CREATE INDEX statements.
+    :return: A list of CREATE INDEX statements, and a mapping from new indices to
+    the previous ones.
     """
+    index_mapping = {}
 
-    def _clean_idxs(indices):
+    def _clean_idxs(indices: list[str]):
         # Remove names of indices. We don't want to collide with the old names;
         # we want the database to generate them for us upon recreating the
         # table.
         cleaned = []
         for index in indices:
             # The index name is always after CREATE [UNIQUE] INDEX; delete it.
-            tokens = index[0].split(" ")
+            tokens = index.split(" ")
             index_idx = tokens.index("INDEX")
-            del tokens[index_idx + 1]
+            is_pk = "(id)" in index
+            # Record what the old index was
+            old_index = tokens[index_idx + 1]
+            # Primary keys during data refresh are based on the table name and may
+            # not be exactly correlated with what the primary keys are actually named
+            new_index = (
+                f"temp_import_{old_index}" if not is_pk else f"temp_import_{table}_pkey"
+            )
+            index_mapping[new_index] = old_index
+            # Update name
+            tokens[index_idx + 1] = new_index
             # The table name is always after ON. Rename it to match the
             # temporary copy of the data.
             on_idx = tokens.index("ON")
             table_name_idx = on_idx + 1
             schema_name, table_name = tokens[table_name_idx].split(".")
             tokens[table_name_idx] = f"{schema_name}.temp_import_{table_name}"
-            if "id" not in index:
+            # Skip the primary key, it already exists
+            if not is_pk:
                 cleaned.append(" ".join(tokens))
 
-        return cleaned
+        return cleaned, index_mapping
 
     # Get all of the old indices from the existing table.
     with conn.cursor() as cur:
         get_idxs = SQL(
-            "SELECT indexdef " "FROM pg_indexes " "WHERE tablename = {table};"
+            "SELECT indexdef FROM pg_indexes WHERE tablename = {table};"
         ).format(table=Literal(table))
         cur.execute(get_idxs)
         idxs = cur.fetchall()
-    cleaned_idxs = _clean_idxs(idxs)
+    cleaned_idxs = _clean_idxs([idx[0] for idx in idxs])
     return cleaned_idxs
 
 
@@ -281,6 +294,7 @@ def reload_upstream(table, progress=None, finish_time=None, approach="advanced")
         # Step 3: Import data into a temporary table
         log.info("Copying upstream data...")
         copy_data = get_copy_data_query(table, shared_cols, approach=approach)
+        log.info(f"Running copy-data query: \n{copy_data.as_string(downstream_cur)}")
         downstream_cur.execute(copy_data)
     downstream_db.commit()
     downstream_db.close()
@@ -294,10 +308,10 @@ def reload_upstream(table, progress=None, finish_time=None, approach="advanced")
     with downstream_db.cursor() as downstream_cur:
         # Step 5: Recreate indices from the original table
         log.info("Copying finished! Recreating database indices...")
-        create_indices = ";\n".join(_generate_indices(downstream_db, table))
+        create_indices, index_mapping = _generate_indices(downstream_db, table)
         _update_progress(progress, 50.0)
         if create_indices != "":
-            downstream_cur.execute(create_indices)
+            downstream_cur.execute(";\n".join(create_indices))
         _update_progress(progress, 70.0)
 
         # Step 6: Recreate constraints from the original table
@@ -309,7 +323,8 @@ def reload_upstream(table, progress=None, finish_time=None, approach="advanced")
 
         # Step 7: Promote the temporary table and delete the original
         log.info("Done remapping constraints! Going live with new table...")
-        go_live = get_go_live_query(table)
+        go_live = get_go_live_query(table, index_mapping)
+        log.info(f"Running go-live: \n{go_live.as_string(downstream_cur)}")
         downstream_cur.execute(go_live)
     downstream_db.commit()
     downstream_db.close()
diff --git a/ingestion_server/ingestion_server/queries.py b/ingestion_server/ingestion_server/queries.py
--- a/ingestion_server/ingestion_server/queries.py
+++ b/ingestion_server/ingestion_server/queries.py
@@ -156,7 +156,7 @@ def get_copy_data_query(
     )
 
 
-def get_go_live_query(table: str):
+def get_go_live_query(table: str, index_mapping: dict[str, str]):
     """
     Get the query for replacing the old table with new temporary table. The
     temporary table with the "temp_import_" prefix replaces the un-prefixed
@@ -165,13 +165,21 @@ def get_go_live_query(table: str):
     :param table: the name of the old table being replaced with the temp
     :return: the SQL query for replacing the old table with new temporary table
     """
+    alters = [
+        SQL("ALTER INDEX {new} RENAME TO {old};").format(
+            new=Identifier(new), old=Identifier(old)
+        )
+        for new, old in index_mapping.items()
+    ]
 
     return SQL(
         """
         DROP TABLE {table};
+        {alters}
         ALTER TABLE {temp_table} RENAME TO {table};
     """
     ).format(
         table=Identifier(table),
+        alters=SQL("\n        ").join(alters),
         temp_table=Identifier(f"temp_import_{table}"),
     )


Test patch:
diff --git a/ingestion_server/test/gen_integration_compose.py b/ingestion_server/test/gen_integration_compose.py
--- a/ingestion_server/test/gen_integration_compose.py
+++ b/ingestion_server/test/gen_integration_compose.py
@@ -67,6 +67,26 @@ def _fixup_env(conf: dict):
     conf["services"]["ingestion_server"]["env_file"] = ["env.integration"]
 
 
+def _remove_volumes(conf: dict):
+    """
+    Remove the volumes from the compose configuration so that the images begin with
+    a fresh start on every startup.
+    :param conf: the Docker Compose configuration
+    """
+    volumes_to_remove = {
+        "db": "api-postgres",
+        "upstream_db": "catalog-postgres",
+        "es": "es-data",
+    }
+
+    for service, volume_to_remove in volumes_to_remove.items():
+        volumes = conf["services"][service]["volumes"]
+        volumes = [volume for volume in volumes if volume_to_remove not in volume]
+        conf["services"][service]["volumes"] = volumes
+
+    conf["volumes"] = {}
+
+
 def _change_directories(conf: dict):
     """
     Update the relative paths of the directories such as build context or bind
@@ -115,6 +135,10 @@ def gen_integration_compose():
         _fixup_env(conf)
         print("done")
 
+        print("│ Removing volumes... ", end="")
+        _remove_volumes(conf)
+        print("done")
+
         print("│ Changing directories... ", end="")
         _change_directories(conf)
         print("done")
diff --git a/ingestion_server/test/integration_test.py b/ingestion_server/test/integration_test.py
--- a/ingestion_server/test/integration_test.py
+++ b/ingestion_server/test/integration_test.py
@@ -163,6 +163,85 @@ def _load_data(cls, conn, table_names):
         conn.commit()
         cur.close()
 
+    @staticmethod
+    def _get_index_parts(index: str, table: str) -> list[str]:
+        """
+        Strip out common keywords from the index to get a the name & columns.
+        Indices take the form of:
+          CREATE [UNIQUE] INDEX {name} ON {table} USING btree {columns}
+        Output will look like: ["my_special_index", "(my_column)"]
+        """
+        for token in [
+            "CREATE",
+            "UNIQUE",
+            "INDEX",
+            "ON",
+            "USING",
+            f"public.{table}",
+            "btree",
+        ]:
+            index = index.replace(f"{token} ", "")
+        return index.split(" ", maxsplit=1)
+
+    @classmethod
+    def _get_indices(cls, conn, table) -> dict[str, str]:
+        """
+        Get the indices on a given table using a given connection.
+        """
+        index_sql = f"SELECT indexdef FROM pg_indexes WHERE tablename = '{table}';"
+        with conn.cursor() as cursor:
+            cursor.execute(index_sql)
+            indices = [cls._get_index_parts(row[0], table) for row in cursor]
+            idx_mapping = {columns: name for name, columns in indices}
+            return idx_mapping
+
+    @classmethod
+    def _get_constraints(cls, conn, table) -> dict[str, str]:
+        """
+        Get the constraints on a given table using a given connection.
+        """
+        constraint_sql = f"""
+             SELECT conname, pg_get_constraintdef(c.oid)
+             FROM pg_constraint AS c
+             JOIN pg_namespace AS n
+             ON n.oid = c.connamespace
+             AND n.nspname = 'public'
+             AND conrelid::regclass::text = '{table}'
+             ORDER BY conrelid::regclass::text, contype DESC;
+        """
+        with conn.cursor() as cursor:
+            cursor.execute(constraint_sql)
+            return {constraint: name for name, constraint in cursor}
+
+    def _ingest_upstream(self, model):
+        """
+        Check that INGEST_UPSTREAM task completes successfully and responds
+        with a callback.
+        """
+        before_indices = self._get_indices(self.downstream_db, model)
+        before_constraints = self._get_constraints(self.downstream_db, model)
+        req = {
+            "model": model,
+            "action": "INGEST_UPSTREAM",
+            "callback_url": bottle_url,
+        }
+        res = requests.post(f"{ingestion_server}/task", json=req)
+        stat_msg = "The job should launch successfully and return 202 ACCEPTED."
+        self.assertEqual(res.status_code, 202, msg=stat_msg)
+
+        # Wait for the task to send us a callback.
+        assert self.__class__.cb_queue.get(timeout=120) == "CALLBACK!"
+
+        # Check that the indices remained the same
+        after_indices = self._get_indices(self.downstream_db, model)
+        after_constraints = self._get_constraints(self.downstream_db, model)
+        assert (
+            before_indices == after_indices
+        ), "Indices in DB don't match the names they had before the go-live"
+        assert (
+            before_constraints == after_constraints
+        ), "Constraints in DB don't match the names they had before the go-live"
+
     @classmethod
     def setUpClass(cls) -> None:
         # Launch a Bottle server to receive and handle callbacks
@@ -185,14 +264,16 @@ def setUpClass(cls) -> None:
         )
 
         # Wait for services to be ready
-        upstream_db, downstream_db = cls._wait_for_dbs()
+        cls.upstream_db, cls.downstream_db = cls._wait_for_dbs()
         cls._wait_for_es()
         cls._wait_for_ing()
 
         # Set up the base scenario for the tests
-        cls._load_schemas(upstream_db, ["audio_view", "audioset_view", "image_view"])
         cls._load_schemas(
-            downstream_db,
+            cls.upstream_db, ["audio_view", "audioset_view", "image_view"]
+        )
+        cls._load_schemas(
+            cls.downstream_db,
             [
                 "api_deletedaudio",
                 "api_deletedimage",
@@ -203,9 +284,7 @@ def setUpClass(cls) -> None:
                 "image",
             ],
         )
-        cls._load_data(upstream_db, ["audio_view", "image_view"])
-        upstream_db.close()
-        downstream_db.close()
+        cls._load_data(cls.upstream_db, ["audio_view", "image_view"])
 
     @classmethod
     def tearDownClass(cls) -> None:
@@ -232,6 +311,11 @@ def tearDownClass(cls) -> None:
             capture_output=True,
         )
 
+        # Close connections with databases
+        for conn in [cls.upstream_db, cls.downstream_db]:
+            if conn:
+                conn.close()
+
     @pytest.mark.order(1)
     def test_list_tasks_empty(self):
         res = requests.get(f"{ingestion_server}/task")
@@ -241,21 +325,7 @@ def test_list_tasks_empty(self):
 
     @pytest.mark.order(2)
     def test_image_ingestion_succeeds(self):
-        """
-        Check that INGEST_UPSTREAM task completes successfully and responds
-        with a callback.
-        """
-        req = {
-            "model": "image",
-            "action": "INGEST_UPSTREAM",
-            "callback_url": bottle_url,
-        }
-        res = requests.post(f"{ingestion_server}/task", json=req)
-        stat_msg = "The job should launch successfully and return 202 ACCEPTED."
-        self.assertEqual(res.status_code, 202, msg=stat_msg)
-
-        # Wait for the task to send us a callback.
-        assert self.__class__.cb_queue.get(timeout=120) == "CALLBACK!"
+        self._ingest_upstream("image")
 
     @pytest.mark.order(3)
     def test_task_count_after_one(self):
@@ -266,21 +336,7 @@ def test_task_count_after_one(self):
 
     @pytest.mark.order(4)
     def test_audio_ingestion_succeeds(self):
-        """
-        Check that INGEST_UPSTREAM task completes successfully and responds
-        with a callback.
-        """
-        req = {
-            "model": "audio",
-            "action": "INGEST_UPSTREAM",
-            "callback_url": bottle_url,
-        }
-        res = requests.post(f"{ingestion_server}/task", json=req)
-        stat_msg = "The job should launch successfully and return 202 ACCEPTED."
-        self.assertEqual(res.status_code, 202, msg=stat_msg)
-
-        # Wait for the task to send us a callback.
-        assert self.__class__.cb_queue.get(timeout=120) == "CALLBACK!"
+        self._ingest_upstream("audio")
 
     @pytest.mark.order(5)
     def test_task_count_after_two(self):
diff --git a/ingestion_server/test/mock_schemas/README.md b/ingestion_server/test/mock_schemas/README.md
new file mode 100644
--- /dev/null
+++ b/ingestion_server/test/mock_schemas/README.md
@@ -0,0 +1,7 @@
+# Mock schemas
+
+Files in this directory were created using the following command:
+
+```bash
+docker run --rm -e PGPASSWORD=<password> postgres:13 pg_dump -d openledger -h <url> -p 5432 -U deploy -t 'public.<table>' --schema-only
+```
diff --git a/ingestion_server/test/mock_schemas/audio.sql b/ingestion_server/test/mock_schemas/audio.sql
--- a/ingestion_server/test/mock_schemas/audio.sql
+++ b/ingestion_server/test/mock_schemas/audio.sql
@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 13.2
+-- Dumped from database version 13.3
 -- Dumped by pg_dump version 13.3 (Debian 13.3-1.pgdg100+1)
 
 SET statement_timeout = 0;
@@ -10,6 +10,7 @@ SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
 SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
 SET check_function_bodies = false;
 SET xmloption = content;
 SET client_min_messages = warning;
@@ -42,7 +43,7 @@ CREATE TABLE public.audio (
     source character varying(80),
     last_synced_with_source timestamp with time zone,
     removed_from_source boolean NOT NULL,
-    view_count integer DEFAULT 0,
+    view_count integer,
     tags jsonb,
     tags_list character varying(255)[],
     meta_data jsonb,
@@ -55,124 +56,148 @@ CREATE TABLE public.audio (
     alt_files jsonb,
     thumbnail character varying(1000),
     filetype character varying(80),
-    audio_set_foreign_identifier character varying(1000),
-    standardized_popularity double precision
+    audio_set_foreign_identifier character varying(1000)
 );
 
 
 ALTER TABLE public.audio OWNER TO deploy;
 
 --
--- Name: audio temp_import_audio_pkey; Type: CONSTRAINT; Schema: public; Owner: deploy
+-- Name: audio_id_seq; Type: SEQUENCE; Schema: public; Owner: deploy
 --
 
-ALTER TABLE ONLY public.audio
-    ADD CONSTRAINT temp_import_audio_pkey PRIMARY KEY (id);
+CREATE SEQUENCE public.audio_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+ALTER TABLE public.audio_id_seq OWNER TO deploy;
+
+--
+-- Name: audio_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: deploy
+--
+
+ALTER SEQUENCE public.audio_id_seq OWNED BY public.audio.id;
+
+
+--
+-- Name: audio id; Type: DEFAULT; Schema: public; Owner: deploy
+--
+
+ALTER TABLE ONLY public.audio ALTER COLUMN id SET DEFAULT nextval('public.audio_id_seq'::regclass);
 
 
 --
--- Name: temp_import_audio_category_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio audio_identifier_key; Type: CONSTRAINT; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_category_idx ON public.audio USING btree (category);
+ALTER TABLE ONLY public.audio
+    ADD CONSTRAINT audio_identifier_key UNIQUE (identifier);
 
 
 --
--- Name: temp_import_audio_category_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio audio_pkey; Type: CONSTRAINT; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_category_idx1 ON public.audio USING btree (category varchar_pattern_ops);
+ALTER TABLE ONLY public.audio
+    ADD CONSTRAINT audio_pkey PRIMARY KEY (id);
 
 
 --
--- Name: temp_import_audio_foreign_identifier_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio audio_url_key; Type: CONSTRAINT; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_foreign_identifier_idx ON public.audio USING btree (foreign_identifier varchar_pattern_ops);
+ALTER TABLE ONLY public.audio
+    ADD CONSTRAINT audio_url_key UNIQUE (url);
 
 
 --
--- Name: temp_import_audio_foreign_identifier_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio unique_provider_audio; Type: CONSTRAINT; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_foreign_identifier_idx1 ON public.audio USING btree (foreign_identifier);
+ALTER TABLE ONLY public.audio
+    ADD CONSTRAINT unique_provider_audio UNIQUE (foreign_identifier, provider);
 
 
 --
--- Name: temp_import_audio_foreign_identifier_provider_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_category_ceb7d386; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_audio_foreign_identifier_provider_idx ON public.audio USING btree (foreign_identifier, provider);
+CREATE INDEX audio_category_ceb7d386 ON public.audio USING btree (category);
 
 
 --
--- Name: temp_import_audio_genres_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_category_ceb7d386_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_genres_idx ON public.audio USING btree (genres);
+CREATE INDEX audio_category_ceb7d386_like ON public.audio USING btree (category varchar_pattern_ops);
 
 
 --
--- Name: temp_import_audio_id_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_foreign_identifier_617f66ad; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_audio_id_idx ON public.audio USING btree (id);
+CREATE INDEX audio_foreign_identifier_617f66ad ON public.audio USING btree (foreign_identifier);
 
 
 --
--- Name: temp_import_audio_identifier_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_foreign_identifier_617f66ad_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_audio_identifier_idx ON public.audio USING btree (identifier);
+CREATE INDEX audio_foreign_identifier_617f66ad_like ON public.audio USING btree (foreign_identifier varchar_pattern_ops);
 
 
 --
--- Name: temp_import_audio_last_synced_with_source_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_genres_e34cc474; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_last_synced_with_source_idx ON public.audio USING btree (last_synced_with_source);
+CREATE INDEX audio_genres_e34cc474 ON public.audio USING btree (genres);
 
 
 --
--- Name: temp_import_audio_provider_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_last_synced_with_source_94c4a383; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_provider_idx ON public.audio USING btree (provider);
+CREATE INDEX audio_last_synced_with_source_94c4a383 ON public.audio USING btree (last_synced_with_source);
 
 
 --
--- Name: temp_import_audio_provider_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_provider_8fe1eb54; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_provider_idx1 ON public.audio USING btree (provider varchar_pattern_ops);
+CREATE INDEX audio_provider_8fe1eb54 ON public.audio USING btree (provider);
 
 
 --
--- Name: temp_import_audio_source_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_provider_8fe1eb54_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_source_idx ON public.audio USING btree (source);
+CREATE INDEX audio_provider_8fe1eb54_like ON public.audio USING btree (provider varchar_pattern_ops);
 
 
 --
--- Name: temp_import_audio_source_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_source_e9ccc813; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_source_idx1 ON public.audio USING btree (source varchar_pattern_ops);
+CREATE INDEX audio_source_e9ccc813 ON public.audio USING btree (source);
 
 
 --
--- Name: temp_import_audio_url_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_source_e9ccc813_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_audio_url_idx ON public.audio USING btree (url);
+CREATE INDEX audio_source_e9ccc813_like ON public.audio USING btree (source varchar_pattern_ops);
 
 
 --
--- Name: temp_import_audio_url_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audio_url_b6a832d3_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audio_url_idx1 ON public.audio USING btree (url varchar_pattern_ops);
+CREATE INDEX audio_url_b6a832d3_like ON public.audio USING btree (url varchar_pattern_ops);
 
 
 --
diff --git a/ingestion_server/test/mock_schemas/audioset.sql b/ingestion_server/test/mock_schemas/audioset.sql
--- a/ingestion_server/test/mock_schemas/audioset.sql
+++ b/ingestion_server/test/mock_schemas/audioset.sql
@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 13.2
+-- Dumped from database version 13.3
 -- Dumped by pg_dump version 13.3 (Debian 13.3-1.pgdg100+1)
 
 SET statement_timeout = 0;
@@ -10,6 +10,7 @@ SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
 SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
 SET check_function_bodies = false;
 SET xmloption = content;
 SET client_min_messages = warning;
@@ -25,8 +26,8 @@ SET default_table_access_method = heap;
 
 CREATE TABLE public.audioset (
     id integer NOT NULL,
-    created_on timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
-    updated_on timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    created_on timestamp with time zone NOT NULL,
+    updated_on timestamp with time zone NOT NULL,
     foreign_identifier character varying(1000),
     title character varying(2000),
     foreign_landing_url character varying(1000),
@@ -43,67 +44,91 @@ CREATE TABLE public.audioset (
 ALTER TABLE public.audioset OWNER TO deploy;
 
 --
--- Name: audioset temp_import_audioset_pkey; Type: CONSTRAINT; Schema: public; Owner: deploy
+-- Name: api_audioset_id_seq; Type: SEQUENCE; Schema: public; Owner: deploy
 --
 
-ALTER TABLE ONLY public.audioset
-    ADD CONSTRAINT temp_import_audioset_pkey PRIMARY KEY (id);
+CREATE SEQUENCE public.api_audioset_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+ALTER TABLE public.api_audioset_id_seq OWNER TO deploy;
+
+--
+-- Name: api_audioset_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: deploy
+--
+
+ALTER SEQUENCE public.api_audioset_id_seq OWNED BY public.audioset.id;
 
 
 --
--- Name: temp_import_audioset_foreign_identifier_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audioset id; Type: DEFAULT; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audioset_foreign_identifier_idx ON public.audioset USING btree (foreign_identifier varchar_pattern_ops);
+ALTER TABLE ONLY public.audioset ALTER COLUMN id SET DEFAULT nextval('public.api_audioset_id_seq'::regclass);
 
 
 --
--- Name: temp_import_audioset_foreign_identifier_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audioset api_audioset_pkey; Type: CONSTRAINT; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audioset_foreign_identifier_idx1 ON public.audioset USING btree (foreign_identifier);
+ALTER TABLE ONLY public.audioset
+    ADD CONSTRAINT api_audioset_pkey PRIMARY KEY (id);
 
 
 --
--- Name: temp_import_audioset_foreign_identifier_provider_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audioset api_audioset_url_key; Type: CONSTRAINT; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_audioset_foreign_identifier_provider_idx ON public.audioset USING btree (foreign_identifier, provider);
+ALTER TABLE ONLY public.audioset
+    ADD CONSTRAINT api_audioset_url_key UNIQUE (url);
+
+
+--
+-- Name: audioset unique_foreign_identifier_provider; Type: CONSTRAINT; Schema: public; Owner: deploy
+--
+
+ALTER TABLE ONLY public.audioset
+    ADD CONSTRAINT unique_foreign_identifier_provider UNIQUE (foreign_identifier, provider);
 
 
 --
--- Name: temp_import_audioset_id_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: api_audioset_foreign_identifier_28c1db59; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_audioset_id_idx ON public.audioset USING btree (id);
+CREATE INDEX api_audioset_foreign_identifier_28c1db59 ON public.audioset USING btree (foreign_identifier);
 
 
 --
--- Name: temp_import_audioset_provider_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: api_audioset_foreign_identifier_28c1db59_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audioset_provider_idx ON public.audioset USING btree (provider);
+CREATE INDEX api_audioset_foreign_identifier_28c1db59_like ON public.audioset USING btree (foreign_identifier varchar_pattern_ops);
 
 
 --
--- Name: temp_import_audioset_provider_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: api_audioset_url_200f9fb4_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audioset_provider_idx1 ON public.audioset USING btree (provider varchar_pattern_ops);
+CREATE INDEX api_audioset_url_200f9fb4_like ON public.audioset USING btree (url varchar_pattern_ops);
 
 
 --
--- Name: temp_import_audioset_url_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audioset_provider_cc061f30; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_audioset_url_idx ON public.audioset USING btree (url);
+CREATE INDEX audioset_provider_cc061f30 ON public.audioset USING btree (provider);
 
 
 --
--- Name: temp_import_audioset_url_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: audioset_provider_cc061f30_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_audioset_url_idx1 ON public.audioset USING btree (url varchar_pattern_ops);
+CREATE INDEX audioset_provider_cc061f30_like ON public.audioset USING btree (provider varchar_pattern_ops);
 
 
 --
diff --git a/ingestion_server/test/mock_schemas/image.sql b/ingestion_server/test/mock_schemas/image.sql
--- a/ingestion_server/test/mock_schemas/image.sql
+++ b/ingestion_server/test/mock_schemas/image.sql
@@ -2,7 +2,7 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 13.2
+-- Dumped from database version 13.3
 -- Dumped by pg_dump version 13.3 (Debian 13.3-1.pgdg100+1)
 
 SET statement_timeout = 0;
@@ -10,6 +10,7 @@ SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
 SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
 SET check_function_bodies = false;
 SET xmloption = content;
 SET client_min_messages = warning;
@@ -41,7 +42,7 @@ CREATE TABLE public.image (
     license_version character varying(25),
     creator character varying(2000),
     creator_url character varying(2000),
-    title character varying(2000),
+    title character varying(5000),
     tags_list character varying(255)[],
     last_synced_with_source timestamp with time zone,
     removed_from_source boolean NOT NULL,
@@ -50,102 +51,110 @@ CREATE TABLE public.image (
     tags jsonb,
     watermarked boolean,
     filetype character varying(80),
-    standardized_popularity double precision
+    standardized_popularity double precision,
+    category character varying(80)
 );
 
 
 ALTER TABLE public.image OWNER TO deploy;
 
 --
--- Name: image temp_import_image_pkey; Type: CONSTRAINT; Schema: public; Owner: deploy
+-- Name: image image_pkey; Type: CONSTRAINT; Schema: public; Owner: deploy
 --
 
 ALTER TABLE ONLY public.image
-    ADD CONSTRAINT temp_import_image_pkey PRIMARY KEY (id);
+    ADD CONSTRAINT image_pkey PRIMARY KEY (id);
 
 
 --
--- Name: temp_import_image_foreign_identifier_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_category_daa5d91c; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_image_foreign_identifier_idx ON public.image USING btree (foreign_identifier varchar_pattern_ops);
+CREATE INDEX image_category_daa5d91c ON public.image USING btree (category);
 
 
 --
--- Name: temp_import_image_foreign_identifier_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_category_daa5d91c_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_image_foreign_identifier_idx1 ON public.image USING btree (foreign_identifier);
+CREATE INDEX image_category_daa5d91c_like ON public.image USING btree (category varchar_pattern_ops);
 
 
 --
--- Name: temp_import_image_foreign_identifier_provider_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_foreign_identifier_4c72d3ee; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_image_foreign_identifier_provider_idx ON public.image USING btree (foreign_identifier, provider);
+CREATE INDEX image_foreign_identifier_4c72d3ee ON public.image USING btree (foreign_identifier);
 
 
 --
--- Name: temp_import_image_id_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_foreign_identifier_4c72d3ee_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_image_id_idx ON public.image USING btree (id);
+CREATE INDEX image_foreign_identifier_4c72d3ee_like ON public.image USING btree (foreign_identifier varchar_pattern_ops);
 
 
 --
--- Name: temp_import_image_identifier_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_identifier_key; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_image_identifier_idx ON public.image USING btree (identifier);
+CREATE UNIQUE INDEX image_identifier_key ON public.image USING btree (identifier);
 
 
 --
--- Name: temp_import_image_last_synced_with_source_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_last_synced_with_source_187adf09; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_image_last_synced_with_source_idx ON public.image USING btree (last_synced_with_source);
+CREATE INDEX image_last_synced_with_source_187adf09 ON public.image USING btree (last_synced_with_source);
 
 
 --
--- Name: temp_import_image_provider_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_provider_7d11f847; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_image_provider_idx ON public.image USING btree (provider);
+CREATE INDEX image_provider_7d11f847 ON public.image USING btree (provider);
 
 
 --
--- Name: temp_import_image_provider_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_provider_7d11f847_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_image_provider_idx1 ON public.image USING btree (provider varchar_pattern_ops);
+CREATE INDEX image_provider_7d11f847_like ON public.image USING btree (provider varchar_pattern_ops);
 
 
 --
--- Name: temp_import_image_source_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_source_d5a89e97; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_image_source_idx ON public.image USING btree (source);
+CREATE INDEX image_source_d5a89e97 ON public.image USING btree (source);
 
 
 --
--- Name: temp_import_image_source_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_source_d5a89e97_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_image_source_idx1 ON public.image USING btree (source varchar_pattern_ops);
+CREATE INDEX image_source_d5a89e97_like ON public.image USING btree (source varchar_pattern_ops);
 
 
 --
--- Name: temp_import_image_url_idx; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_url_c6aabda2_like; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE UNIQUE INDEX temp_import_image_url_idx ON public.image USING btree (url);
+CREATE INDEX image_url_c6aabda2_like ON public.image USING btree (url varchar_pattern_ops);
 
 
 --
--- Name: temp_import_image_url_idx1; Type: INDEX; Schema: public; Owner: deploy
+-- Name: image_url_key; Type: INDEX; Schema: public; Owner: deploy
 --
 
-CREATE INDEX temp_import_image_url_idx1 ON public.image USING btree (url varchar_pattern_ops);
+CREATE UNIQUE INDEX image_url_key ON public.image USING btree (url);
+
+
+--
+-- Name: unique_provider_image; Type: INDEX; Schema: public; Owner: deploy
+--
+
+CREATE UNIQUE INDEX unique_provider_image ON public.image USING btree (foreign_identifier, provider);
 
 
 --
diff --git a/ingestion_server/test/run_test.sh b/ingestion_server/test/run_test.sh
--- a/ingestion_server/test/run_test.sh
+++ b/ingestion_server/test/run_test.sh
@@ -12,7 +12,7 @@ TEST_ARG="${1:-test/}"
 
 PYTHONWARNINGS="ignore:Unverified HTTPS request" \
 PYTHONPATH=. \
-pytest -s --disable-pytest-warnings $TEST_ARG
+pytest -sx -vv --disable-pytest-warnings $TEST_ARG
 
 succeeded=$?
 if [[ $succeeded -eq 0 ]]; then