Repository: mathesar-foundation/mathesar

Issue:
Add support for column mappings for importing records from one table to another.
## Description
<!-- A clear and concise description of what the bug is. -->
To import records from one table to another having different column names, types, etc, we need the ability to specify which column of the temp/import table should map to which specific columns of the target table in which the records are to be imported.  

## Expected behavior
<!-- A clear and concise description of what you expected to happen. -->
To achieve the above we need support for column mappings. Which will be set by the users and sent to the backend with a request to the `/existing_import` like this:
```
{
    "import_target": <target_table_id>,
    "data_files": [<data_file_id>],
    "mappings": [
                    [<temp_table_col1_id>,<target_table_col1_id>],
                    [<temp_table_col2_id>,<target_table_col2_id>]
                ]
}
```


Discussion:

Resolving patch:
diff --git a/db/records/operations/insert.py b/db/records/operations/insert.py
--- a/db/records/operations/insert.py
+++ b/db/records/operations/insert.py
@@ -76,10 +76,25 @@ def insert_records_from_csv(table, engine, csv_filepath, column_names, header, d
                     cursor.copy_expert(copy_sql, temp_file)
 
 
-def insert_from_select(from_table, target_table, engine, mappings=None):
-    target_table_col_list = [col for col in target_table.c if not MathesarColumn.from_column(col).is_default]
+def insert_from_select(from_table, target_table, engine, col_mappings=None):
+    if col_mappings:
+        from_table_col_list, target_table_col_list = zip(
+            *[
+                (from_table.c[from_col], target_table.c[target_col])
+                for from_col, target_col in col_mappings
+            ]
+        )
+    else:
+        from_table_col_list = [
+            col for col in from_table.c
+            if not MathesarColumn.from_column(col).is_default
+        ]
+        target_table_col_list = [
+            col for col in target_table.c
+            if not MathesarColumn.from_column(col).is_default
+        ]
     with engine.begin() as conn:
-        sel = select([col for col in from_table.c if not MathesarColumn.from_column(col).is_default])
+        sel = select(from_table_col_list)
         ins = target_table.insert().from_select(target_table_col_list, sel)
         try:
             result = conn.execute(ins)
diff --git a/mathesar/api/serializers/tables.py b/mathesar/api/serializers/tables.py
--- a/mathesar/api/serializers/tables.py
+++ b/mathesar/api/serializers/tables.py
@@ -195,11 +195,13 @@ class SplitTableResponseSerializer(MathesarErrorMessageMixin, serializers.Serial
 
 
 class MappingSerializer(MathesarErrorMessageMixin, serializers.Serializer):
-    # TBD
-    pass
+    def to_internal_value(self, data):
+        from_col = Column.current_objects.get(id=data[0])
+        target_col = Column.current_objects.get(id=data[1])
+        return [from_col, target_col]
 
 
 class TableImportSerializer(MathesarErrorMessageMixin, serializers.Serializer):
     import_target = serializers.PrimaryKeyRelatedField(queryset=Table.current_objects.all(), required=True)
     data_files = serializers.PrimaryKeyRelatedField(required=True, many=True, queryset=DataFile.objects.all())
-    mappings = MappingSerializer(required=True, allow_null=True)
+    mappings = MappingSerializer(required=True, allow_null=True, many=True)
diff --git a/mathesar/models/base.py b/mathesar/models/base.py
--- a/mathesar/models/base.py
+++ b/mathesar/models/base.py
@@ -445,9 +445,13 @@ def insert_records_to_existing_table(self, existing_table, data_files, mappings=
         from_table = self._sa_table
         target_table = existing_table._sa_table
         engine = self._sa_engine
+        if mappings:
+            col_mappings = [[from_col.name, target_col.name] for from_col, target_col in mappings]
+        else:
+            col_mappings = None
         data_file = data_files[0]
         try:
-            table, _ = insert_from_select(from_table, target_table, engine, mappings)
+            table, _ = insert_from_select(from_table, target_table, engine, col_mappings)
             data_file.table_imported_to = existing_table
         except Exception as e:
             # ToDo raise specific exceptions.


Test patch:
diff --git a/db/tests/conftest.py b/db/tests/conftest.py
--- a/db/tests/conftest.py
+++ b/db/tests/conftest.py
@@ -19,6 +19,8 @@
 FILTER_SORT_SQL = os.path.join(RESOURCES, "filter_sort_create.sql")
 MAGNITUDE_SQL = os.path.join(RESOURCES, "magnitude_testing_create.sql")
 JSON_SQL = os.path.join(RESOURCES, "json_sort.sql")
+BOOKS_FROM_SQL = os.path.join(RESOURCES, "books_import_from.sql")
+BOOKS_TARGET_SQL = os.path.join(RESOURCES, "books_import_target.sql")
 
 
 @pytest.fixture
@@ -84,6 +86,24 @@ def engine_with_magnitude(engine_with_schema):
     yield engine, schema
 
 
+@pytest.fixture
+def engine_with_books_to_import_from(engine_with_schema):
+    engine, schema = engine_with_schema
+    with engine.begin() as conn, open(BOOKS_FROM_SQL) as f:
+        conn.execute(text(f"SET search_path={schema}"))
+        conn.execute(text(f.read()))
+    return engine, schema
+
+
+@pytest.fixture
+def engine_with_books_import_target(engine_with_schema):
+    engine, schema = engine_with_schema
+    with engine.begin() as conn, open(BOOKS_TARGET_SQL) as f:
+        conn.execute(text(f"SET search_path={schema}"))
+        conn.execute(text(f.read()))
+    return engine, schema
+
+
 @pytest.fixture(scope='session')
 def roster_table_name():
     return "Roster"
@@ -124,6 +144,16 @@ def roster_fkey_col(teachers_table_name):
     return f"{teachers_table_name}_{constants.ID}"
 
 
+@pytest.fixture(scope='session')
+def books_import_from_table_name():
+    return "books_from"
+
+
+@pytest.fixture(scope='session')
+def books_import_target_table_name():
+    return "books_target"
+
+
 @pytest.fixture
 def extracted_remainder_roster(engine_with_roster, roster_table_name, roster_extracted_cols, teachers_table_name):
     engine, schema = engine_with_roster
@@ -200,3 +230,19 @@ def uris_table_obj(engine_with_uris, uris_table_name):
             uri_type,
         )
     yield table, engine
+
+
+@pytest.fixture
+def books_table_import_from_obj(engine_with_books_to_import_from, books_import_from_table_name):
+    engine, schema = engine_with_books_to_import_from
+    metadata = MetaData(bind=engine)
+    table = Table(books_import_from_table_name, metadata, schema=schema, autoload_with=engine)
+    return table, engine
+
+
+@pytest.fixture
+def books_table_import_target_obj(engine_with_books_import_target, books_import_target_table_name):
+    engine, schema = engine_with_books_import_target
+    metadata = MetaData(bind=engine)
+    table = Table(books_import_target_table_name, metadata, schema=schema, autoload_with=engine)
+    return table, engine
diff --git a/db/tests/records/operations/test_insert.py b/db/tests/records/operations/test_insert.py
new file mode 100644
--- /dev/null
+++ b/db/tests/records/operations/test_insert.py
@@ -0,0 +1,41 @@
+from db.records.operations.insert import insert_from_select
+from db.records.operations.select import get_records
+
+
+def test_insert_from_select_without_mappings(books_table_import_from_obj, books_table_import_target_obj):
+    # We need engine of the from_table since insert_from_select() is set-up from the 'from_table' side
+    from_table, engine = books_table_import_from_obj
+    target_table, _ = books_table_import_target_obj
+    records_without_mappings = [
+        (1, 'Steve Jobs', 'Walter Issacson'),
+        (2, 'The Idiot', 'Fyodor Dostevsky'),
+        (3, 'David Copperfield', 'Charles Darwin'),
+        (4, 'Fyodor Dostoevsky', 'Crime and Punishment'),
+        (5, 'Cervantes', 'Don Quixote')
+    ]
+    res_table, _ = insert_from_select(from_table, target_table, engine, col_mappings=None)
+    records = get_records(res_table, engine)
+    assert res_table.c['id'] == target_table.c[0]
+    assert res_table.c['title'] == target_table.c[1]
+    assert res_table.c['author'] == target_table.c[2]
+    assert records == records_without_mappings
+
+
+def test_insert_from_select_with_mappings(books_table_import_from_obj, books_table_import_target_obj):
+    # We need engine of the from_table since insert_from_select() is set-up from the 'from_table' side
+    from_table, engine = books_table_import_from_obj
+    target_table, _ = books_table_import_target_obj
+    records_with_mappings = [
+        (1, 'Steve Jobs', 'Walter Issacson'),
+        (2, 'The Idiot', 'Fyodor Dostevsky'),
+        (3, 'David Copperfield', 'Charles Darwin'),
+        (4, 'Crime and Punishment', 'Fyodor Dostoevsky'),
+        (5, 'Don Quixote', 'Cervantes')
+    ]
+    col_mappings = [['book_title', 'title'], ['author_name', 'author']]
+    res_table, _ = insert_from_select(from_table, target_table, engine, col_mappings)
+    records = get_records(res_table, engine)
+    assert res_table.c['id'] == target_table.c[0]
+    assert res_table.c['title'] == target_table.c[1]
+    assert res_table.c['author'] == target_table.c[2]
+    assert records == records_with_mappings
diff --git a/db/tests/resources/books_import_from.sql b/db/tests/resources/books_import_from.sql
new file mode 100644
--- /dev/null
+++ b/db/tests/resources/books_import_from.sql
@@ -0,0 +1,6 @@
+CREATE TABLE "books_from"(
+id SERIAL PRIMARY KEY,
+"author_name" TEXT,
+"book_title" TEXT);
+
+INSERT INTO "books_from"(author_name, book_title) VALUES ('Fyodor Dostoevsky', 'Crime and Punishment'), ('Cervantes', 'Don Quixote');
diff --git a/db/tests/resources/books_import_target.sql b/db/tests/resources/books_import_target.sql
new file mode 100644
--- /dev/null
+++ b/db/tests/resources/books_import_target.sql
@@ -0,0 +1,6 @@
+CREATE TABLE "books_target"(
+id SERIAL PRIMARY KEY,
+"title" TEXT,
+"author" TEXT);
+
+INSERT INTO "books_target"(title, author) VALUES ('Steve Jobs', 'Walter Issacson'), ('The Idiot', 'Fyodor Dostevsky'), ('David Copperfield', 'Charles Darwin');