Repository: mathesar-foundation/mathesar

Issue:
Double quotes in file headers break CSV import
## Description
A CSV file with double quotes in its header names will cause the CSV import to break. This is because we directly insert the CSV header names into the SQL statement that creates the new, imported table.

## Expected behavior
Quotes should either be ignored or included in the column name. Regardless, the import should not fail.

## To Reproduce
Make a copy of any testing CSV, and add double quotes ~around~ inside the name of a column. Then, try to upload.

## Additional context
I think sqlalchemy should automatically escape column names? If this is the case, creating `Column` objects then just using their name or str representation should work.



Discussion:
I'll fix this bug.
[update]: 

I tried these 3 solutions above:

1. `list(map(lambda col: col.replace('"', ""), reader.fieldnames))`

2. `list(map(lambda col: col.strip('\"'), reader.fieldnames))`

3. `list(map(lambda col: eval(col), reader.fieldnames))`

All three solutions can handle our issue **Double quotes in file headers**. But **solution 2** and **solution 3** cannot deal with the specific corner case: `'" column"_one "'` where a double-quote was added in the middle of the header's name.

As a result, I decide to choose **solution 1** as our first feasible solution.

Resolving patch:
diff --git a/db/records.py b/db/records.py
--- a/db/records.py
+++ b/db/records.py
@@ -1,4 +1,5 @@
 import logging
+from psycopg2 import sql
 from sqlalchemy import delete, select, Column, func, true, and_
 from sqlalchemy.inspection import inspect
 from sqlalchemy_filters import apply_filters, apply_sort
@@ -299,23 +300,36 @@ def create_records_from_csv(
         escape=None,
         quote=None,
 ):
-    with open(csv_filename, 'rb') as csv_file:
+    with open(csv_filename, "rb") as csv_file:
         with engine.begin() as conn:
             cursor = conn.connection.cursor()
-            relation = '.'.join('"{}"'.format(part) for part in (table.schema, table.name))
-            formatted_columns = '({})'.format(','.join([f'"{column_name}"' for column_name in column_names]))
-
-            copy_sql = f'COPY {relation} {formatted_columns} FROM STDIN CSV'
-            if header:
-                copy_sql += " HEADER"
-            if delimiter:
-                copy_sql += f" DELIMITER E'{delimiter}'"
-            if escape:
-                copy_sql += f" ESCAPE '{escape}'"
-            if quote:
-                if quote == "'":
-                    quote = "''"
-                copy_sql += f" QUOTE '{quote}'"
+            # We should convert our entire query to sql.SQL class in order to keep its original header's name
+            # When we call sql.Indentifier which will return a Identifier class (based on sql.Composable)
+            # instead of a String. So we have to convert our punctuations to sql.Composable using sql.SQL
+            relation = sql.SQL(".").join(
+                sql.Identifier(part) for part in (table.schema, table.name)
+            )
+            formatted_columns = sql.SQL(",").join(
+                sql.Identifier(column_name) for column_name in column_names
+            )
+
+            copy_sql = sql.SQL(
+                "COPY {relation} ({formatted_columns}) FROM STDIN CSV {header} {delimiter} {escape} {quote}"
+            ).format(
+                relation=relation,
+                formatted_columns=formatted_columns,
+                # If HEADER is not None, we'll pass its value to our entire SQL query
+                header=sql.SQL("HEADER" if header else ""),
+                # If DELIMITER is not None, we'll pass its value to our entire SQL query
+                delimiter=sql.SQL(f"DELIMITER E'{delimiter}'" if delimiter else ""),
+                # If ESCAPE is not None, we'll pass its value to our entire SQL query
+                escape=sql.SQL(f"ESCAPE '{escape}'" if escape else ""),
+                quote=sql.SQL(
+                    ("QUOTE ''''" if quote == "'" else f"QUOTE '{quote}'")
+                    if quote
+                    else ""
+                ),
+            )
 
             cursor.copy_expert(copy_sql, csv_file)
 
diff --git a/mathesar/imports/csv.py b/mathesar/imports/csv.py
--- a/mathesar/imports/csv.py
+++ b/mathesar/imports/csv.py
@@ -7,7 +7,7 @@
 from db import tables, records
 from mathesar.errors import InvalidTableError
 
-ALLOWED_DELIMITERS = ",\t:| "
+ALLOWED_DELIMITERS = ",\t:|"
 SAMPLE_SIZE = 20000
 CHECK_ROWS = 10
 
@@ -80,6 +80,7 @@ def get_sv_reader(file, header, dialect=None):
             f"column_{i}" for i in range(len(reader.fieldnames))
         ]
         file.seek(0)
+
     return reader
 
 


Test patch:
diff --git a/mathesar/tests/data/csv_parsing/double_quote_in_header.csv b/mathesar/tests/data/csv_parsing/double_quote_in_header.csv
new file mode 100644
--- /dev/null
+++ b/mathesar/tests/data/csv_parsing/double_quote_in_header.csv
@@ -0,0 +1,2 @@
+"""Center ""NASA"" USA""","""Status""","""Case Number""","""Patent Number""","""Application SN""","""Title,Patent Expiration Date"""
+NASA Kennedy Space Center,Application,"KSC-12871,0","13/033,085",Polyimide Wire Insulation Repair System,
\ No newline at end of file
diff --git a/mathesar/tests/imports/test_csv.py b/mathesar/tests/imports/test_csv.py
--- a/mathesar/tests/imports/test_csv.py
+++ b/mathesar/tests/imports/test_csv.py
@@ -6,22 +6,22 @@
 
 from mathesar.models import DataFile, Schema
 from mathesar.errors import InvalidTableError
-from mathesar.imports.csv import create_table_from_csv, get_sv_dialect
+from mathesar.imports.csv import create_table_from_csv, get_sv_dialect, get_sv_reader
 from db.schemas import create_schema, get_schema_oid_from_name
 
-TEST_SCHEMA = 'import_csv_schema'
+TEST_SCHEMA = "import_csv_schema"
 
 
 @pytest.fixture
 def data_file(csv_filename):
-    with open(csv_filename, 'rb') as csv_file:
+    with open(csv_filename, "rb") as csv_file:
         data_file = DataFile.objects.create(file=File(csv_file))
     return data_file
 
 
 @pytest.fixture
 def headerless_data_file(headerless_csv_filename):
-    with open(headerless_csv_filename, 'rb') as csv_file:
+    with open(headerless_csv_filename, "rb") as csv_file:
         data_file = DataFile.objects.create(file=File(csv_file), header=False)
     return data_file
 
@@ -45,29 +45,50 @@ def check_csv_upload(table, table_name, schema, num_records, row, cols):
 
 
 def test_csv_upload(data_file, schema):
-    table_name = 'NASA 1'
+    table_name = "NASA 1"
     table = create_table_from_csv(data_file, table_name, schema)
 
     num_records = 1393
-    expected_row = (1, 'NASA Kennedy Space Center', 'Application', 'KSC-12871', '0',
-                    '13/033,085', 'Polyimide Wire Insulation Repair System', None)
-    expected_cols = ['Center', 'Status', 'Case Number', 'Patent Number',
-                     'Application SN', 'Title', 'Patent Expiration Date']
+    expected_row = (
+        1,
+        "NASA Kennedy Space Center",
+        "Application",
+        "KSC-12871",
+        "0",
+        "13/033,085",
+        "Polyimide Wire Insulation Repair System",
+        None,
+    )
+    expected_cols = [
+        "Center",
+        "Status",
+        "Case Number",
+        "Patent Number",
+        "Application SN",
+        "Title",
+        "Patent Expiration Date",
+    ]
     check_csv_upload(
         table, table_name, schema, num_records, expected_row, expected_cols
     )
 
 
 def test_headerless_csv_upload(headerless_data_file, schema):
-    table_name = 'NASA no headers'
-    table = create_table_from_csv(
-        headerless_data_file, table_name, schema
-    )
+    table_name = "NASA no headers"
+    table = create_table_from_csv(headerless_data_file, table_name, schema)
 
     num_records = 1393
-    expected_row = (1, 'NASA Kennedy Space Center', 'Application', 'KSC-12871', '0',
-                    '13/033,085', 'Polyimide Wire Insulation Repair System', None)
-    expected_cols = ['column_' + str(i) for i in range(7)]
+    expected_row = (
+        1,
+        "NASA Kennedy Space Center",
+        "Application",
+        "KSC-12871",
+        "0",
+        "13/033,085",
+        "Polyimide Wire Insulation Repair System",
+        None,
+    )
+    expected_cols = ["column_" + str(i) for i in range(7)]
 
     check_csv_upload(
         table, table_name, schema, num_records, expected_row, expected_cols
@@ -75,8 +96,8 @@ def test_headerless_csv_upload(headerless_data_file, schema):
 
 
 def test_csv_upload_with_duplicate_table_name(data_file, schema):
-    table_name = 'NASA 2'
-    already_defined_str = ('relation "NASA 2" already exists')
+    table_name = "NASA 2"
+    already_defined_str = 'relation "NASA 2" already exists'
 
     table = create_table_from_csv(data_file, table_name, schema)
     assert table is not None
@@ -90,23 +111,23 @@ def test_csv_upload_with_duplicate_table_name(data_file, schema):
 
 
 def test_csv_upload_table_imported_to(data_file, schema):
-    table = create_table_from_csv(data_file, 'NASA', schema)
+    table = create_table_from_csv(data_file, "NASA", schema)
     data_file.refresh_from_db()
     assert data_file.table_imported_to == table
 
 
 get_dialect_test_list = [
-    (',', '"', '', 'mathesar/tests/data/patents.csv'),
-    ('\t', '"', '', 'mathesar/tests/data/patents.tsv'),
-    (',', "'", '', 'mathesar/tests/data/csv_parsing/mixed_quote.csv'),
-    (',', '"', '', 'mathesar/tests/data/csv_parsing/double_quote.csv'),
-    (',', '"', '\\', 'mathesar/tests/data/csv_parsing/escaped_quote.csv'),
+    (",", '"', "", "mathesar/tests/data/patents.csv"),
+    ("\t", '"', "", "mathesar/tests/data/patents.tsv"),
+    (",", "'", "", "mathesar/tests/data/csv_parsing/mixed_quote.csv"),
+    (",", '"', "", "mathesar/tests/data/csv_parsing/double_quote.csv"),
+    (",", '"', "\\", "mathesar/tests/data/csv_parsing/escaped_quote.csv"),
 ]
 
 
 @pytest.mark.parametrize("exp_delim,exp_quote,exp_escape,file", get_dialect_test_list)
 def test_sv_get_dialect(exp_delim, exp_quote, exp_escape, file):
-    with open(file, 'r') as sv_file:
+    with open(file, "r") as sv_file:
         dialect = get_sv_dialect(sv_file)
     assert dialect.delimiter == exp_delim
     assert dialect.quotechar == exp_quote
@@ -114,14 +135,33 @@ def test_sv_get_dialect(exp_delim, exp_quote, exp_escape, file):
 
 
 get_dialect_exceptions_test_list = [
-    ('mathesar/tests/data/csv_parsing/patents_invalid.csv'),
-    ('mathesar/tests/data/csv_parsing/extra_quote_invalid.csv'),
-    ('mathesar/tests/data/csv_parsing/escaped_quote_invalid.csv'),
+    "mathesar/tests/data/csv_parsing/patents_invalid.csv",
+    "mathesar/tests/data/csv_parsing/extra_quote_invalid.csv",
+    "mathesar/tests/data/csv_parsing/escaped_quote_invalid.csv",
 ]
 
 
 @pytest.mark.parametrize("file", get_dialect_exceptions_test_list)
 def test_sv_get_dialect_exceptions(file):
     with pytest.raises(InvalidTableError):
-        with open(file, 'r') as sv_file:
+        with open(file, "r") as sv_file:
             get_sv_dialect(sv_file)
+
+
+get_header_test_list = [
+    "mathesar/tests/data/csv_parsing/double_quote_in_header.csv",
+]
+
+
+@pytest.mark.parametrize("file", get_header_test_list)
+def test_sv_get_header(file):
+    with open(file, "rb") as sv_file:
+        table = get_sv_reader(file=sv_file, header=True)
+        assert table.fieldnames == [
+            "\"Center \"NASA\" USA\"",
+            "\"Status\"",
+            "\"Case Number\"",
+            "\"Patent Number\"",
+            "\"Application SN\"",
+            "\"Title,Patent Expiration Date\"",
+        ]