Repository: mathesar-foundation/mathesar

Issue:
Batch updating column using `/tables/` api should reference column using `id` instead of `column_index`
## Problem
Batch updating Columns using the `columns` field in the `patch` method of the `/tables/{id}/` does not use `id` of the column rather the column to update is identified based on the position of the column objects. This is not ideal due to the following reasons:
- Similar to the `column_index` which is [deprecated in favour](https://github.com/centerofci/mathesar/issues/839) of `id`.
- Column position could have changed resulting in updating a wrong column


## Proposed solution
The following changes have to be made:

-`PATCH` method of the `/tables/{id}` api which has the following structure 

```
{
    "columns": [
        {
            "name": "id",
            "type": "INTEGER"
        },
        {
            "name": "Center",
            "type": "TEXT"
        }
    ]
}
```

should be replaced with 

```
{
    "columns": [
        {
            "id": 1, // Use column id to reference a column, rather than assuming the column based on its index.
            "name": "id",
            "type": "INTEGER"
        },
        {
            "id": 2,
            "name": "Center",
            "type": "TEXT"
        }
    ]
}
```

- The column id's have to be converted into `attnums` which can be fetched from the column object before sending the column data from the service layer to the database layer(before calling any function from the `db` module)


Discussion:
@silentninja Can I work on this?
@ratika-12 This issue needs some understanding of the codebase and Django. I will confirm in a few minutes, sorry for the wait.
@ratika-12 I have assigned the issue to you, thanks!
@ratika-12 I have updated the proposed solution to make it easier to understand. Let me know if anything is not clear. 
@ratika-12 Are you still working on this task? I figured I would follow up since it's been a while and there has not been a PR yet.
@kgodey Yes I am working on this task. I will make a pr for this asap.
@ratika-12 Are you still working on this task? Just wanted to follow up. Also please push your work to a draft PR, so that it is easy to follow up and suggest changes
@kgodey @silentninja Can I work on this if this is not being worked on currently by @ratika-12?
Yeah sure !!
Thanks @Anish9901, assigned to you. And thanks for the update @ratika-12.

Resolving patch:
diff --git a/db/columns/operations/alter.py b/db/columns/operations/alter.py
--- a/db/columns/operations/alter.py
+++ b/db/columns/operations/alter.py
@@ -182,7 +182,7 @@ def _check_type_option_equivalence(type_options_1, type_options_2):
 
 
 def _validate_columns_for_batch_update(table, column_data):
-    ALLOWED_KEYS = ['name', 'plain_type', 'type_options']
+    ALLOWED_KEYS = ['attnum', 'name', 'plain_type', 'type_options']
     if len(column_data) != len(table.columns):
         raise ValueError('Number of columns passed in must equal number of columns in table')
     for single_column_data in column_data:
@@ -193,38 +193,42 @@ def _validate_columns_for_batch_update(table, column_data):
 
 
 def _batch_update_column_types(table_oid, column_data_list, connection, engine):
-    table = reflect_table_from_oid(table_oid, engine, connection)
     for index, column_data in enumerate(column_data_list):
-        if 'plain_type' in column_data:
+        column_attnum = column_data.get('attnum', None)
+        if 'plain_type' in column_data and column_attnum is not None:
             new_type = column_data['plain_type']
             type_options = column_data.get('type_options', {})
             if type_options is None:
                 type_options = {}
-            column_attnum = get_column_attnum_from_name(table_oid, table.columns[index].name, engine, connection)
             retype_column(table_oid, column_attnum, engine, connection, new_type, type_options)
 
 
-def _batch_alter_table_rename_columns(table, column_data_list, connection):
+def _batch_alter_table_rename_columns(table_oid, column_data_list, connection, engine):
+    table = reflect_table_from_oid(table_oid, engine, connection)
     ctx = MigrationContext.configure(connection)
     op = Operations(ctx)
     with op.batch_alter_table(table.name, schema=table.schema) as batch_op:
         for index, column_data in enumerate(column_data_list):
-            column = table.columns[index]
-            if 'name' in column_data and column.name != column_data['name']:
+            column_attnum = column_data.get('attnum', None)
+            if column_attnum is not None:
+                name = get_column_name_from_attnum(table_oid, column_attnum, engine, connection)
+            if 'name' in column_data and name != column_data['name']:
                 batch_op.alter_column(
-                    column.name,
+                    name,
                     new_column_name=column_data['name']
                 )
 
 
-def _batch_alter_table_drop_columns(table, column_data_list, connection):
+def _batch_alter_table_drop_columns(table_oid, column_data_list, connection, engine):
+    table = reflect_table_from_oid(table_oid, engine, connection)
     ctx = MigrationContext.configure(connection)
     op = Operations(ctx)
     with op.batch_alter_table(table.name, schema=table.schema) as batch_op:
         for index, column_data in enumerate(column_data_list):
-            column = table.columns[index]
-            if len(column_data.keys()) == 0:
-                batch_op.drop_column(column.name)
+            column_attnum = column_data.get('attnum', None)
+            if column_attnum is not None and len(column_data.keys()) == 1:
+                name = get_column_name_from_attnum(table_oid, column_attnum, engine, connection)
+                batch_op.drop_column(name)
 
 
 def batch_update_columns(table_oid, engine, column_data_list):
@@ -232,5 +236,5 @@ def batch_update_columns(table_oid, engine, column_data_list):
     _validate_columns_for_batch_update(table, column_data_list)
     with engine.begin() as conn:
         _batch_update_column_types(table_oid, column_data_list, conn, engine)
-        _batch_alter_table_rename_columns(table, column_data_list, conn)
-        _batch_alter_table_drop_columns(table, column_data_list, conn)
+        _batch_alter_table_rename_columns(table_oid, column_data_list, conn, engine)
+        _batch_alter_table_drop_columns(table_oid, column_data_list, conn, engine)
diff --git a/mathesar/api/serializers/columns.py b/mathesar/api/serializers/columns.py
--- a/mathesar/api/serializers/columns.py
+++ b/mathesar/api/serializers/columns.py
@@ -58,6 +58,7 @@ class Meta:
                   'type_options',
                   'display_options'
                   )
+    id = serializers.IntegerField(required=False)
     name = serializers.CharField()
     type = serializers.CharField(source='plain_type')
     type_options = TypeOptionSerializer(required=False, allow_null=True)
diff --git a/mathesar/utils/models.py b/mathesar/utils/models.py
--- a/mathesar/utils/models.py
+++ b/mathesar/utils/models.py
@@ -25,7 +25,8 @@ def update_sa_table(table, validated_data):
     if errors:
         raise base_api_exceptions.GenericAPIException(errors, status_code=status.HTTP_400_BAD_REQUEST)
     try:
-        alter_table(table.name, table.oid, table.schema.name, table.schema._sa_engine, validated_data)
+        data = update_id_to_attnum(table, validated_data)
+        alter_table(table.name, table.oid, table.schema.name, table.schema._sa_engine, data)
         reflect_columns_from_table(table)
     # TODO: Catch more specific exceptions
     except Exception as e:
@@ -43,3 +44,16 @@ def update_sa_schema(schema, validated_data):
     if errors:
         raise ValidationError(errors)
     alter_schema(schema.name, schema._sa_engine, validated_data)
+
+
+def update_id_to_attnum(table, validated_data):
+    if 'columns' in validated_data:
+        data = validated_data.get('columns')
+        queryset = table.columns.all()
+        for column_data in data:
+            col_id = column_data.get('id', None)
+            if col_id is not None:
+                attnum = queryset.get(id=col_id).attnum
+                column_data['attnum'] = attnum
+                column_data.pop('id')
+    return validated_data


Test patch:
diff --git a/db/tests/columns/operations/test_alter.py b/db/tests/columns/operations/test_alter.py
--- a/db/tests/columns/operations/test_alter.py
+++ b/db/tests/columns/operations/test_alter.py
@@ -56,8 +56,8 @@ def _create_pizza_table(engine, schema):
     return create_test_table(table_name, cols, insert_data, schema, engine)
 
 
-def _get_pizza_column_data():
-    return [{
+def _get_pizza_column_data(table_oid, engine):
+    column_data = [{
         'name': 'ID',
         'plain_type': 'VARCHAR'
     }, {
@@ -70,6 +70,10 @@ def _get_pizza_column_data():
         'name': 'Rating',
         'plain_type': 'VARCHAR'
     }]
+    for data in column_data:
+        name = data['name']
+        data['attnum'] = get_column_attnum_from_name(table_oid, name, engine)
+    return column_data
 
 
 @pytest.mark.parametrize(
@@ -435,7 +439,8 @@ def test_batch_update_columns_no_changes(engine_email_type):
     table = _create_pizza_table(engine, schema)
     table_oid = get_oid_from_table(table.name, schema, engine)
 
-    batch_update_columns(table_oid, engine, _get_pizza_column_data())
+    column_data = _get_pizza_column_data(table_oid, engine)
+    batch_update_columns(table_oid, engine, column_data)
     updated_table = reflect_table(table.name, schema, engine)
 
     assert len(table.columns) == len(updated_table.columns)
@@ -450,7 +455,7 @@ def test_batch_update_column_names(engine_email_type):
     table = _create_pizza_table(engine, schema)
     table_oid = get_oid_from_table(table.name, schema, engine)
 
-    column_data = _get_pizza_column_data()
+    column_data = _get_pizza_column_data(table_oid, engine)
     column_data[1]['name'] = 'Pizza Style'
     column_data[2]['name'] = 'Eaten Recently?'
 
@@ -469,7 +474,7 @@ def test_batch_update_column_types(engine_email_type):
     table = _create_pizza_table(engine, schema)
     table_oid = get_oid_from_table(table.name, schema, engine)
 
-    column_data = _get_pizza_column_data()
+    column_data = _get_pizza_column_data(table_oid, engine)
     column_data[0]['plain_type'] = 'DOUBLE PRECISION'
     column_data[2]['plain_type'] = 'BOOLEAN'
 
@@ -488,7 +493,7 @@ def test_batch_update_column_names_and_types(engine_email_type):
     table = _create_pizza_table(engine, schema)
     table_oid = get_oid_from_table(table.name, schema, engine)
 
-    column_data = _get_pizza_column_data()
+    column_data = _get_pizza_column_data(table_oid, engine)
     column_data[0]['name'] = 'Pizza ID'
     column_data[0]['plain_type'] = 'INTEGER'
     column_data[1]['name'] = 'Pizza Style'
@@ -509,9 +514,9 @@ def test_batch_update_column_drop_columns(engine_email_type):
     table = _create_pizza_table(engine, schema)
     table_oid = get_oid_from_table(table.name, schema, engine)
 
-    column_data = _get_pizza_column_data()
-    column_data[0] = {}
-    column_data[1] = {}
+    column_data = _get_pizza_column_data(table_oid, engine)
+    column_data[0] = {'attnum': get_column_attnum_from_name(table_oid, column_data[0]['name'], engine)}
+    column_data[1] = {'attnum': get_column_attnum_from_name(table_oid, column_data[1]['name'], engine)}
 
     batch_update_columns(table_oid, engine, column_data)
     updated_table = reflect_table(table.name, schema, engine)
@@ -528,12 +533,12 @@ def test_batch_update_column_all_operations(engine_email_type):
     table = _create_pizza_table(engine, schema)
     table_oid = get_oid_from_table(table.name, schema, engine)
 
-    column_data = _get_pizza_column_data()
+    column_data = _get_pizza_column_data(table_oid, engine)
     column_data[0]['name'] = 'Pizza ID'
     column_data[0]['plain_type'] = 'INTEGER'
     column_data[1]['name'] = 'Pizza Style'
     column_data[2]['plain_type'] = 'BOOLEAN'
-    column_data[3] = {}
+    column_data[3] = {'attnum': get_column_attnum_from_name(table_oid, column_data[3]['name'], engine)}
 
     batch_update_columns(table_oid, engine, column_data)
     updated_table = reflect_table(table.name, schema, engine)
diff --git a/mathesar/tests/api/test_table_api.py b/mathesar/tests/api/test_table_api.py
--- a/mathesar/tests/api/test_table_api.py
+++ b/mathesar/tests/api/test_table_api.py
@@ -997,8 +997,8 @@ def test_table_viewset_checks_cache(client):
     mock_reflect.assert_called()
 
 
-def _get_patents_column_data():
-    return [{
+def _get_patents_column_data(table):
+    column_data = [{
         'name': 'id',
         'type': 'INTEGER',
     }, {
@@ -1023,6 +1023,11 @@ def _get_patents_column_data():
         'name': 'Patent Expiration Date',
         'type': 'VARCHAR',
     }]
+    bidirectmap = table.get_column_name_id_bidirectional_map()
+    for data in column_data:
+        name = data['name']
+        data['id'] = bidirectmap[name]
+    return column_data
 
 
 def test_table_patch_same_table_name(create_table, client):
@@ -1044,7 +1049,7 @@ def test_table_patch_columns_and_table_name(create_table, client):
 
     body = {
         'name': 'PATCH COLUMNS 1',
-        'columns': _get_patents_column_data()
+        'columns': _get_patents_column_data(table)
     }
     # Need to specify format here because otherwise the body gets sent
     # as a multi-part form, which can't handle nested keys.
@@ -1058,7 +1063,7 @@ def test_table_patch_columns_and_table_name(create_table, client):
 def test_table_patch_columns_no_changes(create_table, client, engine_email_type):
     table_name = 'PATCH columns 2'
     table = create_table(table_name)
-    column_data = _get_patents_column_data()
+    column_data = _get_patents_column_data(table)
 
     body = {
         'columns': column_data
@@ -1073,7 +1078,7 @@ def test_table_patch_columns_no_changes(create_table, client, engine_email_type)
 def test_table_patch_columns_one_name_change(create_table, client, engine_email_type):
     table_name = 'PATCH columns 3'
     table = create_table(table_name)
-    column_data = _get_patents_column_data()
+    column_data = _get_patents_column_data(table)
     column_data[1]['name'] = 'NASA Center'
 
     body = {
@@ -1089,7 +1094,7 @@ def test_table_patch_columns_one_name_change(create_table, client, engine_email_
 def test_table_patch_columns_two_name_changes(create_table, client, engine_email_type):
     table_name = 'PATCH columns 4'
     table = create_table(table_name)
-    column_data = _get_patents_column_data()
+    column_data = _get_patents_column_data(table)
     column_data[1]['name'] = 'NASA Center'
     column_data[2]['name'] = 'Patent Status'
 
@@ -1106,7 +1111,7 @@ def test_table_patch_columns_two_name_changes(create_table, client, engine_email
 def test_table_patch_columns_one_type_change(create_table, client, engine_email_type):
     table_name = 'PATCH columns 5'
     table = create_table(table_name)
-    column_data = _get_patents_column_data()
+    column_data = _get_patents_column_data(table)
     column_data[7]['type'] = 'DATE'
 
     body = {
@@ -1119,8 +1124,8 @@ def test_table_patch_columns_one_type_change(create_table, client, engine_email_
     _check_columns(response_json['columns'], column_data)
 
 
-def _get_data_types_column_data():
-    return [{
+def _get_data_types_column_data(table):
+    column_data = [{
         'name': 'id',
         'type': 'INTEGER'
     }, {
@@ -1136,12 +1141,17 @@ def _get_data_types_column_data():
         'name': 'Decimal',
         'type': 'TEXT'
     }]
+    bidirectmap = table.get_column_name_id_bidirectional_map()
+    for data in column_data:
+        name = data['name']
+        data['id'] = bidirectmap[name]
+    return column_data
 
 
 def test_table_patch_columns_multiple_type_change(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 6'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     column_data[1]['type'] = 'INTEGER'
     column_data[2]['type'] = 'BOOLEAN'
     column_data[4]['type'] = 'NUMERIC'
@@ -1164,8 +1174,8 @@ def _check_columns_with_dropped(response_column_data, request_column_data, dropp
 def test_table_patch_columns_one_drop(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 7'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
-    column_data[1] = {}
+    column_data = _get_data_types_column_data(table)
+    column_data[1] = {'id': column_data[1]['id']}
 
     body = {
         'columns': column_data
@@ -1181,9 +1191,9 @@ def test_table_patch_columns_multiple_drop(create_data_types_table, client, engi
     INDICES_TO_DROP = [1, 2]
     table_name = 'PATCH columns 8'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     for index in INDICES_TO_DROP:
-        column_data[index] = {}
+        column_data[index] = {'id': column_data[index]['id']}
 
     body = {
         'columns': column_data
@@ -1198,7 +1208,7 @@ def test_table_patch_columns_multiple_drop(create_data_types_table, client, engi
 def test_table_patch_columns_diff_name_type_change(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 9'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     column_data[1]['type'] = 'INTEGER'
     column_data[2]['name'] = 'Checkbox'
 
@@ -1215,7 +1225,7 @@ def test_table_patch_columns_diff_name_type_change(create_data_types_table, clie
 def test_table_patch_columns_same_name_type_change(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 10'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     column_data[2]['type'] = 'BOOLEAN'
     column_data[2]['name'] = 'Checkbox'
 
@@ -1232,7 +1242,7 @@ def test_table_patch_columns_same_name_type_change(create_data_types_table, clie
 def test_table_patch_columns_multiple_name_type_change(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 11'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     column_data[1]['type'] = 'INTEGER'
     column_data[1]['name'] = 'Int.'
     column_data[2]['type'] = 'BOOLEAN'
@@ -1251,10 +1261,10 @@ def test_table_patch_columns_multiple_name_type_change(create_data_types_table,
 def test_table_patch_columns_diff_name_type_drop(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 12'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     column_data[1]['type'] = 'INTEGER'
     column_data[2]['name'] = 'Checkbox'
-    column_data[3] = {}
+    column_data[3] = {'id': column_data[3]['id']}
 
     body = {
         'columns': column_data
@@ -1269,11 +1279,11 @@ def test_table_patch_columns_diff_name_type_drop(create_data_types_table, client
 def test_table_patch_columns_same_name_type_drop(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 13'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
-    column_data[1] = {}
+    column_data = _get_data_types_column_data(table)
+    column_data[1] = {'id': column_data[1]['id']}
     column_data[2]['type'] = 'BOOLEAN'
     column_data[2]['name'] = 'Checkbox'
-    column_data[3] = {}
+    column_data[3] = {'id': column_data[3]['id']}
 
     body = {
         'columns': column_data
@@ -1288,7 +1298,7 @@ def test_table_patch_columns_same_name_type_drop(create_data_types_table, client
 def test_table_patch_columns_invalid_type(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 14'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     column_data[3]['type'] = 'BOOLEAN'
 
     body = {
@@ -1304,7 +1314,7 @@ def test_table_patch_columns_invalid_type(create_data_types_table, client, engin
 def test_table_patch_columns_invalid_type_with_name(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 15'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     column_data[1]['name'] = 'hello'
     column_data[3]['type'] = 'BOOLEAN'
 
@@ -1318,14 +1328,14 @@ def test_table_patch_columns_invalid_type_with_name(create_data_types_table, cli
 
     current_table_response = client.get(f'/api/db/v0/tables/{table.id}/')
     # The table should not have changed
-    original_column_data = _get_data_types_column_data()
+    original_column_data = _get_data_types_column_data(table)
     _check_columns(current_table_response.json()['columns'], original_column_data)
 
 
 def test_table_patch_columns_invalid_type_with_type(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 16'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
+    column_data = _get_data_types_column_data(table)
     column_data[1]['type'] = 'INTEGER'
     column_data[3]['type'] = 'BOOLEAN'
 
@@ -1339,15 +1349,15 @@ def test_table_patch_columns_invalid_type_with_type(create_data_types_table, cli
 
     current_table_response = client.get(f'/api/db/v0/tables/{table.id}/')
     # The table should not have changed
-    original_column_data = _get_data_types_column_data()
+    original_column_data = _get_data_types_column_data(table)
     _check_columns(current_table_response.json()['columns'], original_column_data)
 
 
 def test_table_patch_columns_invalid_type_with_drop(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 17'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
-    column_data[1] = {}
+    column_data = _get_data_types_column_data(table)
+    column_data[1] = {'id': column_data[1]['id']}
     column_data[3]['type'] = 'BOOLEAN'
 
     body = {
@@ -1360,15 +1370,15 @@ def test_table_patch_columns_invalid_type_with_drop(create_data_types_table, cli
 
     current_table_response = client.get(f'/api/db/v0/tables/{table.id}/')
     # The table should not have changed
-    original_column_data = _get_data_types_column_data()
+    original_column_data = _get_data_types_column_data(table)
     _check_columns(current_table_response.json()['columns'], original_column_data)
 
 
 def test_table_patch_columns_invalid_type_with_multiple_changes(create_data_types_table, client, engine_email_type):
     table_name = 'PATCH columns 18'
     table = create_data_types_table(table_name)
-    column_data = _get_data_types_column_data()
-    column_data[1] = {}
+    column_data = _get_data_types_column_data(table)
+    column_data[1] = {'id': column_data[1]['id']}
     column_data[2]['name'] = 'Checkbox'
     column_data[2]['type'] = 'BOOLEAN'
     column_data[3]['type'] = 'BOOLEAN'
@@ -1383,5 +1393,5 @@ def test_table_patch_columns_invalid_type_with_multiple_changes(create_data_type
 
     current_table_response = client.get(f'/api/db/v0/tables/{table.id}/')
     # The table should not have changed
-    original_column_data = _get_data_types_column_data()
+    original_column_data = _get_data_types_column_data(table)
     _check_columns(current_table_response.json()['columns'], original_column_data)