Repository: ckan/ckan

Issue:
datastore_create: allow deleting columns
`datastore_create` can be used to create and update datastore tables including adding columns, but if you remove columns from the `fields` list they are not deleted.

Assuming no-one is relying on this behavior, let's allow deleting datastore columns by removing them from the `fields` list passed to `datastore_create`. If someone is relying on columns not passed being kept we could add a `delete_columns` option but that's not the most consistent/obvious API.

## Current behavior:
- Columns passed to `fields` must match, in order, existing columns or a ValidationError is raised. Existing columns missing at the end are ignored.
- Column types must be either included in `fields` or guessable from the first value in the `records` list or a ValidationError is raised.
- Keys included in the first value of the `records` list are added as new columns if they don't match an existing column id. Types for these columns are guessed from the corresponding value in the first record.

## Proposed change:
- When `fields` column list is passed, ignore the order (no longer cause ValidationError) and any existing columns not included will be deleted

### Aside: current datastore_create type guessing logic
| first record value | guessed type |
| --- | --- |
| object, list | `json` (with `nested` composite type used for historical reasons) |
| int | `int4` (with overflow/underflow issues) |
| float | `float8` (IEEE float, fast but not exact) |
| string parseable as int | `int4` (with overflow/underflow issues) |
| string parseable as float | `numeric` (exact, slow) |
| string in one of 8 different date/time formats | `timestamp` with postgres DMY/MDY configuration determining parsing order (errors possible) |
| otherwise | `text` |



Discussion:

Resolving patch:
diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py
--- a/ckanext/datastore/backend/postgres.py
+++ b/ckanext/datastore/backend/postgres.py
@@ -1018,7 +1018,7 @@ def create_table(context: Context, data_dict: dict[str, Any]):
 
 
 def alter_table(context: Context, data_dict: dict[str, Any]):
-    '''Adds new columns and updates column info (stored as comments).
+    '''Add/remove columns and updates column info (stored as comments).
 
     :param resource_id: The resource ID (i.e. postgres table name)
     :type resource_id: string
@@ -1038,20 +1038,15 @@ def alter_table(context: Context, data_dict: dict[str, Any]):
     if not supplied_fields:
         supplied_fields = current_fields
     check_fields(context, supplied_fields)
-    field_ids = _pluck('id', supplied_fields)
     records = data_dict.get('records')
     new_fields: list[dict[str, Any]] = []
+    field_ids: set[str] = set(f['id'] for f in supplied_fields)
+    current_ids: set[str] = set(f['id'] for f in current_fields)
 
-    for num, field in enumerate(supplied_fields):
+    for field in supplied_fields:
         # check to see if field definition is the same or and
         # extension of current fields
-        if num < len(current_fields):
-            if field['id'] != current_fields[num]['id']:
-                raise ValidationError({
-                    'fields': [(u'Supplied field "{0}" not '
-                                u'present or in wrong order').format(
-                        field['id'])]
-                })
+        if field['id'] in current_ids:
             # no need to check type as field already defined.
             continue
 
@@ -1102,6 +1097,11 @@ def alter_table(context: Context, data_dict: dict[str, Any]):
                 identifier(f['id']),
                 info_sql))
 
+    for id_ in current_ids - field_ids - set(f['id'] for f in new_fields):
+        alter_sql.append('ALTER TABLE {0} DROP COLUMN {1};'.format(
+            identifier(data_dict['resource_id']),
+            identifier(id_)))
+
     if alter_sql:
         context['connection'].execute(
             u';'.join(alter_sql).replace(u'%', u'%%'))
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -32,7 +32,7 @@ def datastore_create(context: Context, data_dict: dict[str, Any]):
     The datastore_create action allows you to post JSON data to be
     stored against a resource. This endpoint also supports altering tables,
     aliases and indexes and bulk insertion. This endpoint can be called
-    multiple times to initially insert more data, add fields, change the
+    multiple times to initially insert more data, add/remove fields, change the
     aliases or indexes as well as the primary keys.
 
     To create an empty datastore resource and a CKAN resource at the same time,
@@ -80,7 +80,8 @@ def datastore_create(context: Context, data_dict: dict[str, Any]):
 
     Please note that setting the ``aliases``, ``indexes`` or ``primary_key``
     replaces the existing aliases or constraints. Setting ``records`` appends
-    the provided records to the resource.
+    the provided records to the resource. Setting ``fields`` without including
+    all existing fields will remove the others and the data they contain.
 
     **Results:**
 


Test patch:
diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py
--- a/ckanext/datastore/tests/test_create.py
+++ b/ckanext/datastore/tests/test_create.py
@@ -307,6 +307,30 @@ def test_calculate_record_count(self):
         last_analyze = when_was_last_analyze(resource["id"])
         assert last_analyze is not None
 
+    def test_remove_columns(self):
+        resource = factories.Resource()
+        data = {
+            "resource_id": resource["id"],
+            "fields": [
+                {"id": "col_a", "type": "text"},
+                {"id": "col_b", "type": "text"},
+                {"id": "col_c", "type": "text"},
+                {"id": "col_d", "type": "text"},
+            ],
+            "force": True,
+        }
+        helpers.call_action("datastore_create", **data)
+        data = {
+            "resource_id": resource["id"],
+            "fields": [
+                {"id": "col_a", "type": "text"},
+                {"id": "col_c", "type": "text"},
+            ],
+            "force": True,
+        }
+        helpers.call_action("datastore_create", **data)
+        info = helpers.call_action("datastore_info", id=resource["id"])
+        assert [f['id'] for f in info['fields']] == ['col_a', 'col_c']
 
 
 class TestDatastoreCreate(object):
@@ -1125,43 +1149,6 @@ def test_guess_types(self, app):
             u"nested",  # count3
         ], types
 
-        ### fields resupplied in wrong order
-
-        data = {
-            "resource_id": resource.id,
-            "fields": [
-                {"id": "author", "type": "text"},
-                {"id": "count"},
-                {"id": "date"},  # date and book in wrong order
-                {"id": "book"},
-                {"id": "count2"},
-                {"id": "extra", "type": "text"},
-                {"id": "date2"},
-            ],
-            "records": [
-                {
-                    "book": "annakarenina",
-                    "author": "tolstoy",
-                    "count": 1,
-                    "date": "2005-12-01",
-                    "count2": 2,
-                    "count3": 432,
-                    "date2": "2005-12-01",
-                }
-            ],
-        }
-
-        headers = {"Authorization": self.sysadmin_token}
-        res = app.post(
-            "/api/action/datastore_create",
-            json=data,
-            headers=headers,
-            status=409,
-        )
-        res_dict = json.loads(res.data)
-
-        assert res_dict["success"] is False
-
     @pytest.mark.ckan_config("ckan.plugins", "datastore")
     @pytest.mark.usefixtures("with_plugins")
     def test_datastore_create_with_invalid_data_value(self, app):