Repository: ckan/ckan

Issue:
lenient datastore_upsert
## CKAN version
master

## Describe the bug
passing an empty value for a date or other numeric field could store null in the field instead of raising an error

### Steps to reproduce
upsert `''` as a value to a datetime, numeric, etc. datastore field

### Expected behavior
not cause a db error

### Additional details


Discussion:

Resolving patch:
diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py
--- a/ckanext/datastore/backend/postgres.py
+++ b/ckanext/datastore/backend/postgres.py
@@ -1142,6 +1142,8 @@ def upsert_data(context: Context, data_dict: dict[str, Any]):
                 if value is not None and field['type'].lower() == 'nested':
                     # a tuple with an empty second value
                     value = (json.dumps(value), '')
+                elif value == '' and field['type'] != 'text':
+                    value = None
                 row.append(value)
             rows.append(row)
 
@@ -1187,6 +1189,8 @@ def upsert_data(context: Context, data_dict: dict[str, Any]):
                 if value is not None and field['type'].lower() == 'nested':
                     # a tuple with an empty second value
                     record[field['id']] = (json.dumps(value), '')
+                elif value == '' and field['type'] != 'text':
+                    record[field['id']] = None
 
             non_existing_field_names = [
                 field for field in record


Test patch:
diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py
--- a/ckanext/datastore/tests/test_upsert.py
+++ b/ckanext/datastore/tests/test_upsert.py
@@ -613,6 +613,38 @@ def test_id_instead_of_pk_update(self):
         assert search_result["records"][0]["book"] == "The boy"
         assert search_result["records"][0]["author"] == "F Torres"
 
+    @pytest.mark.ckan_config("ckan.plugins", "datastore")
+    @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+    def test_empty_string_instead_of_null(self):
+        resource = factories.Resource()
+        data = {
+            "resource_id": resource["id"],
+            "force": True,
+            "primary_key": "pk",
+            "fields": [
+                {"id": "pk", "type": "text"},
+                {"id": "n", "type": "int"},
+                {"id": "d", "type": "date"},
+            ],
+            "records": [{"pk": "1000", "n": "5", "d": "2020-02-02"}],
+        }
+        helpers.call_action("datastore_create", **data)
+
+        data = {
+            "resource_id": resource["id"],
+            "force": True,
+            "method": "upsert",
+            "records": [
+                {"pk": "1000", "n": "", "d": ""}
+            ],
+        }
+        helpers.call_action("datastore_upsert", **data)
+
+        search_result = _search(resource["id"])
+        assert search_result["total"] == 1
+        rec = search_result["records"][0]
+        assert rec == {'_id': 1, 'pk': '1000', 'n': None, 'd': None}
+
 
 @pytest.mark.usefixtures("with_request_context")
 class TestDatastoreInsert(object):
@@ -719,6 +751,37 @@ def test_key_already_exists(self):
             context.value
         )
 
+    @pytest.mark.ckan_config("ckan.plugins", "datastore")
+    @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+    def test_empty_string_instead_of_null(self):
+        resource = factories.Resource()
+        data = {
+            "resource_id": resource["id"],
+            "force": True,
+            "primary_key": "pk",
+            "fields": [
+                {"id": "pk", "type": "text"},
+                {"id": "n", "type": "int"},
+                {"id": "d", "type": "date"},
+            ],
+        }
+        helpers.call_action("datastore_create", **data)
+
+        data = {
+            "resource_id": resource["id"],
+            "force": True,
+            "method": "insert",
+            "records": [
+                {"pk": "1000", "n": "", "d": ""}
+            ],
+        }
+        helpers.call_action("datastore_upsert", **data)
+
+        search_result = _search(resource["id"])
+        assert search_result["total"] == 1
+        rec = search_result["records"][0]
+        assert rec == {'_id': 1, 'pk': '1000', 'n': None, 'd': None}
+
 
 @pytest.mark.usefixtures("with_request_context")
 class TestDatastoreUpdate(object):
@@ -951,3 +1014,35 @@ def test_id_instead_of_pk_update(self):
         assert search_result["records"][0]["pk"] == "1000"
         assert search_result["records"][0]["book"] == "The boy"
         assert search_result["records"][0]["author"] == "F Torres"
+
+    @pytest.mark.ckan_config("ckan.plugins", "datastore")
+    @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+    def test_empty_string_instead_of_null(self):
+        resource = factories.Resource()
+        data = {
+            "resource_id": resource["id"],
+            "force": True,
+            "primary_key": "pk",
+            "fields": [
+                {"id": "pk", "type": "text"},
+                {"id": "n", "type": "int"},
+                {"id": "d", "type": "date"},
+            ],
+            "records": [{"pk": "1000", "n": "5", "d": "2020-02-02"}],
+        }
+        helpers.call_action("datastore_create", **data)
+
+        data = {
+            "resource_id": resource["id"],
+            "force": True,
+            "method": "update",
+            "records": [
+                {"pk": "1000", "n": "", "d": ""}
+            ],
+        }
+        helpers.call_action("datastore_upsert", **data)
+
+        search_result = _search(resource["id"])
+        assert search_result["total"] == 1
+        rec = search_result["records"][0]
+        assert rec == {'_id': 1, 'pk': '1000', 'n': None, 'd': None}