Repository: OpenEnergyPlatform/oeplatform

Issue:
Delete a row using the http-api leads to server error if table includes not nullable fileds
## Description of the issue

I have a table like this one:

```python
table_schema = {
    "columns": [
        # NOTE: first column should be numerical column named `id` .
        # Use `bigserial` if you want the database to create the re
        {"name": "id", "data_type": "bigserial", "primary_key": True},
        {"name": "name", "data_type": "varchar(18)", "is_nullable": False},
        {"name": "is_active", "data_type": "boolean"},
        {"name": "capacity_mw", "data_type": "float"},
        {"name": "installation_datetime_utc", "data_type": "datetime"},
    ]
}
```
If i try to delete a row by id this error is raised:
```bash
api.error.APIError: NotNullViolation('null value in column "name" of relation "_test_data_upload_delete" violates not-null 
constraint\nDETAIL:  Failing row contains (997236, null, test, 2024-01-24 14:05:19.724128, f, f, update, f, 1, null, null, null, null).\n')
```

looking at my table schema the name column cant be null but the error indicated that the last 4 parameters of the insert statements are null but at least the first one should not be null because it is the name column that is not nullable.

This is related to the versioning system we have in place that keeps track of any changes to the data and stored all delete / edited values in the so called _ schemas on the oedb. In my case the table was created in model_draft schema and the error happend in _model_draft schema.

This is the insert statement that is generated by the oep:
INSERT INTO _model_draft._test_data_upload_delete (_message, _user, _type, id) VALUES (%(_message_m0)s, %(_user_m0)s, %(_type_m0)s, %(id_m0)s)

This is the api enpoint im sending the request to:
Internal Server Error: /api/v0/schema/model_draft/tables/test_data_upload/rows/1

## Steps to Reproduce
1. Create a table using my table schema 
2. Insert data
3. Delete a row using the http-api rows/id endpoint with a http delete request

## Ideas of solution

Make sure that non-deletable fields are registered when creating the insert statement. 

Can you find the reason why all fields in the table are registered as null even though they contain data?
With the exception of the id field, which is filled in correctly, but this information is also part of the http request.

## Context and Environment
Complete stacktrace:

``` bash
Internal Server Error: /api/v0/schema/model_draft/tables/test_data_upload/rows/1
Traceback (most recent call last):
  File "/home/jh/github/oeplatform/api/actions.py", line 1384, in _execute_sqla
    cursor.execute(str(compiled), params)
psycopg2.errors.NotNullViolation: null value in column "name" of relation "_test_data_upload_delete" violates not-null constraint
DETAIL:  Failing row contains (997235, null, test, 2024-01-24 13:35:55.809135, f, f, update, f, 1, null, null, null, null).


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/django/core/handlers/exception.py", line 47, in inner
    response = get_response(request)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/django/core/handlers/base.py", line 181, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
    return view_func(*args, **kwargs)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/django/views/generic/base.py", line 70, in view
    return self.dispatch(request, *args, **kwargs)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/rest_framework/views.py", line 509, in dispatch
    response = self.handle_exception(exc)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/rest_framework/views.py", line 469, in handle_exception
    self.raise_uncaught_exception(exc)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/rest_framework/views.py", line 480, in raise_uncaught_exception
    raise exc
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/rest_framework/views.py", line 506, in dispatch
    response = handler(request, *args, **kwargs)
  File "/home/jh/github/oeplatform/api/views.py", line 192, in wrapper
    return f(caller, request, *args, **kwargs)
  File "/home/jh/github/oeplatform/api/views.py", line 827, in delete
    result = self.__delete_rows(request, schema, table, row_id)
  File "/home/jh/github/oeplatform/api/views.py", line 89, in wrapper
    result = f(*args, **kwargs)
  File "/home/jh/github/oeplatform/api/views.py", line 857, in __delete_rows
    return actions.data_delete(query, context)
  File "/home/jh/github/oeplatform/api/actions.py", line 1202, in data_delete
    result = __change_rows(request, context, target_table, setter, ["id"])
  File "/home/jh/github/oeplatform/api/actions.py", line 1182, in __change_rows
    _execute_sqla(query, cursor)
  File "/home/jh/github/oeplatform/api/actions.py", line 1386, in _execute_sqla
    raise APIError(repr(e))
api.error.APIError: NotNullViolation('null value in column "name" of relation "_test_data_upload_delete" violates not-null constraint\nDETAIL:  Failing row contains (997235, null, test, 2024-01-24 13:35:55.809135, f, f, update, f, 1, null, null, null, null).\n')
[24/Jan/2024 13:35:55] "DELETE /api/v0/schema/model_draft/tables/test_data_upload/rows/1 HTTP/1.1" 500 153457
['id', '_user', '_message', '_type']
INSERT INTO _model_draft._test_data_upload_delete (_message, _user, _type, id) VALUES (%(_message_m0)s, %(_user_m0)s, %(_type_m0)s, %(id_m0)s)
Internal Server Error: /api/v0/schema/model_draft/tables/test_data_upload/rows/1
Traceback (most recent call last):
  File "/home/jh/github/oeplatform/api/actions.py", line 1384, in _execute_sqla
    cursor.execute(str(compiled), params)
psycopg2.errors.NotNullViolation: null value in column "name" of relation "_test_data_upload_delete" violates not-null constraint
DETAIL:  Failing row contains (997236, null, test, 2024-01-24 14:05:19.724128, f, f, update, f, 1, null, null, null, null).
```


## Workflow checklist
- [ ] I am aware of the workflow in [CONTRIBUTING.md](https://github.com/OpenEnergyPlatform/oeplatform/blob/develop/CONTRIBUTING.md)

Delete a row using the http-api leads to server error if table includes not nullable fileds
## Description of the issue

I have a table like this one:

```python
table_schema = {
    "columns": [
        # NOTE: first column should be numerical column named `id` .
        # Use `bigserial` if you want the database to create the re
        {"name": "id", "data_type": "bigserial", "primary_key": True},
        {"name": "name", "data_type": "varchar(18)", "is_nullable": False},
        {"name": "is_active", "data_type": "boolean"},
        {"name": "capacity_mw", "data_type": "float"},
        {"name": "installation_datetime_utc", "data_type": "datetime"},
    ]
}
```
If i try to delete a row by id this error is raised:
```bash
api.error.APIError: NotNullViolation('null value in column "name" of relation "_test_data_upload_delete" violates not-null 
constraint\nDETAIL:  Failing row contains (997236, null, test, 2024-01-24 14:05:19.724128, f, f, update, f, 1, null, null, null, null).\n')
```

looking at my table schema the name column cant be null but the error indicated that the last 4 parameters of the insert statements are null but at least the first one should not be null because it is the name column that is not nullable.

This is related to the versioning system we have in place that keeps track of any changes to the data and stored all delete / edited values in the so called _ schemas on the oedb. In my case the table was created in model_draft schema and the error happend in _model_draft schema.

This is the insert statement that is generated by the oep:
INSERT INTO _model_draft._test_data_upload_delete (_message, _user, _type, id) VALUES (%(_message_m0)s, %(_user_m0)s, %(_type_m0)s, %(id_m0)s)

This is the api enpoint im sending the request to:
Internal Server Error: /api/v0/schema/model_draft/tables/test_data_upload/rows/1

## Steps to Reproduce
1. Create a table using my table schema 
2. Insert data
3. Delete a row using the http-api rows/id endpoint with a http delete request

## Ideas of solution

Make sure that non-deletable fields are registered when creating the insert statement. 

Can you find the reason why all fields in the table are registered as null even though they contain data?
With the exception of the id field, which is filled in correctly, but this information is also part of the http request.

## Context and Environment
Complete stacktrace:

``` bash
Internal Server Error: /api/v0/schema/model_draft/tables/test_data_upload/rows/1
Traceback (most recent call last):
  File "/home/jh/github/oeplatform/api/actions.py", line 1384, in _execute_sqla
    cursor.execute(str(compiled), params)
psycopg2.errors.NotNullViolation: null value in column "name" of relation "_test_data_upload_delete" violates not-null constraint
DETAIL:  Failing row contains (997235, null, test, 2024-01-24 13:35:55.809135, f, f, update, f, 1, null, null, null, null).


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/django/core/handlers/exception.py", line 47, in inner
    response = get_response(request)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/django/core/handlers/base.py", line 181, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
    return view_func(*args, **kwargs)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/django/views/generic/base.py", line 70, in view
    return self.dispatch(request, *args, **kwargs)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/rest_framework/views.py", line 509, in dispatch
    response = self.handle_exception(exc)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/rest_framework/views.py", line 469, in handle_exception
    self.raise_uncaught_exception(exc)
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/rest_framework/views.py", line 480, in raise_uncaught_exception
    raise exc
  File "/home/jh/github/oeplatform/env/lib/python3.10/site-packages/rest_framework/views.py", line 506, in dispatch
    response = handler(request, *args, **kwargs)
  File "/home/jh/github/oeplatform/api/views.py", line 192, in wrapper
    return f(caller, request, *args, **kwargs)
  File "/home/jh/github/oeplatform/api/views.py", line 827, in delete
    result = self.__delete_rows(request, schema, table, row_id)
  File "/home/jh/github/oeplatform/api/views.py", line 89, in wrapper
    result = f(*args, **kwargs)
  File "/home/jh/github/oeplatform/api/views.py", line 857, in __delete_rows
    return actions.data_delete(query, context)
  File "/home/jh/github/oeplatform/api/actions.py", line 1202, in data_delete
    result = __change_rows(request, context, target_table, setter, ["id"])
  File "/home/jh/github/oeplatform/api/actions.py", line 1182, in __change_rows
    _execute_sqla(query, cursor)
  File "/home/jh/github/oeplatform/api/actions.py", line 1386, in _execute_sqla
    raise APIError(repr(e))
api.error.APIError: NotNullViolation('null value in column "name" of relation "_test_data_upload_delete" violates not-null constraint\nDETAIL:  Failing row contains (997235, null, test, 2024-01-24 13:35:55.809135, f, f, update, f, 1, null, null, null, null).\n')
[24/Jan/2024 13:35:55] "DELETE /api/v0/schema/model_draft/tables/test_data_upload/rows/1 HTTP/1.1" 500 153457
['id', '_user', '_message', '_type']
INSERT INTO _model_draft._test_data_upload_delete (_message, _user, _type, id) VALUES (%(_message_m0)s, %(_user_m0)s, %(_type_m0)s, %(id_m0)s)
Internal Server Error: /api/v0/schema/model_draft/tables/test_data_upload/rows/1
Traceback (most recent call last):
  File "/home/jh/github/oeplatform/api/actions.py", line 1384, in _execute_sqla
    cursor.execute(str(compiled), params)
psycopg2.errors.NotNullViolation: null value in column "name" of relation "_test_data_upload_delete" violates not-null constraint
DETAIL:  Failing row contains (997236, null, test, 2024-01-24 14:05:19.724128, f, f, update, f, 1, null, null, null, null).
```


## Workflow checklist
- [ ] I am aware of the workflow in [CONTRIBUTING.md](https://github.com/OpenEnergyPlatform/oeplatform/blob/develop/CONTRIBUTING.md)



Discussion:

Resolving patch:
diff --git a/api/actions.py b/api/actions.py
--- a/api/actions.py
+++ b/api/actions.py
@@ -32,7 +32,8 @@
     load_cursor_from_context,
     load_session_from_context,
 )
-from dataedit.models import Schema as DBSchema, PeerReview
+from dataedit.models import PeerReview
+from dataedit.models import Schema as DBSchema
 from dataedit.models import Table as DBTable
 from dataedit.structures import TableTags as OEDBTableTags
 from dataedit.structures import Tag as OEDBTag
@@ -47,6 +48,8 @@
 __UPDATE = 1
 __DELETE = 2
 
+ID_COLUMN_NAME = "id"
+
 
 MAX_IDENTIFIER_LENGTH = 50  # postgres limit minus pre/suffix for meta tables
 IDENTIFIER_PATTERN = re.compile("^[a-z][a-z0-9_]{0,%s}$" % (MAX_IDENTIFIER_LENGTH - 1))
@@ -1183,6 +1186,38 @@ def __change_rows(request, context, target_table, setter, fields=None):
     return {"rowcount": rows["rowcount"]}
 
 
+def drop_not_null_constraints_from_delete_meta_table(
+    meta_table_delete: str, meta_schema: str
+) -> None:
+    # https://github.com/OpenEnergyPlatform/oeplatform/issues/1548
+    # we only want id column (and meta colums, wich start with a "_")
+
+    engine = _get_engine()
+
+    # find not nullable columns in meta tables
+    query = f"""
+    SELECT column_name
+    FROM information_schema.columns
+    WHERE table_name = '{meta_table_delete}'
+    AND table_schema = '{meta_schema}'
+    AND is_nullable = 'NO'
+    """
+    column_names = [x[0] for x in engine.execute(query).fetchall()]
+    # filter meta columns and id
+    column_names = [
+        c for c in column_names if c != ID_COLUMN_NAME and not c.startswith("_")
+    ]
+
+    if not column_names:
+        # nothing to do
+        return
+
+    # drop not null from these columns
+    col_drop = ", ".join(f'ALTER "{c}" DROP NOT NULL' for c in column_names)
+    query = f'ALTER TABLE "{meta_schema}"."{meta_table_delete}" {col_drop};'
+    engine.execute(query)
+
+
 def data_delete(request, context=None):
     orig_table = get_or_403(request, "table")
     if orig_table.startswith("_") or orig_table.endswith("_cor"):
@@ -1199,6 +1234,10 @@ def data_delete(request, context=None):
     target_table = get_delete_table_name(orig_schema, orig_table)
     setter = []
     cursor = load_cursor_from_context(context)
+
+    meta_schema = get_meta_schema_name(schema)
+    drop_not_null_constraints_from_delete_meta_table(target_table, meta_schema)
+
     result = __change_rows(request, context, target_table, setter, ["id"])
     if orig_schema in PLAYGROUNDS + UNVERSIONED_SCHEMAS:
         apply_changes(schema, table, cursor)
@@ -2085,6 +2124,15 @@ def getValue(schema, table, column, id):
 
 
 def apply_changes(schema, table, cursor=None):
+    """Apply changes from the meta tables to the actual table.
+
+    Meta tables are :
+    * _<NAME>_insert
+    * _<NAME>_update
+    * _<NAME>_delete
+
+    """
+
     def add_type(d, type):
         d["_type"] = type
         return d


Test patch:
diff --git a/api/tests/test_regression/test_issue_1548_delete_not_null.py b/api/tests/test_regression/test_issue_1548_delete_not_null.py
new file mode 100644
--- /dev/null
+++ b/api/tests/test_regression/test_issue_1548_delete_not_null.py
@@ -0,0 +1,25 @@
+from api.tests import APITestCaseWithTable
+
+
+class Test1548(APITestCaseWithTable):
+    """Delete a row using the http-api leads to server error
+    if table includes not nullable fields"""
+
+    test_structure = {
+        "columns": [
+            {"name": "name", "data_type": "varchar(18)", "is_nullable": False},
+            {"name": "value", "data_type": "int", "is_nullable": False},
+        ]
+    }
+    test_data = [{"name": "name1", "value": 10}, {"name": "name2", "value": 20}]
+
+    def test1548(self):
+        self.assertEqual(len(self.api_req("GET", path="rows/")), 2)
+        # id=1 was generated automatically
+        res = self.api_req("GET", path="rows/1")
+        self.assertEqual(res["id"], 1)
+        # now we delete it (This created the error)
+        self.api_req("DELETE", path="rows/1")
+        # check that delete worked
+        self.api_req("GET", path="rows/1", exp_code=404)
+        self.assertEqual(len(self.api_req("GET", path="rows/")), 1)