Repository: OpenEnergyPlatform/oeplatform

Issue:
I get error "column id does not exist" on insert only
This seems to be intended behaviour, but it is not consistent.

* If tables need an id column, the create table request should already fail if the column does not exist
* Furthermore, the insert fails only via the API request. It is still possible to do it via the advanced api insert call.
* This is also not well documented

We need to decide if this should be the intended behaviour or not

* if not: remove the restriction
* if yes: make it consistent for all API calls


I can write the patches, but we first need to decide what the intended behaviour is supposed to be



Discussion:
## Example
```python
import os
import requests

host='openenergy-platform.org'
token = os.environ['OEP_API_TOKEN']
api_url = 'https://%s/api/v0/' % (host,)
schema = 'sandbox'
table = 'test_column_id'
api_table_url = '%s/schema/%s/tables/%s/' % (api_url, schema, table)
ses = requests.Session()
ses.headers = {'Authorization': 'Token %s' % token}

col = 'not_id' # if you change this to 'id', everything is fine
table_schema = {
    'columns': [
        {'name': col, 'data_type': 'int'}
    ]
}

# create table, no problem
res = ses.put(api_table_url, json={'query': table_schema})

# insert values with API: PROBLEM if col != 'id'
res = ses.post(api_table_url + 'rows/new', json={'query': [{col: 1}, {col: 2}]})
print(res.json()) # {'reason': 'column "id" does not exist'}

# insert values with advanced API
res = ses.post(api_url + 'advanced/connection/open')
connection_id = res.json()['content']['connection_id']

res = ses.post(api_url + 'advanced/insert', json={
    "connection_id": connection_id,
    "query": {
        "schema": schema,
        "table": table,
        "fields": [col],
        "values": [[3], [4]]
    }
})
res = ses.post(api_url + 'advanced/connection/commit', json={'connection_id': connection_id})
res = ses.post(api_url + 'advanced/connection/close', json={'connection_id': connection_id})

# select data
res = ses.get(api_table_url + 'rows')
print(res.json())  # only data from advanced api insert: [{'not_id': 3}, {'not_id': 4}]


# delete table
res = ses.delete(api_table_url)
```
discussed in dev meeting: probably enforce existance of id column (numerical, primary key)

Resolving patch:
diff --git a/api/actions.py b/api/actions.py
--- a/api/actions.py
+++ b/api/actions.py
@@ -749,12 +749,12 @@ def assert_valid_table_name(table):
                        "and start with a letter.")
 
 
-def table_create(schema, table, columns, constraints_definitions, cursor, table_metadata=None):
+def table_create(schema, table, column_definitions, constraints_definitions, cursor, table_metadata=None):
     """
     Creates a new table.
     :param schema: schema
     :param table: table
-    :param columns: Description of columns
+    :param column_definitions: Description of columns
     :param constraints: Description of constraints
     :return: Dictionary with results
     """
@@ -796,7 +796,22 @@ def table_create(schema, table, columns, constraints_definitions, cursor, table_
 
     metadata = MetaData()
 
-    columns = [get_column_definition_query(c) for c in columns]
+    primary_key_col_names = None
+    columns_by_name = {}    
+    
+    columns = []
+    for cdef in column_definitions:
+        col = get_column_definition_query(cdef)
+        columns.append(col)
+
+        # check for duplicate column names
+        if col.name in columns_by_name:
+            raise APIError("Duplicate column name: %s" % col.name)
+        columns_by_name[col.name] = col
+        if col.primary_key:
+            if primary_key_col_names:
+                raise APIError("Multiple definitions of primary key")
+            primary_key_col_names = [col.name]
 
     constraints = []
 
@@ -814,8 +829,14 @@ def table_create(schema, table, columns, constraints_definitions, cursor, table_
             if "columns" in constraint:
                 ccolumns = constraint["columns"]
             else:
-                ccolumns = [constraint["constraint_parameter"]]
-            constraints.append(sa.schema.PrimaryKeyConstraint(*ccolumns, **kwargs))
+                ccolumns = [constraint["constraint_parameter"]]                        
+            
+            if primary_key_col_names:
+                raise APIError("Multiple definitions of primary key")
+            primary_key_col_names = ccolumns
+            
+            const = sa.schema.PrimaryKeyConstraint(*ccolumns, **kwargs)            
+            constraints.append(const)
         elif constraint_type == "unique":
             kwargs = {}
             cname = constraint.get("name")
@@ -828,6 +849,28 @@ def table_create(schema, table, columns, constraints_definitions, cursor, table_
             constraints.append(sa.schema.UniqueConstraint(*ccolumns, **kwargs))
 
     assert_valid_table_name(table)
+
+    # autogenerate id column if missing
+    if 'id' not in columns_by_name:        
+        columns_by_name["id"] = sa.Column("id", sa.BigInteger, autoincrement=True)
+        columns.insert(0, columns_by_name["id"])
+    
+    # check id column type
+    id_col_type = str(columns_by_name["id"].type).upper()
+    if not "INT" in id_col_type or "SERIAL" in id_col_type:
+        raise APIError("Id column must be of int type")
+    
+    # autogenerate primary key
+    if not primary_key_col_names:
+        constraints.append(sa.schema.PrimaryKeyConstraint("id"))
+        primary_key_col_names = ["id"]
+    
+    # check pk == id
+    if tuple(primary_key_col_names) != ("id",):
+        raise APIError("Primary key must be column id")
+
+    
+
     t = Table(table, metadata, *(columns + constraints), schema=schema, comment=comment_on_table)
     t.create(_get_engine())
 
diff --git a/api/urls.py b/api/urls.py
--- a/api/urls.py
+++ b/api/urls.py
@@ -57,6 +57,7 @@
     url(
         r"^v0/advanced/insert",
         views.create_ajax_handler(actions.data_insert, requires_cursor=True),
+        name='api_insert',
     ),
     url(
         r"^v0/advanced/delete",
@@ -108,14 +109,17 @@
     url(
         r"^v0/advanced/connection/open",
         views.create_ajax_handler(actions.open_raw_connection),
+        name='api_con_open',
     ),
     url(
         r"^v0/advanced/connection/close$",
         views.create_ajax_handler(actions.close_raw_connection),
+        name='api_con_close',
     ),
     url(
         r"^v0/advanced/connection/commit",
         views.create_ajax_handler(actions.commit_raw_connection),
+        name='api_con_commit',
     ),
     url(
         r"^v0/advanced/connection/rollback",


Test patch:
diff --git a/api/tests/test_regression/test_issue_739_enforce_id_pk.py b/api/tests/test_regression/test_issue_739_enforce_id_pk.py
new file mode 100644
--- /dev/null
+++ b/api/tests/test_regression/test_issue_739_enforce_id_pk.py
@@ -0,0 +1,109 @@
+"""
+inserting the string "1000" in a varchar column 
+that has unique constraint  leads to an error.
+"""
+import json
+
+from api.tests import APITestCase
+from django.urls import reverse
+from rest_framework.test import APIClient
+
+
+class Test_issue_739_enforce_id_pk(APITestCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.cli = APIClient()
+        cls.cli.credentials(HTTP_AUTHORIZATION="Token %s" % cls.token)
+
+    def create_table(self, table_definition, table=None):
+        table = table or "test_id_pk_table"
+        table_url = reverse(
+            "api_table", kwargs={"schema": self.test_schema, "table": table}
+        )
+        return self.cli.put(table_url, {"query": table_definition}, format="json")
+
+    def test_check_insert_issue_739(self):
+        table = "test_id_data"
+
+        res = self.create_table(
+            {"columns": [{"name": "not_id", "data_type": "int"}]}, table
+        )
+        self.assertEqual(res.status_code, 201, msg=res.json())
+
+        # insert record via normal api
+        # this fails if there is no column named `id`
+        # which should be auto generated, if missing
+        url = reverse(
+            "api_rows_new", kwargs={"schema": self.test_schema, "table": table}
+        )
+        res = self.cli.post(url, {"query": [{"not_id": 99}]}, format="json")
+        self.assertEqual(res.status_code, 201)
+
+        # retrieve data
+        url = reverse("api_rows", kwargs={"schema": self.test_schema, "table": table})
+        res = json.loads(self.cli.get(url).getvalue().decode())
+        self.assertEqual(len(res), 1)
+        self.assertGreaterEqual(res[0]["id"], 0)  # auto generated id
+
+    def test_id_type_and_pk(self):
+
+        res = self.create_table(
+            {"columns": [{"name": "id", "data_type": "int", "primary_key": True}]},
+            "test_id_ok",
+        )
+        self.assertEqual(res.status_code, 201)
+
+        res = self.create_table(
+            {"columns": [{"name": "id", "data_type": "int"}]}, "test_id_auto_create_pk"
+        )
+        self.assertEqual(res.status_code, 201)
+
+        res = self.create_table(
+            {"columns": [{"name": "id", "data_type": "float"}]}, "test_id_wrong_type"
+        )
+        self.assertEqual(res.status_code, 400)
+
+        res = self.create_table(
+            {
+                "columns": [
+                    {"name": "id", "data_type": "int"},
+                    {"name": "id2", "data_type": "int", "primary_key": True},
+                ]
+            },
+            "test_id_wrong_pk2",
+        )
+        self.assertEqual(res.status_code, 400)
+
+    def test_err_on_col_duplicate(self):
+        res = self.create_table(
+            {
+                "columns": [
+                    {"name": "id", "data_type": "float"},
+                    {"name": "id", "data_type": "int"},
+                ]
+            }
+        )
+        self.assertEqual(res.status_code, 400)
+
+    def test_err_on_multiple_pk(self):
+        res = self.create_table(
+            {
+                "columns": [
+                    {"name": "id1", "data_type": "float", "primary_key": True},
+                    {"name": "id2", "data_type": "int", "primary_key": True},
+                ]
+            }
+        )
+        self.assertEqual(res.status_code, 400)
+
+        res = self.create_table(
+            {
+                "columns": [
+                    {"name": "id1", "data_type": "float", "primary_key": True},
+                    {"name": "id2", "data_type": "int"},
+                ],
+                "constraints": [{"type": "primary_key", "columns": ["id2"]}],
+            }
+        )
+        self.assertEqual(res.status_code, 400)