Repository: mathesar-foundation/mathesar

Issue:
Unknown server_default settings silently deleted
## Description
<!-- A clear and concise description of what the bug is. -->

When the column-type-altering function doesn't understand a `server_default` value for a column (the DB-level default setting), it silently removes / deletes that default when altering the column type.  This has the potential to lose metadata about a column that we'd be unable to recover.  Further, there would be no warning to the user in this case.

## Expected behavior
<!-- A clear and concise description of what you expected to happen. -->

If we try to alter a column's type, and cannot deal with the `server_default` setting, we should refuse, or at least throw a warning where the user can see it.

## To Reproduce
<!-- How can we recreate this bug? Please try to provide a Minimal, Complete, and Verifiable (http://stackoverflow.com/help/mcve) example if code-related. -->

Change the type of any column with a dynamic default (e.g., change a `DATE` column with default `NOW()` to `VARCHAR`), and note that the default has disappeared on the DB (this will be invisible in the API as noted in the description above).

## Environment
 - OS: (_eg._ macOS 10.14.6; Fedora 32)
 - Browser: (_eg._ Safari; Firefox)
 - Browser Version: (_eg._ 13; 73)
 - Other info:

## Additional context
<!-- Add any other context about the problem or screenshots here. -->



Discussion:

Resolving patch:
diff --git a/db/columns/exceptions.py b/db/columns/exceptions.py
--- a/db/columns/exceptions.py
+++ b/db/columns/exceptions.py
@@ -12,3 +12,7 @@ class InvalidTypeOptionError(Exception):
 
 class DagCycleError(Exception):
     pass
+
+
+class DynamicDefaultWarning(Warning):
+    pass
diff --git a/db/columns/operations/select.py b/db/columns/operations/select.py
--- a/db/columns/operations/select.py
+++ b/db/columns/operations/select.py
@@ -1,10 +1,21 @@
 import warnings
 
+from pglast import Node, parse_sql
 from sqlalchemy import Table, MetaData, and_, select, text, func
 
+from db.columns.exceptions import DynamicDefaultWarning
 from db.tables.operations.select import reflect_table_from_oid
 from db.utils import execute_statement
 
+# These tags define which nodes in the AST built by pglast we consider to be
+# "dynamic" when found in a column default clause.  The nodes are best
+# documented by C header files that define the underlying structs:
+# https://github.com/pganalyze/libpg_query/blob/13-latest/src/postgres/include/nodes/parsenodes.h
+# https://github.com/pganalyze/libpg_query/blob/13-latest/src/postgres/include/nodes/primnodes.h
+# It's possible that more dynamic nodes will be found.  Their tags should be
+# added to this set.
+DYNAMIC_NODE_TAGS = {"SQLValueFunction", "FuncCall"}
+
 
 def get_column_index_from_name(table_oid, column_name, engine, connection_to_use=None):
     with warnings.catch_warnings():
@@ -32,47 +43,52 @@ def get_column_index_from_name(table_oid, column_name, engine, connection_to_use
     return result - 1 - dropped_count
 
 
-def get_column_default(table_oid, column_index, engine, connection_to_use=None):
+def get_column_default_dict(table_oid, column_index, engine, connection_to_use=None):
     table = reflect_table_from_oid(table_oid, engine, connection_to_use)
     column = table.columns[column_index]
-    if column.server_default is None:
-        return None
+    default_dict = None
+    if column.server_default is not None:
+        default_dict = {
+            "server_default": column.server_default,
+            "is_dynamic": _is_default_expr_dynamic(column.server_default),
+            "sql_text": str(column.server_default.arg)
+        }
+    else:
+        return
+    if default_dict.get("is_dynamic"):
+        warnings.warn(
+            "Dynamic column defaults are read only", DynamicDefaultWarning
+        )
+    else:
+        # Defaults are stored as text with SQL casts appended
+        # Ex: "'test default string'::character varying" or "'2020-01-01'::date"
+        # Here, we execute the cast to get the proper python value
+        default_dict.update(
+            executed_constant=execute_statement(
+                engine,
+                select(text(default_dict['sql_text'])),
+                connection_to_use
+            ).first()[0]
+        )
+    return default_dict
 
-    metadata = MetaData()
-    with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", message="Did not recognize type")
-        pg_attribute = Table("pg_attribute", metadata, autoload_with=engine)
-        pg_attrdef = Table("pg_attrdef", metadata, autoload_with=engine)
 
-    query = (
-        select(pg_attrdef.c.adbin)
-        .select_from(
-            pg_attrdef
-            .join(
-                pg_attribute,
-                and_(
-                    pg_attribute.c.attnum == pg_attrdef.c.adnum,
-                    pg_attribute.c.attrelid == pg_attrdef.c.adrelid
-                )
-            )
-        )
-        .where(and_(
-            pg_attribute.c.attrelid == table_oid,
-            pg_attribute.c.attname == column.name,
-            pg_attribute.c.attnum >= 1,
-        ))
+def get_column_default(table_oid, column_index, engine, connection_to_use=None):
+    default_dict = get_column_default_dict(
+        table_oid, column_index, engine, connection_to_use=connection_to_use
     )
+    if default_dict is None:
+        return
+    else:
+        executed_constant = default_dict.get('executed_constant')
 
-    result = execute_statement(engine, query, connection_to_use).first()[0]
+    return executed_constant if executed_constant is not None else default_dict['sql_text']
 
-    # Here, we get the 'adbin' value for the current column, stored in the attrdef
-    # system table. The prefix of this value tells us whether the default is static
-    # ('{CONSTANT') or generated ('{FUNCEXPR'). We do not return generated defaults.
-    if result.startswith("{FUNCEXPR"):
-        return None
 
-    default_textual_sql = column.server_default.arg.text
-    # Defaults are stored as text with SQL casts appended
-    # Ex: "'test default string'::character varying" or "'2020-01-01'::date"
-    # Here, we execute the cast to get the proper python value
-    return execute_statement(engine, select(text(default_textual_sql)), connection_to_use).first()[0]
+def _is_default_expr_dynamic(server_default):
+    prepared_expr = f"""SELECT {server_default.arg.text};"""
+    expr_ast_root = Node(parse_sql(prepared_expr))
+    ast_nodes = {
+        n.node_tag for n in expr_ast_root.traverse() if isinstance(n, Node)
+    }
+    return not ast_nodes.isdisjoint(DYNAMIC_NODE_TAGS)
diff --git a/mathesar/api/viewsets/columns.py b/mathesar/api/viewsets/columns.py
--- a/mathesar/api/viewsets/columns.py
+++ b/mathesar/api/viewsets/columns.py
@@ -1,10 +1,13 @@
+import warnings
 from psycopg2.errors import DuplicateColumn, UndefinedFunction
 from rest_framework import status, viewsets
 from rest_framework.exceptions import NotFound, ValidationError, APIException
 from rest_framework.response import Response
 from sqlalchemy.exc import ProgrammingError
 
-from db.columns.exceptions import InvalidDefaultError, InvalidTypeOptionError, InvalidTypeError
+from db.columns.exceptions import (
+    DynamicDefaultWarning, InvalidDefaultError, InvalidTypeOptionError, InvalidTypeError
+)
 from mathesar.api.pagination import ColumnLimitOffsetPagination
 from mathesar.api.serializers.columns import ColumnSerializer
 from mathesar.api.utils import get_table_or_404
@@ -81,32 +84,40 @@ def partial_update(self, request, pk=None, table_pk=None):
         table = get_table_or_404(table_pk)
         serializer = ColumnSerializer(data=request.data, partial=True)
         serializer.is_valid(raise_exception=True)
-        try:
-            column = table.alter_column(pk, serializer.validated_data)
-        except ProgrammingError as e:
-            if type(e.orig) == UndefinedFunction:
-                raise ValidationError('This type cast is not implemented')
-            else:
-                raise ValidationError
-        except IndexError:
-            raise NotFound
-        except TypeError:
-            raise ValidationError("Unknown type_option passed")
-        except InvalidDefaultError:
-            raise ValidationError(
-                f'default "{request.data["default"]}" is'
-                f' invalid for this column'
-            )
-        except InvalidTypeOptionError:
-            type_options = request.data.get('type_options', '')
-            raise ValidationError(
-                f'parameter dict {type_options} is'
-                f' invalid for type {request.data["type"]}'
-            )
-        except InvalidTypeError:
-            raise ValidationError('This type casting is invalid.')
-        except Exception as e:
-            raise APIException(e)
+        with warnings.catch_warnings():
+            warnings.filterwarnings("error", category=DynamicDefaultWarning)
+            try:
+                column = table.alter_column(pk, serializer.validated_data)
+            except ProgrammingError as e:
+                if type(e.orig) == UndefinedFunction:
+                    raise ValidationError('This type cast is not implemented')
+                else:
+                    raise ValidationError
+            except IndexError:
+                raise NotFound
+            except TypeError:
+                raise ValidationError("Unknown type_option passed")
+            except InvalidDefaultError:
+                raise ValidationError(
+                    f'default "{request.data["default"]}" is'
+                    f' invalid for this column'
+                )
+            except DynamicDefaultWarning:
+                raise ValidationError(
+                    'Changing type of columns with dynamically-generated'
+                    ' defaults is not supported.'
+                    ' Delete or change the default first.'
+                )
+            except InvalidTypeOptionError:
+                type_options = request.data.get('type_options', '')
+                raise ValidationError(
+                    f'parameter dict {type_options} is'
+                    f' invalid for type {request.data["type"]}'
+                )
+            except InvalidTypeError:
+                raise ValidationError('This type casting is invalid.')
+            except Exception as e:
+                raise APIException(e)
         out_serializer = ColumnSerializer(column)
         return Response(out_serializer.data)
 


Test patch:
diff --git a/db/tests/columns/operations/test_select.py b/db/tests/columns/operations/test_select.py
--- a/db/tests/columns/operations/test_select.py
+++ b/db/tests/columns/operations/test_select.py
@@ -1,9 +1,15 @@
+import warnings
 from alembic.migration import MigrationContext
 from alembic.operations import Operations
 import pytest
-from sqlalchemy import String, Integer, Column, Table, MetaData, Sequence, DateTime, func
-
-from db.columns.operations.select import get_column_default, get_column_index_from_name
+from sqlalchemy import (
+    String, Integer, Column, Table, MetaData, DateTime, func, text, DefaultClause
+)
+
+from db.columns.exceptions import DynamicDefaultWarning
+from db.columns.operations.select import (
+    get_column_default, get_column_index_from_name, _is_default_expr_dynamic
+)
 from db.tables.operations.select import get_oid_from_table
 from db.tests.columns.utils import column_test_dict, get_default
 
@@ -111,8 +117,8 @@ def test_get_column_default(engine_with_schema, filler, col_type):
 
 get_column_generated_default_test_list = [
     Column("generated_default_col", Integer, primary_key=True),
-    Column("generated_default_col", Integer, Sequence("test_id")),
-    Column("generated_default_col", DateTime, server_default=func.now())
+    Column("generated_default_col", DateTime, server_default=func.now()),
+    Column("generated_default_col", DateTime, server_default=func.current_timestamp())
 ]
 
 
@@ -127,9 +133,27 @@ def test_get_column_generated_default(engine_with_schema, col):
     )
     table.create()
     table_oid = get_oid_from_table(table_name, schema, engine)
-    default = get_column_default(table_oid, 0, engine)
-    created_default = get_default(engine, table)
+    with warnings.catch_warnings(), pytest.raises(DynamicDefaultWarning):
+        warnings.filterwarnings("error", category=DynamicDefaultWarning)
+        get_column_default(table_oid, 0, engine)
+
+
+default_expression_test_list = [
+    ("CURRENT_TIMESTAMP", True),
+    ("CURRENT_TIMESTAMP::CHAR(64)", True),
+    ("NOW()", True),
+    ("myfunc()", True),
+    ("now()", True),
+    ("now()::VARCHAR", True),
+    ("'now()'::VARCHAR", False),
+    ("'3'::NUMERIC", False),
+    ("'3'::CHAR", False),
+    ("'abcde'::CHAR(3)", False),
+    ("'abcde'::CHAR(5)", False),
+]
+
 
-    # We shouldn't evaluate generated defaults
-    assert default is None
-    assert default != created_default
+@pytest.mark.parametrize("default_expr,is_dynamic", default_expression_test_list)
+def test_is_default_expr_dynamic(default_expr, is_dynamic):
+    default_clause = DefaultClause(text(default_expr))
+    assert _is_default_expr_dynamic(default_clause) is is_dynamic
diff --git a/db/tests/columns/utils.py b/db/tests/columns/utils.py
--- a/db/tests/columns/utils.py
+++ b/db/tests/columns/utils.py
@@ -1,13 +1,16 @@
 from datetime import date
 
-from sqlalchemy import String, Integer, Boolean, Date, select, Table, MetaData
+from sqlalchemy import (
+    CHAR, String, Integer, Boolean, Date, select, Table, MetaData
+)
 
 
 column_test_dict = {
     Integer: {"start": "0", "set": "5", "expt": 5},
     String: {"start": "default", "set": "test", "expt": "test"},
     Boolean: {"start": "false", "set": "true", "expt": True},
-    Date: {"start": "2019-01-01", "set": "2020-01-01", "expt": date(2020, 1, 1)}
+    Date: {"start": "2019-01-01", "set": "2020-01-01", "expt": date(2020, 1, 1)},
+    CHAR: {"start": "a", "set": "b", "expt": "b"}
 }
 
 
diff --git a/db/tests/types/operations/test_cast.py b/db/tests/types/operations/test_cast.py
--- a/db/tests/types/operations/test_cast.py
+++ b/db/tests/types/operations/test_cast.py
@@ -770,13 +770,9 @@ def test_alter_column_casts_data_gen(
     table_oid = get_oid_from_table(TABLE_NAME, schema, engine)
     actual_default = get_column_default(table_oid, 0, engine)
     # TODO This needs to be sorted out by fixing how server_default is set.
-    # Also, we cannot support (currently) defaults for CHAR type columns, since
-    # these are always function expressions
     if all([
             source_type != get_qualified_name(MathesarCustomType.MONEY.value),
             target_type != MathesarCustomType.MONEY.value,
-            source_type != PostgresType.CHARACTER.value,
-            target_type != "char",
     ]):
         assert actual_default == out_val
 
diff --git a/mathesar/tests/api/test_column_api.py b/mathesar/tests/api/test_column_api.py
--- a/mathesar/tests/api/test_column_api.py
+++ b/mathesar/tests/api/test_column_api.py
@@ -51,7 +51,7 @@ def test_column_list(column_test_table, client):
             'index': 0,
             'nullable': False,
             'primary_key': True,
-            'default': None,
+            'default': """nextval('"Patents".anewtable_mycolumn0_seq'::regclass)""",
             'valid_target_types': [
                 'BIGINT', 'BOOLEAN', 'CHAR', 'DECIMAL', 'DOUBLE PRECISION',
                 'FLOAT', 'INTEGER', 'MATHESAR_TYPES.MONEY', 'NUMERIC', 'REAL',
@@ -117,7 +117,7 @@ def test_column_list(column_test_table, client):
                 'index': 0,
                 'nullable': False,
                 'primary_key': True,
-                'default': None,
+                'default': """nextval('"Patents".anewtable_mycolumn0_seq'::regclass)""",
                 'valid_target_types': [
                     'BIGINT', 'BOOLEAN', 'CHAR', 'DECIMAL', 'DOUBLE PRECISION',
                     'FLOAT', 'INTEGER', 'MATHESAR_TYPES.MONEY', 'NUMERIC',
@@ -352,6 +352,16 @@ def test_column_update_default_invalid_cast(column_test_table, client):
     assert response.status_code == 400
 
 
+def test_column_update_type_dynamic_default(column_test_table, client):
+    cache.clear()
+    type_ = "NUMERIC"
+    data = {"type": type_}
+    response = client.patch(
+        f"/api/v0/tables/{column_test_table.id}/columns/0/", data=data
+    )
+    assert response.status_code == 400
+
+
 def test_column_update_type(column_test_table, client):
     cache.clear()
     type_ = "BOOLEAN"