Repository: mathesar-foundation/mathesar

Issue:
Controlled Casting
**Problem**
<!-- Please provide a clear and concise description of the problem that this feature request is designed to solve.-->
In the course of implementing #92 , I've realized that the default casting in PostgreSQL doesn't quite serve our needs.  For example, many different strings cast to booleans, but this loses information.  Another example:  Numeric 1s and 0s _can't_ cast to boolean.  When casting `NUMERIC` to `INTEGER` values are silently rounded.  And so on.

**Proposed solution**
<!-- A clear and concise description of your proposed solution or feature. -->

We need to define functions for all supported casting to be used in altering column types and column inference.

**Additional context**
<!-- Add any other context or screenshots about the feature request here.-->
This will probably involve a number of PRs and sub-issues.



Discussion:

Resolving patch:
diff --git a/db/types/alteration.py b/db/types/alteration.py
new file mode 100644
--- /dev/null
+++ b/db/types/alteration.py
@@ -0,0 +1,273 @@
+from sqlalchemy import text, DDL, MetaData, Table
+from db.types import base, email
+
+BOOLEAN = "boolean"
+EMAIL = email.QUALIFIED_EMAIL
+INTERVAL = "interval"
+NUMERIC = "numeric"
+TEXT = "text"
+VARCHAR = "varchar"
+
+
+def get_supported_alter_column_types(engine):
+    dialect_types = engine.dialect.ischema_names
+    type_map = {
+        # Default Postgres types
+        "boolean": dialect_types.get("boolean"),
+        "interval": dialect_types.get("interval"),
+        "numeric": dialect_types.get("numeric"),
+        "string": dialect_types.get("name"),
+        # Custom Mathesar types
+        "email": dialect_types.get(email.QUALIFIED_EMAIL)
+    }
+    return {k: v for k, v in type_map.items() if v is not None}
+
+
+def alter_column_type(
+        schema, table_name, column_name, target_type_str, engine
+):
+    _preparer = engine.dialect.identifier_preparer
+    supported_types = get_supported_alter_column_types(engine)
+    target_type = supported_types.get(target_type_str.lower())
+    with engine.begin() as conn:
+        metadata = MetaData(bind=engine, schema=schema)
+        table = Table(
+            table_name, metadata, schema=schema, autoload_with=engine
+        )
+        column = table.columns[column_name]
+        prepared_table_name = _preparer.format_table(table)
+        prepared_column_name = _preparer.format_column(column)
+        prepared_type_name = target_type().compile(dialect=engine.dialect)
+        cast_function_name = get_cast_function_name(prepared_type_name)
+        alter_stmt = f"""
+        ALTER TABLE {prepared_table_name}
+          ALTER COLUMN {prepared_column_name}
+          TYPE {prepared_type_name}
+          USING {cast_function_name}({prepared_column_name});
+        """
+        conn.execute(DDL(alter_stmt))
+
+
+def install_all_casts(engine):
+    create_boolean_casts(engine)
+    create_email_casts(engine)
+    create_interval_casts(engine)
+    create_numeric_casts(engine)
+    create_varchar_casts(engine)
+
+
+def create_boolean_casts(engine):
+    """
+    Using the `create_cast_functions` python function, we create a
+    PL/pgSQL function to cast types corresponding to each of:
+
+    boolean -> boolean:  Identity. No remarks
+    text -> boolean:     We only cast 't', 'f', 'true', or 'false' all
+                         others raise a custom exception.
+    numeric -> boolean:  We only cast 1 -> true, 0 -> false (this is not
+                         default behavior for PostgreSQL). Others raise a
+                         custom exception.
+    """
+    not_bool_exception_str = f"RAISE EXCEPTION '% is not a {BOOLEAN}', $1;"
+    type_body_map = {
+        BOOLEAN: """
+        BEGIN
+          RETURN $1;
+        END;
+        """,
+        TEXT: f"""
+        DECLARE
+        istrue {BOOLEAN};
+        BEGIN
+          SELECT lower($1)='t' OR lower($1)='true' INTO istrue;
+          IF istrue OR lower($1)='f' OR lower($1)='false' THEN
+            RETURN istrue;
+          END IF;
+          {not_bool_exception_str}
+        END;
+        """,
+        NUMERIC: f"""
+        BEGIN
+          IF $1<>0 AND $1<>1 THEN
+            {not_bool_exception_str}
+          END IF;
+          RETURN $1<>0;
+        END;
+        """,
+    }
+    create_cast_functions(BOOLEAN, type_body_map, engine)
+
+
+def create_numeric_casts(engine):
+    """
+    Using the `create_cast_functions` python function, we create a
+    PL/pgSQL function to cast types corresponding to each of:
+
+    numeric -> numeric:  Identity. No remarks
+    boolean -> numeric:  We cast TRUE -> 1, FALSE -> 0
+    text -> numeric:     We use the default PostgreSQL behavior.
+    """
+    type_body_map = {
+        NUMERIC: """
+        BEGIN
+          RETURN $1;
+        END;
+        """,
+        BOOLEAN: f"""
+        BEGIN
+          IF $1 THEN
+            RETURN 1::{NUMERIC};
+          END IF;
+          RETURN 0;
+        END;
+        """,
+        TEXT: f"""
+        BEGIN
+          RETURN $1::{NUMERIC};
+        END;
+        """,
+    }
+    create_cast_functions(NUMERIC, type_body_map, engine)
+
+
+def create_email_casts(engine):
+    """
+    Using the `create_cast_functions` python function, we create a
+    PL/pgSQL function to cast types corresponding to each of:
+
+    email -> email:  Identity. No remarks
+    text -> email:   We use the default PostgreSQL behavior (this will
+                     just check that the TEXT object satisfies the email
+                     DOMAIN).
+    """
+    type_body_map = {
+        EMAIL: """
+        BEGIN
+          RETURN $1;
+        END;
+        """,
+        TEXT: f"""
+        BEGIN
+          RETURN $1::{EMAIL};
+        END;
+        """,
+    }
+    create_cast_functions(EMAIL, type_body_map, engine)
+
+
+def create_interval_casts(engine):
+    """
+    Using the `create_cast_functions` python function, we create a
+    PL/pgSQL function to cast types corresponding to each of:
+
+    interval -> interval:  Identity. No remarks
+    text -> interval:      We first check that the text *cannot* be cast
+                           to a numeric, and then try to cast the text to an
+                           interval.
+    """
+    type_body_map = {
+        INTERVAL: """
+        BEGIN
+          RETURN $1;
+        END;
+        """,
+        # We need to check that a string isn't a valid number before
+        # casting to intervals (since a number is more likely)
+        TEXT: f"""
+        BEGIN
+          PERFORM $1::{NUMERIC};
+          RAISE EXCEPTION '% is a {NUMERIC}', $1;
+          EXCEPTION
+            WHEN sqlstate '22P02' THEN
+              RETURN $1::{INTERVAL};
+        END;
+        """,
+    }
+    create_cast_functions(INTERVAL, type_body_map, engine)
+
+
+def create_varchar_casts(engine):
+    """
+    Using the `create_cast_functions` python function, we create a
+    PL/pgSQL function to cast types corresponding to each of:
+
+    varchar -> varchar:   Identity. No remarks
+    boolean -> varchar:   We use the default PostgreSQL cast behavior.
+    email -> varchar:     We use the default PostgreSQL cast behavior.
+    interval -> varchar:  We use the default PostgreSQL cast behavior.
+    numeric -> varchar:   We use the default PostgreSQL cast behavior.
+    text -> varchar:      We use the default PostgreSQL cast behavior.
+    """
+    type_body_map = {
+        VARCHAR: """
+        BEGIN
+          RETURN $1;
+        END;
+        """,
+        BOOLEAN: f"""
+        BEGIN
+          RETURN $1::{VARCHAR};
+        END;
+        """,
+        EMAIL: f"""
+        BEGIN
+          RETURN $1::{VARCHAR};
+        END;
+        """,
+        INTERVAL: f"""
+        BEGIN
+          RETURN $1::{VARCHAR};
+        END;
+        """,
+        NUMERIC: f"""
+        BEGIN
+          RETURN $1::{VARCHAR};
+        END;
+        """,
+        TEXT: f"""
+        BEGIN
+          RETURN $1::{VARCHAR};
+        END;
+        """,
+    }
+    create_cast_functions(VARCHAR, type_body_map, engine)
+
+
+def create_cast_functions(target_type, type_body_map, engine):
+    """
+    This python function writes a number of PL/pgSQL functions that cast
+    between types supported by Mathesar, and installs them on the DB
+    using the given engine.  Each generated PL/pgSQL function has the
+    name `cast_to_<target_type>`.  We utilize the function overloading of
+    PL/pgSQL to use the correct function body corresponding to a given
+    input (source) type.
+
+    Args:
+        target_type:   string corresponding to the target type of the
+                       cast function.
+        type_body_map: dictionary that gives a map between source types
+                       and the body of a PL/pgSQL function to cast a
+                       given source type to the target type.
+        engine:        an SQLAlchemy engine.
+    """
+    for type_, body in type_body_map.items():
+        query = assemble_function_creation_sql(type_, target_type, body)
+        with engine.begin() as conn:
+            conn.execute(text(query))
+
+
+def assemble_function_creation_sql(argument_type, target_type, function_body):
+    function_name = get_cast_function_name(target_type)
+    return f"""
+    CREATE OR REPLACE FUNCTION {function_name}({argument_type})
+    RETURNS {target_type}
+    AS $$
+    {function_body}
+    $$ LANGUAGE plpgsql;
+    """
+
+
+def get_cast_function_name(target_type):
+    unqualified_type_name = target_type.split('.')[-1].lower()
+    bare_function_name = f"cast_to_{unqualified_type_name}"
+    return f"{base.get_qualified_name(bare_function_name)}"
diff --git a/db/types/base.py b/db/types/base.py
--- a/db/types/base.py
+++ b/db/types/base.py
@@ -1,6 +1,5 @@
-from sqlalchemy import create_engine, MetaData, Table, DDL
+from sqlalchemy import create_engine
 from db import constants
-from db.types import email
 
 SCHEMA = f"{constants.MATHESAR_PREFIX}types"
 # Since we want to have our identifiers quoted appropriately for use in
@@ -10,41 +9,3 @@
 
 def get_qualified_name(name):
     return ".".join([preparer.quote_schema(SCHEMA), name])
-
-
-def get_supported_alter_column_types(engine):
-    dialect_types = engine.dialect.ischema_names
-    type_map = {
-        # Default Postgres types
-        "boolean": dialect_types.get("boolean"),
-        "interval": dialect_types.get("interval"),
-        "numeric": dialect_types.get("numeric"),
-        "string": dialect_types.get("name"),
-        # Custom Mathesar types
-        "email": dialect_types.get(email.QUALIFIED_EMAIL)
-    }
-    return {k: v for k, v in type_map.items() if v is not None}
-
-
-def alter_column_type(
-        schema, table_name, column_name, target_type_str, engine
-):
-    _preparer = engine.dialect.identifier_preparer
-    supported_types = get_supported_alter_column_types(engine)
-    target_type = supported_types.get(target_type_str.lower())
-    with engine.begin() as conn:
-        metadata = MetaData(bind=engine, schema=schema)
-        table = Table(
-            table_name, metadata, schema=schema, autoload_with=engine
-        )
-        column = table.columns[column_name]
-        prepared_table_name = _preparer.format_table(table)
-        prepared_column_name = _preparer.format_column(column)
-        prepared_type_name = target_type().compile(dialect=engine.dialect)
-        alter_stmt = f"""
-        ALTER TABLE {prepared_table_name}
-          ALTER COLUMN {prepared_column_name}
-          TYPE {prepared_type_name}
-          USING {prepared_column_name}::{prepared_type_name};
-        """
-        conn.execute(DDL(alter_stmt))
diff --git a/db/types/install.py b/db/types/install.py
--- a/db/types/install.py
+++ b/db/types/install.py
@@ -1,10 +1,11 @@
-from db.types import base, email
+from db.types import base, alteration, email
 from db.schemas import create_schema
 
 
 def install_mathesar_on_database(engine):
     create_type_schema(engine)
     email.create_email_type(engine)
+    alteration.install_all_casts(engine)
 
 
 def create_type_schema(engine):


Test patch:
diff --git a/db/tests/types/test_base.py b/db/tests/types/test_alteration.py
similarity index 93%
rename from db/tests/types/test_base.py
rename to db/tests/types/test_alteration.py
--- a/db/tests/types/test_base.py
+++ b/db/tests/types/test_alteration.py
@@ -6,7 +6,7 @@
 from sqlalchemy.schema import CreateSchema, DropSchema
 from db import types
 from db.engine import _add_custom_types_to_engine
-from db.types import base, install
+from db.types import alteration, base, install
 
 TEST_SCHEMA = "test_schema"
 
@@ -37,13 +37,13 @@ def engine_email_type(temporary_testing_schema):
 
 
 def test_get_alter_column_types_with_standard_engine(engine):
-    type_dict = base.get_supported_alter_column_types(engine)
+    type_dict = alteration.get_supported_alter_column_types(engine)
     assert len(type_dict) > 0
     assert all([type_ not in type_dict for type_ in types.CUSTOM_TYPE_DICT])
 
 
 def test_get_alter_column_types_with_custom_engine(engine_with_types):
-    type_dict = base.get_supported_alter_column_types(engine_with_types)
+    type_dict = alteration.get_supported_alter_column_types(engine_with_types)
     assert all(
         [
             type_ in type_dict.values()
@@ -78,7 +78,7 @@ def test_alter_column_type_alters_column_type(
         schema=schema
     )
     input_table.create()
-    base.alter_column_type(
+    alteration.alter_column_type(
         schema, TABLE_NAME, COLUMN_NAME, target_type, engine,
     )
     metadata = MetaData(bind=engine)
@@ -94,8 +94,6 @@ def test_alter_column_type_alters_column_type(
 
 
 type_test_data_list = [
-    (String, "boolean", "0", False),
-    (String, "boolean", "1", True),
     (String, "boolean", "false", False),
     (String, "boolean", "true", True),
     (String, "boolean", "f", False),
@@ -136,7 +134,7 @@ def test_alter_column_type_casts_column_data(
     ins = input_table.insert(values=(value,))
     with engine.begin() as conn:
         conn.execute(ins)
-    base.alter_column_type(
+    alteration.alter_column_type(
         schema, TABLE_NAME, COLUMN_NAME, target_type, engine,
     )
     metadata = MetaData(bind=engine)
@@ -155,8 +153,11 @@ def test_alter_column_type_casts_column_data(
 
 
 type_test_bad_data_list = [
+    (String, "boolean", "0"),
+    (String, "boolean", "1"),
     (String, "boolean", "cat"),
     (String, "interval", "1 potato"),
+    (String, "interval", "3"),
     (String, "numeric", "abc"),
     (String, "email", "alice-example.com"),
 ]
@@ -183,6 +184,6 @@ def test_alter_column_type_raises_on_bad_column_data(
     with engine.begin() as conn:
         conn.execute(ins)
     with pytest.raises(Exception):
-        base.alter_column_type(
+        alteration.alter_column_type(
             schema, TABLE_NAME, COLUMN_NAME, target_type, engine,
         )