Repository: mathesar-foundation/mathesar

Issue:
Partial data loss when attempting to unset scale type_options independently of precision
## Reproduce

1. Set up a Number column with "Decimal Places" = `2` and "Max Digits" = `100`.

1. Add row with a value of `1.23`.

1. Now imagine we'd like to also store the value `6.789` without loss of precision. We'll need to modify or remove the "Decimal Places" setting.

1. Edit the DB type options. Remove the `2` in the "Decimal Places" field (leaving the field blank) and save.

1. Observe a PATCH request to the columns API.

1. Expect the response to have `type_options` of `{ precision: 100, scale: null }` or `{ precision: 100 }`.

1. Observe the response to have `type_options` of `{ precision: 100, scale: 0 }`.

1. Expect the cell value within the newly-added row to still be `1.23`.

1. Observe that the value of the cell has changed to `1` because `scale` has been changed to `0`. This is especially bad because it's **data loss**.


## Notes

- The [Postgres docs on numeric types ](https://www.postgresql.org/docs/current/datatype-numeric.html) appear to indicate that Postgres supports NUMERIC with a precision value and without a scale value.




Discussion:
@seancolsen Setting a column to `NUMERIC(100)` without giving a scale is equivalent to setting `NUMERIC(100, 0)` at the DB layer (see the docs you linked). There is no way to create a column with a defined precision and an arbitrary scale. I do see how this could be confusing. I suggest we either leave it as-is to replicate the underlying DB behavior, or disallow giving a precision without a scale to avoid confusion.
> disallow giving a precision without a scale to avoid confusion.

Yes. This is what I'd like to do. Can we do this at the API level with some simple request validation? That will ensure maximal safety against this sort of data loss. Plus it conveniently allows us to side-step the UX design questions related to decimal places and max digits, e.g. in #1420. For the purpose of this ticket, I don't care if there's an unintelligible error message or poor user experience -- I'm just trying to safeguard against data loss from a user doing something like entering `2` into "Max Digits" when they meant to enter `2` into "Decimal Places".
I can even make it an intelligible error message.

Resolving patch:
diff --git a/mathesar/api/db/viewsets/columns.py b/mathesar/api/db/viewsets/columns.py
--- a/mathesar/api/db/viewsets/columns.py
+++ b/mathesar/api/db/viewsets/columns.py
@@ -47,11 +47,6 @@ def create(self, request, table_pk=None):
         serializer = ColumnSerializer(data=request.data, context={'request': request})
         serializer.is_valid(raise_exception=True)
         type_options = request.data.get('type_options', None)
-        if type_options is not None:
-            scale = type_options.get('scale', None)
-            precision = type_options.get('precision', None)
-            if scale is not None and precision is None:
-                request.data['type_options']['precision'] = 1000
         if 'source_column' in serializer.validated_data:
             column = table.duplicate_column(
                 serializer.validated_data['source_column'],
diff --git a/mathesar/api/serializers/columns.py b/mathesar/api/serializers/columns.py
--- a/mathesar/api/serializers/columns.py
+++ b/mathesar/api/serializers/columns.py
@@ -3,17 +3,19 @@
 from rest_framework.fields import empty, SerializerMethodField
 from rest_framework.settings import api_settings
 
+from db.columns.exceptions import InvalidTypeError
+from db.columns.exceptions import InvalidTypeOptionError
+from db.types.base import PostgresType
+from db.types.operations.convert import get_db_type_enum_from_id
+from mathesar.api.exceptions.database_exceptions import (
+    exceptions as database_api_exceptions
+)
 from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin
 from mathesar.api.serializers.shared_serializers import (
     DisplayOptionsMappingSerializer,
     DISPLAY_OPTIONS_SERIALIZER_MAPPING_KEY,
 )
-from mathesar.api.exceptions.database_exceptions import (
-    exceptions as database_api_exceptions,
-)
-from db.columns.exceptions import InvalidTypeError
 from mathesar.models.base import Column
-from db.types.operations.convert import get_db_type_enum_from_id
 
 
 class InputValueField(serializers.CharField):
@@ -37,8 +39,18 @@ class TypeOptionSerializer(MathesarErrorMessageMixin, serializers.Serializer):
     fields = serializers.CharField(required=False)
 
     def validate(self, attrs):
-        if attrs.get('scale', None) is not None and attrs.get('precision', None) is None:
-            attrs['precision'] = 1000
+        db_type = self.context.get('db_type', None)
+        scale = attrs.get('scale', None)
+        precision = attrs.get('precision', None)
+        if (
+            db_type == PostgresType.NUMERIC
+            and (scale is None) != (precision is None)
+        ):
+            raise database_api_exceptions.InvalidTypeOptionAPIException(
+                InvalidTypeOptionError,
+                message='Both scale and precision fields are required.',
+                status_code=status.HTTP_400_BAD_REQUEST,
+            )
         return super().validate(attrs)
 
     def run_validation(self, data=empty):


Test patch:
diff --git a/mathesar/tests/api/test_column_api.py b/mathesar/tests/api/test_column_api.py
--- a/mathesar/tests/api/test_column_api.py
+++ b/mathesar/tests/api/test_column_api.py
@@ -234,7 +234,6 @@ def test_column_create_invalid_default(column_test_table, client):
     "db_type,type_options,expected_type_options",
     [
         (PostgresType.NUMERIC, {"precision": 5, "scale": 3}, {"precision": 5, "scale": 3}),
-        (PostgresType.NUMERIC, {"scale": 3}, {"precision": 1000, "scale": 3}),
         (PostgresType.CHARACTER_VARYING, {"length": 5}, {"length": 5}),
         (PostgresType.CHARACTER, {"length": 5}, {"length": 5}),
         (PostgresType.INTERVAL, {"precision": 5}, {"precision": 5}),
@@ -265,6 +264,8 @@ def test_column_create_retrieve_options(column_test_table, client, db_type, type
 
 invalid_type_options = [
     {"precision": 5, "scale": 8},
+    {"precision": 1000},
+    {"scale": 5},
     {"precision": "asd"},
     {"nonoption": 34},
     {"length": "two"},
@@ -488,7 +489,7 @@ def test_column_update_name_type_nullable_default(column_test_table, client):
 
 def test_column_update_type_options(column_test_table, client):
     db_type = PostgresType.NUMERIC
-    type_options = {"scale": 1}
+    type_options = {'precision': 1000, "scale": 1}
     expected_type_options = {'precision': 1000, 'scale': 1}
     data = {"type": db_type.id, "type_options": type_options}
     column = column_test_table.get_columns_by_name(['mycolumn3'])[0]