Repository: man-group/ArcticDB (language: python)

Issue:
Do not allow writing incompatible data in case dynamic schema is disabled
### Describe the bug

ArcticDB allows the user to create a column one type then append an incompatible type to it. The writing succeeds and then it fails on the read side. Not all combinations succeed on the write side. It seems like ArcticDB allows appending columns of the same `ValueType` without checking the size of the values. E.g. creating a column of `int32` type and then appending `int64` to it can be done, but the read will fail. Same goes for `float32` and `float64`. Trying to append string to an int/float column works as expected and fails. No problems are detected if dynamic schema is enabled.

### Steps/Code to Reproduce

```python
ac = Arctic('lmdb://test')
lib = ac.get_library('test', create_if_missing=True)
# Writing and appending works just fine
df = pd.DataFrame({"col1": [1, 2]}, dtype=np.int32)
lib.write('test', df)
df = pd.DataFrame({"col1": [5]}, , dtype=np.int64)
lib.append('test', df)
# Reading fails
out = lmdb_version_store_v2.read("sym")
# Throws: arcticdb_ext.exceptions.InternalException: E_ASSERTION_FAILURE Column type conversion from TD<type=INT32, dim=0> to TD<type=INT64, dim=0> not implemented in column 0:col1 -> 0:col1
```

### Expected Results

Throw exception at the moment when we try to append incompatible data, rather than accepting it and failing on the read side.

### OS, Python Version and ArcticDB Version

Python: all
ArcticDB: [v4.1.0](https://github.com/man-group/ArcticDB/releases/tag/v4.1.0)

### Backend storage used

_No response_

### Additional Context

_No response_


Discussion / hints:

Fix patch:
diff --git a/cpp/arcticdb/version/schema_checks.hpp b/cpp/arcticdb/version/schema_checks.hpp
index e2478de6ea..526d9b7fb7 100644
--- a/cpp/arcticdb/version/schema_checks.hpp
+++ b/cpp/arcticdb/version/schema_checks.hpp
@@ -66,10 +66,8 @@ inline bool columns_match(const StreamDescriptor &left, const StreamDescriptor &
         const TypeDescriptor &left_type = left.fields(i).type();
         const TypeDescriptor &right_type = right.fields(i).type();
 
-        const bool valid_common_type = has_valid_common_type(left_type, right_type).has_value();
-        const bool trivial_type_compatibility = trivially_compatible_types(left_type, right_type);
-
-        if (!trivial_type_compatibility and !valid_common_type)
+        if (!trivially_compatible_types(left_type, right_type) &&
+            !(is_empty_type(left_type.data_type()) || is_empty_type(right_type.data_type())))
             return false;
     }
     return true;


Test patch:
diff --git a/python/tests/unit/arcticdb/version_store/test_column_type_changes.py b/python/tests/unit/arcticdb/version_store/test_column_type_changes.py
new file mode 100644
index 0000000000..1d7051787b
--- /dev/null
+++ b/python/tests/unit/arcticdb/version_store/test_column_type_changes.py
@@ -0,0 +1,103 @@
+"""
+Copyright 2024 Man Group Operations Limited
+
+Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+
+As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+"""
+import numpy as np
+import pandas as pd
+import pytest
+
+from arcticdb_ext.version_store import StreamDescriptorMismatch
+from arcticdb.util.test import assert_frame_equal
+
+
+@pytest.mark.parametrize("dynamic_schema", [True, False])
+def test_changing_numeric_type(version_store_factory, dynamic_schema):
+    lib = version_store_factory(dynamic_schema=dynamic_schema)
+    sym_append = "test_changing_numeric_type_append"
+    sym_update = "test_changing_numeric_type_update"
+    df_write = pd.DataFrame({"col": np.arange(3, dtype=np.uint8)}, index=pd.date_range("2024-01-01", periods=3))
+    df_append = pd.DataFrame({"col": np.arange(1, dtype=np.uint16)}, index=pd.date_range("2024-01-04", periods=1))
+    df_update = pd.DataFrame({"col": np.arange(1, dtype=np.uint16)}, index=pd.date_range("2024-01-02", periods=1))
+
+    lib.write(sym_append, df_write)
+    lib.write(sym_update, df_write)
+
+    if not dynamic_schema:
+        with pytest.raises(StreamDescriptorMismatch):
+            lib.append(sym_append, df_append)
+        with pytest.raises(StreamDescriptorMismatch):
+            lib.update(sym_update, df_update)
+    else:
+        lib.append(sym_append, df_append)
+        lib.update(sym_update, df_update)
+
+        expected_append = pd.concat([df_write, df_append])
+        received_append = lib.read(sym_append).data
+        assert_frame_equal(expected_append, received_append)
+
+        expected_update = pd.DataFrame({"col": np.array([0, 0, 2], dtype=np.uint16)}, index=pd.date_range("2024-01-01", periods=3))
+        received_update = lib.read(sym_update).data
+        assert_frame_equal(expected_update, received_update)
+
+
+@pytest.mark.parametrize("dynamic_schema, dynamic_strings_first", [
+    (True, True),
+    (True, False),
+    pytest.param(False,
+                 True,
+                 marks=pytest.mark.xfail(
+                     reason="""Issue with appending/updating a dynamic string column with fixed-width strings
+                     https://github.com/man-group/ArcticDB/issues/1204"""
+                 )
+                 ),
+    (False, False),
+])
+def test_changing_string_type(version_store_factory, dynamic_schema, dynamic_strings_first):
+    lib = version_store_factory(dynamic_strings=True, dynamic_schema=dynamic_schema)
+    sym_append = "test_changing_string_type_append"
+    sym_update = "test_changing_string_type_update"
+    df_write = pd.DataFrame({"col": ["a", "bb", "ccc"]}, index=pd.date_range("2024-01-01", periods=3))
+    df_append = pd.DataFrame({"col": ["dddd"]}, index=pd.date_range("2024-01-04", periods=1))
+    df_update = pd.DataFrame({"col": ["dddd"]}, index=pd.date_range("2024-01-02", periods=1))
+
+    lib.write(sym_append, df_write, dynamic_strings=dynamic_strings_first)
+    lib.write(sym_update, df_write, dynamic_strings=dynamic_strings_first)
+
+    lib.append(sym_append, df_append, dynamic_strings=not dynamic_strings_first)
+    lib.update(sym_update, df_update, dynamic_strings=not dynamic_strings_first)
+
+    expected_append = pd.concat([df_write, df_append])
+    received_append = lib.read(sym_append).data
+    assert_frame_equal(expected_append, received_append)
+
+    expected_update = pd.DataFrame({"col": ["a", "dddd", "ccc"]}, index=pd.date_range("2024-01-01", periods=3))
+    received_update = lib.read(sym_update).data
+    assert_frame_equal(expected_update, received_update)
+
+
+@pytest.mark.parametrize("dynamic_schema", [True, False])
+@pytest.mark.parametrize("wider_strings_first", [True, False])
+def test_changing_fixed_string_width(version_store_factory, dynamic_schema, wider_strings_first):
+    lib = version_store_factory(dynamic_strings=False, dynamic_schema=dynamic_schema)
+    sym_append = "test_changing_fixed_string_width_append"
+    sym_update = "test_changing_fixed_string_width_update"
+    df_write = pd.DataFrame({"col": ["aa", "bb", "cc"]}, index=pd.date_range("2024-01-01", periods=3))
+    df_append = pd.DataFrame({"col": ["d" * (1 if wider_strings_first else 3)]}, index=pd.date_range("2024-01-04", periods=1))
+    df_update = pd.DataFrame({"col": ["d" * (1 if wider_strings_first else 3)]}, index=pd.date_range("2024-01-02", periods=1))
+
+    lib.write(sym_append, df_write)
+    lib.write(sym_update, df_write)
+
+    lib.append(sym_append, df_append)
+    lib.update(sym_update, df_update)
+
+    expected_append = pd.concat([df_write, df_append])
+    received_append = lib.read(sym_append).data
+    assert_frame_equal(expected_append, received_append)
+
+    expected_update = pd.DataFrame({"col": ["aa", "d" * (1 if wider_strings_first else 3), "cc"]}, index=pd.date_range("2024-01-01", periods=3))
+    received_update = lib.read(sym_update).data
+    assert_frame_equal(expected_update, received_update)
\ No newline at end of file