Repository: man-group/ArcticDB (language: python)

Issue:
Add type handler for numpy arrays
### Description

As part of #177, we added functionality to support custom types. We want to support NumPy arrays as cell values enabling convenient storage of nested/multi-dimensional data.

### Why?

Users often want to store nested data in their DataFrames. For example, you can store Python lists of Python strings in DataFrames to store multiple strings per DataFrame cell. ArcticDB will require this data to be pickled, which limits the functionality:

```
In [2]: pd.DataFrame({'column': [['many', 'values'], ['in', 'one', 'row']]})
Out[2]:
           column
0  [many, values]
1  [in, one, row]
```

### Requirements

- [ ] Add support for NumPy arrays as a custom support
- [ ] Enable storage of numeric types as the secondary type (within-array type)
- [ ] Enable storage of Python strings as the secondary type (within-array type)

### Follow-ups

- Will analytics/QueryBuilder work similarly to how it does with Pandas?
- Will we also support NumPy fixed width strings?


Discussion / hints:

Fix patch:
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 88e5d91d5e..9b83b5b36f 100755
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -6,7 +6,6 @@ project(arcticdb VERSION 0.0.1)
 
 enable_testing()
 
-
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 option(
@@ -14,7 +13,40 @@ option(
     "Whether ArcticDB's build system relies on conda for resolving its dependencies."
     OFF
 )
+option(ARCTICDB_USING_ADDRESS_SANITIZER "Enable address sanitizer." OFF)
+option(ARCTICDB_USING_THREAD_SANITIZER "Enable thread sanitizer." OFF)
+option(ARCTICDB_USING_UB_SANITIZER "Enable undefined behavior sanitizer." OFF)
+
+set(ARCTICDB_SANITIZER_FLAGS)
+if(${ARCTICDB_USING_ADDRESS_SANITIZER})
+    list(APPEND ARCTICDB_SANITIZER_FLAGS "-fsanitize=address;-shared-libasan")
+    link_libraries(atomic)
+    message(STATUS "Building ArcticDB with Address Sanitizer")
+endif()
+if(${ARCTICDB_USING_THREAD_SANITIZER})
+    list(APPEND ARCTICDB_SANITIZER_FLAGS "-fsanitize=thread")
+    link_libraries(atomic)
+    message(STATUS "Building ArcticDB with Thread Sanitizer")
+endif()
+if(${ARCTICDB_USING_UB_SANITIZER})
+    list(APPEND ARCTICDB_SANITIZER_FLAGS "-fsanitize=undefined")
+    message(STATUS "Building ArcticDB with UB Sanitizer")
+endif()
 
+if(ARCTICDB_SANITIZER_FLAGS)
+    list(APPEND ARCTICDB_SANITIZER_FLAGS "-g;-fno-omit-frame-pointer;-fno-optimize-sibling-calls;")
+    if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+        add_link_options(-fuse-ld=lld)
+        # According to clang's docs only LLD can be used for lining https://clang.llvm.org/docs/AddressSanitizer.html#usage
+        message(STATUS "Clang compiler detected. Enforcing lld compiler.")
+    else()
+        message(WARNING "Sanitizers are tested only under Clang compiler and LLD linker.")
+    endif()
+    # Note: Sanitizer flags are set globally for all targets created after
+    add_compile_options("${ARCTICDB_SANITIZER_FLAGS}")
+    add_link_options(${ARCTICDB_SANITIZER_FLAGS})
+    message(STATUS "Building ArcticDB with sanitizers. Compiler flags: ${ARCTICDB_SANITIZER_FLAGS}")
+endif()
 
 if(WIN32)
     add_compile_definitions(NOGDI)
diff --git a/cpp/arcticdb/CMakeLists.txt b/cpp/arcticdb/CMakeLists.txt
index a6e47223e2..b10f52bf0f 100644
--- a/cpp/arcticdb/CMakeLists.txt
+++ b/cpp/arcticdb/CMakeLists.txt
@@ -178,11 +178,14 @@ set(arcticdb_srcs
         async/task_scheduler.hpp
         async/tasks.hpp
         codec/codec.hpp
+        codec/encode_common.hpp
         codec/codec-inl.hpp
         codec/core.hpp
         codec/lz4.hpp
+        codec/magic_words.hpp
         codec/passthrough.hpp
         codec/slice_data_sink.hpp
+        codec/typed_block_encoder_impl.hpp
         codec/zstd.hpp
         column_store/block.hpp
         column_store/chunked_buffer.hpp
@@ -357,11 +360,14 @@ set(arcticdb_srcs
         async/task_scheduler.cpp
         async/tasks.cpp
         codec/codec.cpp
+        codec/encode_v1.cpp
+        codec/encode_v2.cpp
         codec/encoding_sizes.cpp
         codec/segment.cpp
         codec/variant_encoded_field_collection.cpp
         column_store/chunked_buffer.cpp
         column_store/column.cpp
+        column_store/column_data.cpp
         column_store/memory_segment_impl.cpp
         column_store/memory_segment_impl.cpp
         column_store/string_pool.cpp
@@ -823,6 +829,7 @@ if(${TEST})
             ${COMMON_PUBLIC_TEST_LIBRARIES}
             PRIVATE
             ${AWSSDK_LINK_LIBRARIES}
+            arcticdb_core_static
             )
 
         target_include_directories(test_unit_arcticdb PRIVATE
diff --git a/cpp/arcticdb/codec/codec-inl.hpp b/cpp/arcticdb/codec/codec-inl.hpp
index ba6a08f7e8..a799def672 100644
--- a/cpp/arcticdb/codec/codec-inl.hpp
+++ b/cpp/arcticdb/codec/codec-inl.hpp
@@ -9,23 +9,16 @@
 #error "This should only be included by codec.hpp"
 #endif
 
-#include <arcticdb/codec/core.hpp>
 #include <arcticdb/codec/encoding_sizes.hpp>
 #include <arcticdb/codec/passthrough.hpp>
 #include <arcticdb/codec/zstd.hpp>
 #include <arcticdb/codec/lz4.hpp>
 #include <arcticdb/codec/encoded_field.hpp>
-//#include <arcticdb/codec/tp4.hpp>
-#include <arcticdb/codec/slice_data_sink.hpp>
+#include <arcticdb/codec/magic_words.hpp>
 
-#include <arcticdb/util/pb_util.hpp>
 #include <arcticdb/util/bitset.hpp>
-#include <arcticdb/entity/performance_tracing.hpp>
 
 #include <arcticdb/util/buffer.hpp>
-#include <arcticdb/util/hash.hpp>
-#include <arcticdb/util/sparse_utils.hpp>
-
 
 #include <google/protobuf/text_format.h>
 
@@ -33,99 +26,6 @@
 
 namespace arcticdb {
 
-template<template<typename> class TypedBlock, class TD>
-struct TypedBlockEncoderImpl {
-
-    static size_t max_compressed_size(
-        const arcticdb::proto::encoding::VariantCodec &codec_opts,
-            const TypedBlock<TD> &typed_block) {
-        switch (codec_opts.codec_case()) {
-            case arcticdb::proto::encoding::VariantCodec::kZstd:
-                return get_zstd_compressed_size(typed_block);
-            case arcticdb::proto::encoding::VariantCodec::kLz4:
-                return get_lz4_compressed_size(typed_block);
-            case arcticdb::proto::encoding::VariantCodec::kPassthrough :
-                return get_passthrough_compressed_size(typed_block);
-            default:
-                return get_passthrough_compressed_size(typed_block);
-        }
-    }
-    /**
-     * Perform encoding of in memory field for storage
-     * @param f  view on the field data to encode. Can mutate underlying buffer
-     * @param field description of the encoding operation
-     * @param out output buffer to write the encoded values to. Must be resized if pos becomes > size
-     * @param pos position in bytes in the buffer where to start writing.
-     *          Modified to reflect the position after the last byte written
-     * @param opt Option used to dispatch to the appropriate encoder and configure it
-     */
-    template <typename EncodedFieldType>
-    static void encode(
-            const arcticdb::proto::encoding::VariantCodec &codec_opts,
-            TypedBlock<TD> &typed_block,
-            EncodedFieldType &field,
-            Buffer &out,
-            std::ptrdiff_t &pos) {
-        switch (codec_opts.codec_case()) {
-            case arcticdb::proto::encoding::VariantCodec::kZstd:
-                encode_zstd(codec_opts.zstd(), typed_block, field, out, pos);
-                break;
-            case arcticdb::proto::encoding::VariantCodec::kLz4:
-                encode_lz4(codec_opts.lz4(), typed_block, field, out, pos);
-                break;
-            case arcticdb::proto::encoding::VariantCodec::kPassthrough :
-                encode_passthrough(typed_block, field, out, pos);
-                break;
-//            case arcticdb::proto::encoding::VariantCodec::kTp4:
-//                encode_tp4(codec_opts.tp4(), field, field, out, pos);
-//                break;
-            default:
-                encode_passthrough(typed_block, field, out, pos);
-        }
-    }
-
-    template <typename EncodedFieldType>
-    static void encode_passthrough(
-        TypedBlock<TD> &typed_block,
-        EncodedFieldType &field,
-        Buffer &out,
-        std::ptrdiff_t &pos) {
-        arcticdb::detail::PassthroughEncoder<TypedBlock, TD>::encode(typed_block, field, out, pos);
-    }
-
-    template <typename EncodedFieldType>
-    static void encode_zstd(
-        const arcticdb::proto::encoding::VariantCodec::Zstd &opts,
-        TypedBlock<TD> &typed_block,
-        EncodedFieldType& field,
-        Buffer &out,
-        std::ptrdiff_t &pos) {
-        arcticdb::detail::ZstdEncoder<TypedBlock, TD>::encode(opts, typed_block, field, out, pos);
-    }
-
-    template <typename EncodedFieldType>
-    static void encode_lz4(
-        const arcticdb::proto::encoding::VariantCodec::Lz4 &opts,
-        TypedBlock<TD> &typed_block,
-        EncodedFieldType &field,
-        Buffer &out,
-        std::ptrdiff_t &pos) {
-        arcticdb::detail::Lz4Encoder<TypedBlock, TD>::encode(opts, typed_block, field, out, pos);
-    }
-
-    static size_t get_passthrough_compressed_size(const TypedBlock<TD> &typed_block ) {
-        return arcticdb::detail::PassthroughEncoder<TypedBlock, TD>::max_compressed_size(typed_block);
-    }
-
-    static size_t get_zstd_compressed_size(const TypedBlock<TD> &typed_block) {
-        return arcticdb::detail::ZstdEncoder<TypedBlock, TD>::max_compressed_size(typed_block);
-    }
-
-    static size_t get_lz4_compressed_size(const TypedBlock<TD> &typed_block) {
-        return arcticdb::detail::Lz4Encoder<TypedBlock, TD>::max_compressed_size(typed_block);
-    }
-};
-
 template<typename T, typename BlockType>
 void decode_block(const BlockType &block, const std::uint8_t *input, T *output) {
     ARCTICDB_SUBSAMPLE_AGG(DecodeBlock)
@@ -157,44 +57,71 @@ void decode_block(const BlockType &block, const std::uint8_t *input, T *output)
     }
 }
 
+template<typename FieldType, class DataSink>
+inline void read_shapes(
+    FieldType& encoded_field,
+    DataSink& data_sink,
+    uint8_t const *& data_in,
+    int shapes_block,
+    shape_t*& shapes_out
+) {
+    const auto& shape = encoded_field.shapes(shapes_block);
+    decode_block<shape_t>(shape, data_in, shapes_out);
+    data_in += shape.out_bytes();
+    shapes_out += shape.in_bytes() / sizeof(shape_t);
+    data_sink.advance_shapes(shape.in_bytes());
+}
+
 template<class DataSink, typename NDArrayEncodedFieldType>
 std::size_t decode_ndarray(
-    const TypeDescriptor &td,
-    const NDArrayEncodedFieldType &field,
-    const std::uint8_t *input,
-    DataSink &data_sink,
-    std::optional<util::BitMagic>& bv) {
+    const TypeDescriptor& td,
+    const NDArrayEncodedFieldType& field,
+    const std::uint8_t* input,
+    DataSink& data_sink,
+    std::optional<util::BitMagic>& bv,
+    EncodingVersion encoding_version
+) {
     ARCTICDB_SUBSAMPLE_AGG(DecodeNdArray)
     std::size_t read_bytes = 0;
     td.visit_tag([&](auto type_desc_tag) {
         using TD = std::decay_t<decltype(type_desc_tag)>;
         using T = typename TD::DataTypeTag::raw_type;
 
-        auto shape_size = encoding_sizes::shape_uncompressed_size(field);
-        shape_t *shapes_out = nullptr;
-        if(shape_size > 0) {
-            shapes_out = data_sink.allocate_shapes(shape_size);
-            util::check(td.dimension() == Dimension::Dim0 || field.shapes_size() == field.values_size(),
-                        "Mismatched field and value sizes: {} != {}", field.shapes_size(), field.values_size());
+        const auto data_size = encoding_sizes::data_uncompressed_size(field);
+        const bool is_empty_array = (data_size == 0) && type_desc_tag.dimension() > Dimension::Dim0;
+        // Empty array will not contain actual data, however, its sparse map should be loaded
+        // so that we can distinguish None from []
+        if(data_size == 0 && !is_empty_array) {
+            util::check(type_desc_tag.data_type() == DataType::EMPTYVAL,
+                "NDArray of type {} should not be of size 0!",
+                datatype_to_str(type_desc_tag.data_type()));
+            return;
         }
 
-        auto data_size = encoding_sizes::data_uncompressed_size(field);
-        auto data_begin = static_cast<uint8_t *>(data_sink.allocate_data(data_size));
-        util::check(data_begin != nullptr, "Failed to allocate data of size {}", data_size);
+        auto data_begin = is_empty_array ? nullptr : static_cast<uint8_t*>(data_sink.allocate_data(data_size));
+        util::check(is_empty_array || data_begin != nullptr, "Failed to allocate data of size {}", data_size);
         auto data_out = data_begin;
         auto data_in = input;
         auto num_blocks = field.values_size();
 
         ARCTICDB_TRACE(log::codec(), "Decoding ndarray with type {}, uncompressing {} ({}) bytes in {} blocks",
             td, data_size, encoding_sizes::ndarray_field_compressed_size(field), num_blocks);
-
+        shape_t *shapes_out = nullptr;
+        if constexpr(TD::DimensionTag::value != Dimension::Dim0) {
+            const auto shape_size = encoding_sizes::shape_uncompressed_size(field);
+            if(shape_size > 0) {
+                shapes_out = data_sink.allocate_shapes(shape_size);
+                if(encoding_version == EncodingVersion::V2)
+                    read_shapes(field, data_sink, data_in, 0, shapes_out);
+            }
+        }
         for (auto block_num = 0; block_num < num_blocks; ++block_num) {
-            if (td.dimension() != Dimension::Dim0) {
-                const auto& shape = field.shapes(block_num);
-                decode_block<shape_t>(shape, data_in, shapes_out);
-                data_in += shape.out_bytes();
-                shapes_out += shape.in_bytes() / sizeof(shape_t);
-                data_sink.advance_shapes(shape.in_bytes());
+            if constexpr(TD::DimensionTag::value != Dimension::Dim0) {
+                // In V1 encoding each block of values is preceded by a block of shapes.
+                // In V2 encoding all shapes are put in a single block placed at the beginning of the block chain.
+                if(encoding_version == EncodingVersion::V1) {
+                    read_shapes(field, data_sink, data_in, block_num, shapes_out);
+                }
             }
 
             const auto& block_info = field.values(block_num);
@@ -209,7 +136,7 @@ std::size_t decode_ndarray(
 
         if(field.sparse_map_bytes()) {
             util::check_magic<util::BitMagicStart>(data_in);
-            const auto bitmap_size =   field.sparse_map_bytes() - (sizeof(util::BitMagicStart) + sizeof(util::BitMagicEnd)); //TODO functions
+            const auto bitmap_size = field.sparse_map_bytes() - util::combined_bit_magic_delimiters_size();
             bv = util::deserialize_bytes_to_bitmap(data_in, bitmap_size);
             util::check_magic<util::BitMagicEnd>(data_in);
             data_sink.set_allow_sparse(true);
@@ -217,12 +144,12 @@ std::size_t decode_ndarray(
 
         read_bytes = encoding_sizes::ndarray_field_compressed_size(field);
         util::check(data_in - input == intptr_t(read_bytes),
-                    "Decoding compressed size mismatch, expected decode size {} to equal total size {}", data_in - input ,
-                    read_bytes);
+            "Decoding compressed size mismatch, expected decode size {} to equal total size {}", data_in - input,
+             read_bytes);
 
         util::check(data_out - data_begin == intptr_t(data_size),
-                    "Decoding uncompressed size mismatch, expected position {} to be equal to data size {}",
-                    data_out - data_begin, data_size);
+            "Decoding uncompressed size mismatch, expected position {} to be equal to data size {}",
+             data_out - data_begin, data_size);
     });
     return read_bytes;
 }
@@ -233,19 +160,20 @@ std::size_t decode_field(
     const EncodedFieldType &field,
     const std::uint8_t *input,
     DataSink &data_sink,
-    std::optional<util::BitMagic>& bv) {
+    std::optional<util::BitMagic>& bv,
+    EncodingVersion encoding_version) {
     size_t magic_size = 0u;
-    if constexpr(std::is_same_v<EncodedFieldType, arcticdb::EncodedField>) {
+    if constexpr(std::is_same_v<EncodedFieldType, EncodedField>) {
         magic_size += sizeof(ColumnMagic);
-        check_magic<ColumnMagic>(input);
+        util::check_magic<ColumnMagic>(input);
     }
 
     switch (field.encoding_case()) {
         case EncodedFieldType::kNdarray:
-            return decode_ndarray(td, field.ndarray(), input, data_sink, bv) + magic_size;
+            return decode_ndarray(td, field.ndarray(), input, data_sink, bv, encoding_version) + magic_size;
         default:
             util::raise_error_msg("Unsupported encoding {}", field);
     }
 }
 
-} // namespace arcticdb
\ No newline at end of file
+} // namespace arcticdb
diff --git a/cpp/arcticdb/codec/codec.cpp b/cpp/arcticdb/codec/codec.cpp
index a32742f231..64338f4360 100644
--- a/cpp/arcticdb/codec/codec.cpp
+++ b/cpp/arcticdb/codec/codec.cpp
@@ -4,51 +4,32 @@
  *
  * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
  */
-
 #include <arcticdb/codec/codec.hpp>
 #include <arcticdb/stream/protobuf_mappings.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
-#include <arcticdb/storage/common.hpp>
-#include <arcticdb/util/configs_map.hpp>
+#include <arcticdb/codec/default_codecs.hpp>
 #include <arcticdb/codec/encoded_field.hpp>
 #include <arcticdb/codec/encoded_field_collection.hpp>
 
+
 #include <string>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 
+
+#include <arcticdb/codec/encode_common.hpp>
+
+
 namespace arcticdb {
 
-std::pair<size_t, size_t> ColumnEncoder::max_compressed_size(
-        const arcticdb::proto::encoding::VariantCodec &codec_opts,
-        ColumnData &column_data) {
-    return column_data.type().visit_tag([&codec_opts, &column_data](auto type_desc_tag) {
-        size_t max_compressed_bytes = 0;
-        size_t uncompressed_bytes = 0;
-        using TDT = decltype(type_desc_tag);
-        using Encoder = BlockEncoder<TDT>;
-        ARCTICDB_TRACE(log::codec(), "Column data has {} blocks", column_data.num_blocks());
-
-        while (auto block = column_data.next<TDT>()) {
-            const auto nbytes = block->nbytes();
-            if constexpr(!is_empty_type(TDT::DataTypeTag::data_type)) {
-                util::check(nbytes > 0, "Zero-sized block");
-                uncompressed_bytes += nbytes;
-            }
-            // For the empty type the column will contain 0 size of user data however the encoder might need add some
-            // encoder specific data to the buffer, thus the uncompressed size will be 0 but the max_compressed_bytes
-            // might be non-zero.
-            max_compressed_bytes += Encoder::max_compressed_size(codec_opts, *block);
-        }
+Segment encode_v2(
+    SegmentInMemory&& in_mem_seg,
+    const arcticdb::proto::encoding::VariantCodec& codec_opts
+);
 
-        if (column_data.bit_vector() != nullptr && column_data.bit_vector()->count() > 0)   {
-            bm::serializer<util::BitMagic>::statistics_type stat{};
-            column_data.bit_vector()->calc_stat(&stat);
-            uncompressed_bytes += stat.memory_used;
-            max_compressed_bytes += stat.max_serialize_mem;
-        }
-        return std::make_pair(uncompressed_bytes, max_compressed_bytes);
-    });
-}
+Segment encode_v1(
+    SegmentInMemory&& in_mem_seg,
+    const arcticdb::proto::encoding::VariantCodec& codec_opts
+);
 
 constexpr TypeDescriptor metadata_type_desc() {
     return TypeDescriptor{
@@ -62,375 +43,17 @@ constexpr TypeDescriptor encoded_blocks_type_desc() {
     };
 }
 
-size_t calc_column_blocks_size(const Column& col) {
-    size_t bytes = EncodedField::Size;
-    if(col.type().dimension() != entity::Dimension::Dim0)
-        bytes += sizeof(EncodedBlock);
-
-    bytes += sizeof(EncodedBlock) * col.num_blocks();
-    ARCTICDB_DEBUG(log::version(), "Encoded block size: {} + shapes({}) + {} * {} = {}",
-                 EncodedField::Size,
-                 col.type().dimension() != entity::Dimension::Dim0 ? sizeof(EncodedBlock) : 0u,
-                 sizeof(EncodedBlock),
-                 col.num_blocks(),
-                 bytes);
-
-    return bytes;
-}
-
-size_t encoded_blocks_size(const SegmentInMemory& in_mem_seg) {
-    size_t bytes = 0;
-    for (std::size_t c = 0; c < in_mem_seg.num_columns(); ++c) {
-        const auto& col = in_mem_seg.column(position_t(c));
-        bytes += calc_column_blocks_size(col);
-    }
-    return bytes;
-}
-
-struct SizeResult {
-    size_t max_compressed_bytes_;
-    size_t uncompressed_bytes_;
-    shape_t encoded_blocks_bytes_;
-};
-
-void calc_metadata_size(
-    const SegmentInMemory& in_mem_seg,
-    const arcticdb::proto::encoding::VariantCodec& codec_opts,
-    SizeResult& result
-    ) {
-    using BytesTypeDescriptorTag = TypeDescriptorTag<DataTypeTag<DataType::UINT8>, DimensionTag<Dimension::Dim1>>;
-    using BytesEncoder = BlockEncoder<BytesTypeDescriptorTag>;
-
-    if (in_mem_seg.metadata()) {
-        auto metadata_bytes = static_cast<shape_t>(in_mem_seg.metadata()->ByteSizeLong());
-        result.uncompressed_bytes_ += metadata_bytes;
-        result.max_compressed_bytes_ += BytesEncoder::max_compressed_size(codec_opts, TypedBlockData<BytesTypeDescriptorTag>(metadata_bytes, &metadata_bytes));
-        shape_t shapes_bytes = sizeof(shape_t);
-        result.uncompressed_bytes_ += shapes_bytes;
-        result.max_compressed_bytes_ += BytesEncoder::max_compressed_size(codec_opts, TypedBlockData<BytesTypeDescriptorTag>(shapes_bytes, &shapes_bytes));
-        ARCTICDB_TRACE(log::codec(), "Metadata requires {} max_compressed_bytes", result.max_compressed_bytes_);
-    }
-}
-
-void calc_columns_size(
-    const SegmentInMemory& in_mem_seg,
-    const arcticdb::proto::encoding::VariantCodec& codec_opts,
-    SizeResult& result
-    ) {
-    ColumnEncoder encoder;
-    for (std::size_t c = 0; c < in_mem_seg.num_columns(); ++c) {
-        auto column_data = in_mem_seg.column_data(c);
-        const auto [uncompressed, required] = encoder.max_compressed_size(codec_opts, column_data);
-        result.uncompressed_bytes_ += uncompressed;
-        result.max_compressed_bytes_ += required;
-        ARCTICDB_TRACE(log::codec(), "Column {} requires {} max_compressed_bytes, total {}", c, required, result.max_compressed_bytes_);
-    }
-}
-
-void calc_string_pool_size(
-    const SegmentInMemory& in_mem_seg,
-    const arcticdb::proto::encoding::VariantCodec& codec_opts,
-    SizeResult& result
-    ) {
-    if (in_mem_seg.has_string_pool()) {
-        ColumnEncoder encoder;
-        auto string_col = in_mem_seg.string_pool_data();
-        const auto [uncompressed, required] = encoder.max_compressed_size(codec_opts, string_col);
-        result.uncompressed_bytes_ += uncompressed;
-        result.max_compressed_bytes_ += required;
-        ARCTICDB_TRACE(log::codec(), "String pool requires {} max_compressed_bytes, total {}", required, result.max_compressed_bytes_);
-    }
-}
-
-SizeResult max_compressed_size_v1(const SegmentInMemory &in_mem_seg, const arcticdb::proto::encoding::VariantCodec &codec_opts) {
-    ARCTICDB_SAMPLE(GetSegmentCompressedSize, 0)
-    SizeResult result{};
-    calc_metadata_size(in_mem_seg, codec_opts, result);
-
-    if(in_mem_seg.row_count() > 0) {
-        calc_columns_size(in_mem_seg, codec_opts, result);
-        calc_string_pool_size(in_mem_seg, codec_opts, result);
-    }
-    ARCTICDB_TRACE(log::codec(), "Max compressed size {}", result.max_compressed_bytes_);
-    return result;
-}
-
-void calc_encoded_blocks_size(
-    const SegmentInMemory& in_mem_seg,
-    const arcticdb::proto::encoding::VariantCodec& codec_opts,
-    SizeResult& result
-    ) {
-    using BytesTypeDescriptorTag = TypeDescriptorTag<DataTypeTag<DataType::UINT8>, DimensionTag<Dimension::Dim1>>;
-    using BytesEncoder = BlockEncoder<BytesTypeDescriptorTag>;
-
-    result.encoded_blocks_bytes_ = static_cast<shape_t>(encoded_blocks_size(in_mem_seg));
-    result.uncompressed_bytes_ += result.encoded_blocks_bytes_;
-    result.max_compressed_bytes_ += BytesEncoder::max_compressed_size(codec_opts, TypedBlockData<BytesTypeDescriptorTag>(result.encoded_blocks_bytes_, &result.encoded_blocks_bytes_));
-}
-
-void calc_stream_descriptor_fields_size(
-    const SegmentInMemory& in_mem_seg,
-    const arcticdb::proto::encoding::VariantCodec& codec_opts,
-    SizeResult& result
-    ) {
-    ColumnEncoder encoder;
-    auto segment_fields = in_mem_seg.descriptor().fields().column_data();
-    const auto [uncompressed, required] = encoder.max_compressed_size(codec_opts, segment_fields);
-    result.uncompressed_bytes_ += uncompressed;
-    result.max_compressed_bytes_ += required;
-
-    // Calculate index fields size
-    if(in_mem_seg.index_fields()) {
-        auto index_field_data = in_mem_seg.index_fields()->column_data();
-        const auto [idx_uncompressed, idx_required] = encoder.max_compressed_size(codec_opts, index_field_data);
-        result.uncompressed_bytes_ += idx_uncompressed;
-        result.max_compressed_bytes_ += idx_required;
-    }
-}
-
-SizeResult max_compressed_size_v2(const SegmentInMemory &in_mem_seg, const arcticdb::proto::encoding::VariantCodec &codec_opts) {
-    ARCTICDB_SAMPLE(GetSegmentCompressedSize, 0)
-    SizeResult result{};
-    result.max_compressed_bytes_ += sizeof(MetadataMagic);
-    calc_metadata_size(in_mem_seg, codec_opts, result);
-    result.max_compressed_bytes_ += sizeof(DescriptorMagic);
-    result.max_compressed_bytes_ += sizeof(IndexMagic);
-    calc_stream_descriptor_fields_size(in_mem_seg, codec_opts, result);
-    result.max_compressed_bytes_ += sizeof(EncodedMagic);
-    calc_encoded_blocks_size(in_mem_seg, codec_opts, result);
-
-    // Calculate fields collection size
-    if(in_mem_seg.row_count() > 0) {
-        result.max_compressed_bytes_ += sizeof(ColumnMagic) * in_mem_seg.descriptor().field_count();
-        calc_columns_size(in_mem_seg, codec_opts, result);
-        result.max_compressed_bytes_ += sizeof(StringPoolMagic);
-        calc_string_pool_size(in_mem_seg, codec_opts, result);
-    }
-    ARCTICDB_TRACE(log::codec(), "Max compressed size {}", result.max_compressed_bytes_);
-    return result;
-}
-
-void encode_metadata(
-    const SegmentInMemory& in_mem_seg,
-    arcticdb::proto::encoding::SegmentHeader& segment_header,
-    const arcticdb::proto::encoding::VariantCodec& codec_opts,
-    Buffer& out_buffer,
-    std::ptrdiff_t& pos
-    ) {
-    if (in_mem_seg.metadata()) {
-        const auto bytes_count = static_cast<shape_t>(in_mem_seg.metadata()->ByteSizeLong());
-        ARCTICDB_TRACE(log::codec(), "Encoding {} bytes of metadata", bytes_count);
-        auto encoded_field = segment_header.mutable_metadata_field();
-
-        constexpr int max_stack_alloc = 1 << 11;
-        bool malloced{false};
-        uint8_t* meta_ptr{nullptr};
-        if(bytes_count > max_stack_alloc) {
-            meta_ptr = reinterpret_cast<uint8_t*>(malloc(bytes_count));
-            malloced = true;
-        } else {
-            meta_ptr = reinterpret_cast<uint8_t*>(alloca(bytes_count));
-        }
-        ChunkedBuffer meta_buffer;
-        meta_buffer.add_external_block(meta_ptr, bytes_count, 0u);
-        google::protobuf::io::ArrayOutputStream aos(&meta_buffer[0], static_cast<int>(bytes_count));
-        in_mem_seg.metadata()->SerializeToZeroCopyStream(&aos);
-
-        using BytesTypeDescriptorTag = TypeDescriptorTag<DataTypeTag<DataType::UINT8>, DimensionTag<Dimension::Dim1>>;
-        using BytesEncoder = BlockEncoder<BytesTypeDescriptorTag>;
-        auto typed_block = TypedBlockData<BytesTypeDescriptorTag>(
-            meta_buffer.data(),
-            &bytes_count,
-            bytes_count,
-            1u,
-            meta_buffer.block_and_offset(0).block_);
-
-        BytesEncoder::encode(codec_opts, typed_block, *encoded_field, out_buffer, pos);
-        ARCTICDB_DEBUG(log::codec(), "Encoded metadata to position {}", pos);
-        if(malloced)
-            free(meta_ptr);
+Segment encode_dispatch(
+    SegmentInMemory&& in_mem_seg,
+    const arcticdb::proto::encoding::VariantCodec &codec_opts,
+    EncodingVersion encoding_version) {
+    if(encoding_version == EncodingVersion::V2) {
+        return encode_v2(std::move(in_mem_seg), codec_opts);
     } else {
-        ARCTICDB_DEBUG(log::codec(), "Not encoding any metadata");
-    }
-}
-
-void encode_string_pool(
-    const SegmentInMemory& in_mem_seg,
-    arcticdb::proto::encoding::SegmentHeader& segment_header,
-    const arcticdb::proto::encoding::VariantCodec& codec_opts,
-    Buffer& out_buffer,
-    std::ptrdiff_t& pos
-    ) {
-    if (in_mem_seg.has_string_pool()) {
-        ColumnEncoder encoder;
-        ARCTICDB_TRACE(log::codec(), "Encoding string pool to position {}", pos);
-        auto *encoded_field = segment_header.mutable_string_pool_field();
-        auto col = in_mem_seg.string_pool_data();
-        encoder.encode(codec_opts, col, *encoded_field, out_buffer, pos);
-        ARCTICDB_TRACE(log::codec(), "Encoded string pool to position {}", pos);
+        return encode_v1(std::move(in_mem_seg), codec_opts);
     }
 }
 
-void encode_field_descriptors(
-    const SegmentInMemory& in_mem_seg,
-    arcticdb::proto::encoding::SegmentHeader& segment_header,
-    const arcticdb::proto::encoding::VariantCodec& codec_opts,
-    Buffer& out_buffer,
-    std::ptrdiff_t& pos
-    ) {
-    ColumnEncoder encoder;
-    ARCTICDB_TRACE(log::codec(), "Encoding field descriptors to position {}", pos);
-    auto *encoded_field = segment_header.mutable_descriptor_field();
-    auto col = in_mem_seg.descriptor().fields().column_data();
-    encoder.encode(codec_opts, col, *encoded_field, out_buffer, pos);
-    ARCTICDB_TRACE(log::codec(), "Encoded field descriptors to position {}", pos);
-
-    write_magic<IndexMagic>(out_buffer, pos);
-    if (in_mem_seg.index_fields()) {
-        ARCTICDB_TRACE(log::codec(), "Encoding index fields descriptors to position {}", pos);
-        auto *index_field = segment_header.mutable_index_descriptor_field();
-        auto index_col = in_mem_seg.index_fields()->column_data();
-        encoder.encode(codec_opts, index_col, *index_field, out_buffer, pos);
-        ARCTICDB_TRACE(log::codec(), "Encoded index field descriptors to position {}", pos);
-    }
-}
-
-void encode_encoded_fields(
-        arcticdb::proto::encoding::SegmentHeader& segment_header,
-        const arcticdb::proto::encoding::VariantCodec& codec_opts,
-        Buffer& out_buffer,
-        std::ptrdiff_t& pos,
-        const ChunkedBuffer& encoded_blocks_buffer
-        ) {
-    using BytesTypeDescriptorTag = TypeDescriptorTag<DataTypeTag<DataType::UINT8>, DimensionTag<Dimension::Dim1>>;
-    using BytesEncoder = BlockEncoder<BytesTypeDescriptorTag>;
-
-    ARCTICDB_TRACE(log::codec(), "Encoding encoded blocks to position {}", pos);
-    auto encoded_field = segment_header.mutable_column_fields();
-    encoded_field->set_offset(static_cast<uint32_t>(pos));
-    write_magic<EncodedMagic>(out_buffer, pos);
-
-    const auto bytes_count = static_cast<shape_t>(encoded_blocks_buffer.bytes());
-    auto typed_block = TypedBlockData<BytesTypeDescriptorTag>(
-            encoded_blocks_buffer.data(),
-            &bytes_count,
-            bytes_count,
-            1u,
-            encoded_blocks_buffer.block_and_offset(0).block_);
-
-    BytesEncoder::encode(codec_opts, typed_block, *encoded_field, out_buffer, pos);
-    ARCTICDB_TRACE(log::codec(), "Encoded encoded blocks to position {}", pos);
-}
-
-Segment encode_v2(SegmentInMemory&& s, const arcticdb::proto::encoding::VariantCodec &codec_opts) {
-    ARCTICDB_SAMPLE(EncodeSegment, 0)
-
-    auto in_mem_seg = std::move(s);
-    auto arena = std::make_unique<google::protobuf::Arena>();
-    auto segment_header = google::protobuf::Arena::CreateMessage<arcticdb::proto::encoding::SegmentHeader>(arena.get());
-    auto& seg_descriptor = in_mem_seg.descriptor();
-    *segment_header->mutable_stream_descriptor() = std::move(seg_descriptor.mutable_proto());
-    segment_header->set_compacted(in_mem_seg.compacted());
-    segment_header->set_encoding_version(static_cast<uint16_t>(EncodingVersion::V2));
-
-    std::ptrdiff_t pos = 0;
-    static auto block_to_header_ratio = ConfigsMap::instance()->get_int("Codec.EstimatedHeaderRatio", 75);
-    const auto preamble = in_mem_seg.num_blocks() * block_to_header_ratio;
-    auto [max_compressed_size, uncompressed_size, encoded_buffer_size] = max_compressed_size_v2(in_mem_seg, codec_opts);
-    ARCTICDB_TRACE(log::codec(), "Estimated max buffer requirement: {}", max_compressed_size);
-    auto out_buffer = std::make_shared<Buffer>(max_compressed_size + encoded_buffer_size, preamble);
-    ARCTICDB_TRACE(log::codec(), "Encoding descriptor: {}", segment_header->stream_descriptor().DebugString());
-    auto *tsd = segment_header->mutable_stream_descriptor();
-    tsd->set_in_bytes(uncompressed_size);
-
-    write_magic<MetadataMagic>(*out_buffer, pos);
-    encode_metadata(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
-    write_magic<DescriptorMagic>(*out_buffer, pos);
-    encode_field_descriptors(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
-
-    auto encoded_fields_buffer = ChunkedBuffer::presized(static_cast<size_t>(encoded_buffer_size));
-    auto encoded_field_pos = 0u;
-    ColumnEncoder encoder;
-    if(in_mem_seg.row_count() > 0) {
-        ARCTICDB_TRACE(log::codec(), "Encoding fields");
-        for (std::size_t c = 0; c < in_mem_seg.num_columns(); ++c) {
-            write_magic<ColumnMagic>(*out_buffer, pos);
-            auto col = in_mem_seg.column_data(c);
-            auto column_field = new(encoded_fields_buffer.data() + encoded_field_pos) EncodedField;
-            ARCTICDB_TRACE(log::codec(), "Beginning encoding of column {}: ({}) to position {}", c, seg_descriptor.field(c).name(), pos);
-            encoder.encode(codec_opts, col, *column_field, *out_buffer, pos);
-            ARCTICDB_TRACE(log::codec(), "Encoded column {}: ({}) to position {}", c, seg_descriptor.field(c).name(), pos);
-            encoded_field_pos += encoded_field_bytes(*column_field);
-            util::check(encoded_field_pos <= encoded_fields_buffer.bytes(), "Encoded field buffer overflow {} > {}", encoded_field_pos, encoded_fields_buffer.bytes());
-            auto pos_tmp     ARCTICDB_UNUSED = encoded_field_bytes(*column_field);
-            auto field_in_here ARCTICDB_UNUSED = reinterpret_cast<EncodedField*>(encoded_fields_buffer.data());
-            //log::version().debug("{}", dump_bytes(out_buffer->data(), out_buffer->bytes(), 100u));
-        }
-        auto field_here ARCTICDB_UNUSED = reinterpret_cast<EncodedField*>(encoded_fields_buffer.data());
-        write_magic<StringPoolMagic>(*out_buffer, pos);
-        encode_string_pool(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
-    }
-
-    auto field_before ARCTICDB_UNUSED = reinterpret_cast<EncodedField*>(encoded_fields_buffer.data());
-    encode_encoded_fields(*segment_header, codec_opts, *out_buffer, pos, encoded_fields_buffer);
-    auto field ARCTICDB_UNUSED = reinterpret_cast<EncodedField*>(encoded_fields_buffer.data());
-    out_buffer->set_bytes(pos);
-    tsd->set_out_bytes(pos);
-
-    ARCTICDB_DEBUG(log::codec(), "Encoded header: {}", tsd->DebugString());
-    ARCTICDB_DEBUG(log::codec(), "Block count {} header size {} ratio {}",
-        in_mem_seg.num_blocks(), segment_header->ByteSizeLong(),
-        in_mem_seg.num_blocks() ? segment_header->ByteSizeLong() / in_mem_seg.num_blocks() : 0);
-    return {std::move(arena), segment_header, std::move(out_buffer), seg_descriptor.fields_ptr()};
-}
-
-Segment encode_v1(SegmentInMemory&& s, const arcticdb::proto::encoding::VariantCodec &codec_opts) {
-    /*
-     * This takes an in memory segment with all the metadata, column tensors etc., loops through each column
-     * and based on the type of the column, calls the typed block encoder for that column.
-     */
-    ARCTICDB_SAMPLE(EncodeSegment, 0)
-    auto in_mem_seg = std::move(s);
-    auto arena = std::make_unique<google::protobuf::Arena>();
-    auto segment_header = google::protobuf::Arena::CreateMessage<arcticdb::proto::encoding::SegmentHeader>(arena.get());
-    *segment_header->mutable_stream_descriptor() = in_mem_seg.descriptor().copy_to_proto();
-    segment_header->set_compacted(in_mem_seg.compacted());
-    std::ptrdiff_t pos = 0;
-    static auto block_to_header_ratio = ConfigsMap::instance()->get_int("Codec.EstimatedHeaderRatio", 75);
-    const auto preamble = in_mem_seg.num_blocks() * block_to_header_ratio;
-    auto [max_compressed_size, uncompressed_size, encoded_blocks_bytes] = max_compressed_size_v1(in_mem_seg, codec_opts);
-    ARCTICDB_TRACE(log::codec(), "Estimated max buffer requirement: {}", max_compressed_size);
-    auto out_buffer = std::make_shared<Buffer>(max_compressed_size, preamble);
-    ColumnEncoder encoder;
-
-    ARCTICDB_TRACE(log::codec(), "Encoding descriptor: {}", segment_header->stream_descriptor().DebugString());
-    auto *tsd = segment_header->mutable_stream_descriptor();
-    tsd->set_in_bytes(uncompressed_size);
-
-    encode_metadata(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
-
-    if(in_mem_seg.row_count() > 0) {
-        ARCTICDB_TRACE(log::codec(), "Encoding fields");
-        for (std::size_t c = 0; c < in_mem_seg.num_columns(); ++c) {
-            auto col = in_mem_seg.column_data(c);
-            auto *encoded_field = segment_header->mutable_fields()->Add();
-            encoder.encode(codec_opts, col, *encoded_field, *out_buffer, pos);
-            ARCTICDB_TRACE(log::codec(), "Encoded column {}: ({}) to position {}", c, in_mem_seg.descriptor().fields(c).name(), pos);
-        }
-        encode_string_pool(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
-    }
-    ARCTICDB_DEBUG(log::codec(), "Setting buffer bytes to {}", pos);
-    out_buffer->set_bytes(pos);
-    tsd->set_out_bytes(pos);
-    ARCTICDB_DEBUG(log::codec(), "Encoded header: {}", tsd->DebugString());
-    if(!segment_header->has_metadata_field())
-        ARCTICDB_DEBUG(log::codec(), "No metadata field");
-    ARCTICDB_DEBUG(log::codec(), "Block count {} header size {} ratio {}",
-                  in_mem_seg.num_blocks(), segment_header->ByteSizeLong(),
-                  in_mem_seg.num_blocks() ? segment_header->ByteSizeLong() / in_mem_seg.num_blocks() : 0);
-    return {std::move(arena), segment_header, std::move(out_buffer), in_mem_seg.descriptor().fields_ptr()};
-}
-
 namespace {
 class MetaBuffer {
   public:
@@ -470,24 +93,6 @@ class MetaBuffer {
 };
 }
 
-size_t encode_bitmap(const util::BitMagic &sparse_map, Buffer &out, std::ptrdiff_t &pos) {
-    ARCTICDB_DEBUG(log::version(), "Encoding sparse map of count: {}", sparse_map.count());
-    bm::serializer<bm::bvector<> > bvs;
-    bm::serializer<bm::bvector<> >::buffer sbuf;
-    bvs.serialize(sparse_map, sbuf);
-    auto sz = sbuf.size();
-    auto total_sz = sz + sizeof(util::BitMagicStart) + sizeof(util::BitMagicEnd);
-    out.assert_size(pos + total_sz);
-
-    uint8_t* target = out.data() + pos;
-    util::write_magic<util::BitMagicStart>(target);
-    std::memcpy(target, sbuf.data(), sz);
-    target += sz;
-    util::write_magic<util::BitMagicEnd>(target);
-    pos = pos + static_cast<ptrdiff_t>(total_sz);
-    return total_sz;
-}
-
 std::optional<google::protobuf::Any> decode_metadata(
     const arcticdb::proto::encoding::SegmentHeader& hdr,
     const uint8_t*& data,
@@ -497,7 +102,7 @@ std::optional<google::protobuf::Any> decode_metadata(
         auto meta_type_desc = metadata_type_desc();
         MetaBuffer meta_buf;
         std::optional<util::BitMagic> bv;
-        data += decode_field(meta_type_desc, hdr.metadata_field(), data, meta_buf, bv);
+        data += decode_field(meta_type_desc, hdr.metadata_field(), data, meta_buf, bv, to_encoding_version(hdr.encoding_version()));
         ARCTICDB_TRACE(log::codec(), "Decoded metadata to position {}", data - begin);
         google::protobuf::io::ArrayInputStream ais(meta_buf.buffer().data(),
                                                    static_cast<int>(meta_buf.buffer().bytes()));
@@ -527,7 +132,7 @@ std::optional<google::protobuf::Any> decode_metadata_from_segment(const Segment
     const auto begin = data;
     const auto has_magic_numbers = EncodingVersion(hdr.encoding_version()) == EncodingVersion::V2;
     if(has_magic_numbers)
-        check_magic<MetadataMagic>(data);
+        util::check_magic<MetadataMagic>(data);
 
     return decode_metadata(hdr, data, begin);
 }
@@ -541,9 +146,8 @@ Buffer decode_encoded_fields(
         std::optional<util::BitMagic> bv;
         if(hdr.has_column_fields()) {
             constexpr auto type_desc = encoded_blocks_type_desc();
-            decode_field(type_desc, hdr.column_fields(), data, meta_buffer, bv);
+            decode_field(type_desc, hdr.column_fields(), data, meta_buffer, bv, to_encoding_version(hdr.encoding_version()));
         }
-
         ARCTICDB_TRACE(log::codec(), "Decoded encoded fields at position {}", data-begin);
         return meta_buffer.detach_buffer();
 }
@@ -563,7 +167,8 @@ std::optional<FieldCollection> decode_index_fields(
                        hdr.index_descriptor_field(),
                        data,
                        fields,
-                       bv);
+                       bv,
+                       to_encoding_version(hdr.encoding_version()));
 
         ARCTICDB_TRACE(log::codec(), "Decoded index descriptor to position {}", data-begin);
         return std::make_optional<FieldCollection>(std::move(fields));
@@ -594,7 +199,8 @@ std::optional<FieldCollection> decode_descriptor_fields(
                        hdr.descriptor_field(),
                        data,
                        fields,
-                       bv);
+                       bv,
+                       to_encoding_version(hdr.encoding_version()));
 
         ARCTICDB_TRACE(log::codec(), "Decoded descriptor fields to position {}", data-begin);
         return std::make_optional<FieldCollection>(std::move(fields));
@@ -611,7 +217,7 @@ std::optional<std::tuple<google::protobuf::Any, arcticdb::proto::descriptors::Ti
     util::check(data != nullptr, "Got null data ptr from segment");
     const auto has_magic_numbers = EncodingVersion(hdr.encoding_version()) == EncodingVersion::V2;
     if(has_magic_numbers)
-        check_magic<MetadataMagic>(data);
+        util::check_magic<MetadataMagic>(data);
 
     auto maybe_any = decode_metadata(hdr, data, begin);
     if(!maybe_any)
@@ -620,13 +226,13 @@ std::optional<std::tuple<google::protobuf::Any, arcticdb::proto::descriptors::Ti
     auto tsd = timeseries_descriptor_from_any(*maybe_any);
 
     if(has_magic_numbers)
-        check_magic<DescriptorMagic>(data);
+        util::check_magic<DescriptorMagic>(data);
 
     if(hdr.has_descriptor_field() && hdr.descriptor_field().has_ndarray())
         data += encoding_sizes::ndarray_field_compressed_size(hdr.descriptor_field().ndarray());
 
     if(has_magic_numbers)
-        check_magic<IndexMagic>(data);
+        util::check_magic<IndexMagic>(data);
 
     auto maybe_fields = decode_index_fields(hdr, data, begin, end);
     if(!maybe_fields) {
@@ -657,11 +263,11 @@ std::pair<std::optional<google::protobuf::Any>, StreamDescriptor> decode_metadat
     const uint8_t* end) {
     util::check(data != nullptr, "Got null data ptr from segment");
     if(EncodingVersion(hdr.encoding_version()) == EncodingVersion::V2)
-        check_magic<MetadataMagic>(data);
+        util::check_magic<MetadataMagic>(data);
 
     auto maybe_any = decode_metadata(hdr, data, begin);
     if(EncodingVersion(hdr.encoding_version()) == EncodingVersion::V2)
-        check_magic<DescriptorMagic>(data);
+        util::check_magic<DescriptorMagic>(data);
 
     auto maybe_fields = decode_descriptor_fields(hdr, data, begin, end);
     if(!maybe_fields) {
@@ -696,7 +302,8 @@ void decode_string_pool( const arcticdb::proto::encoding::SegmentHeader& hdr,
                        hdr.string_pool_field(),
                        data,
                        res.string_pool(),
-                       bv);
+                       bv,
+                       to_encoding_version(hdr.encoding_version()));
 
         ARCTICDB_TRACE(log::codec(), "Decoded string pool to position {}", data-begin);
     }
@@ -711,20 +318,20 @@ void decode_v2(const Segment& segment,
     const auto [begin, end] = get_segment_begin_end(segment, hdr);
     auto encoded_fields_ptr = end;
     auto data = begin;
-    check_magic<MetadataMagic>(data);
+    util::check_magic<MetadataMagic>(data);
     decode_metadata(hdr, data, begin, res);
     util::check(hdr.has_descriptor_field(), "Expected descriptor field in v2 encoding");
-    check_magic<DescriptorMagic>(data);
+    util::check_magic<DescriptorMagic>(data);
     if(hdr.has_descriptor_field() && hdr.descriptor_field().has_ndarray())
         data += encoding_sizes::field_compressed_size(hdr.descriptor_field());
 
-    check_magic<IndexMagic>(data);
+    util::check_magic<IndexMagic>(data);
     auto index_fields = decode_index_fields(hdr, data, begin, end);
     if(index_fields)
         res.set_index_fields(std::make_shared<FieldCollection>(std::move(*index_fields)));
 
     util::check(hdr.has_column_fields(), "Expected column fields in v2 encoding");
-    check_magic<EncodedMagic>(encoded_fields_ptr);
+    util::check_magic<EncodedMagic>(encoded_fields_ptr);
     if (data!=end) {
         auto encoded_fields_buffer = decode_encoded_fields(hdr, encoded_fields_ptr, begin);
         const auto fields_size = desc.fields().size();
@@ -741,7 +348,7 @@ void decode_v2(const Segment& segment,
             util::check(data!=end, "Reached end of input block with {} fields to decode", fields_size-i);
             if(auto col_index = res.column_index(field_name)) {
                 auto& col = res.column(static_cast<position_t>(*col_index));
-                data += decode_field(res.field(*col_index).type(), encoded_field, data, col, col.opt_sparse_map());
+                data += decode_field(res.field(*col_index).type(), encoded_field, data, col, col.opt_sparse_map(), to_encoding_version(hdr.encoding_version()));
             } else {
                 data += encoding_sizes::field_compressed_size(encoded_field) + sizeof(ColumnMagic);
             }
@@ -749,7 +356,7 @@ void decode_v2(const Segment& segment,
             ARCTICDB_TRACE(log::codec(), "Decoded column {} to position {}", i, data-begin);
         }
 
-        check_magic<StringPoolMagic>(data);
+        util::check_magic<StringPoolMagic>(data);
         decode_string_pool(hdr, data, begin, end, res);
 
         res.set_row_data(static_cast<ssize_t>(start_row + seg_row_count-1));
@@ -782,7 +389,7 @@ void decode_v1(const Segment& segment,
             util::check(data!=end, "Reached end of input block with {} fields to decode", fields_size-i);
             if(auto col_index = res.column_index(field_name)) {
                 auto& col = res.column(static_cast<position_t>(*col_index));
-                data += decode_field(res.field(*col_index).type(), field, data, col, col.opt_sparse_map());
+                data += decode_field(res.field(*col_index).type(), field, data, col, col.opt_sparse_map(), to_encoding_version(hdr.encoding_version()));
             } else
                 data += encoding_sizes::field_compressed_size(field);
 
@@ -825,4 +432,79 @@ SegmentInMemory decode_segment(Segment&& s) {
     return res;
 }
 
+static void hash_field(const arcticdb::proto::encoding::EncodedField &field, HashAccum &accum) {
+    auto &n = field.ndarray();
+    for(auto i = 0; i < n.shapes_size(); ++i) {
+        auto v = n.shapes(i).hash();
+        accum(&v);
+    }
+
+    for(auto j = 0; j < n.values_size(); ++j) {
+        auto v = n.values(j).hash();
+        accum(&v);
+    }
+}
+
+HashedValue hash_segment_header(const arcticdb::proto::encoding::SegmentHeader &hdr) {
+    HashAccum accum;
+    if (hdr.has_metadata_field()) {
+        hash_field(hdr.metadata_field(), accum);
+    }
+    for (int i = 0; i < hdr.fields_size(); ++i) {
+        hash_field(hdr.fields(i), accum);
+    }
+    if(hdr.has_string_pool_field()) {
+        hash_field(hdr.string_pool_field(), accum);
+    }
+    return accum.digest();
+}
+
+void add_bitmagic_compressed_size(
+    const ColumnData& column_data,
+    size_t& max_compressed_bytes,
+    size_t& uncompressed_bytes
+) {
+    if (column_data.bit_vector() != nullptr && column_data.bit_vector()->count() > 0)   {
+        bm::serializer<util::BitMagic>::statistics_type stat{};
+        column_data.bit_vector()->calc_stat(&stat);
+        uncompressed_bytes += stat.memory_used;
+        max_compressed_bytes += stat.max_serialize_mem;
+    }
+}
+
+/// @brief Write the sparse map to the out buffer
+/// Bitmagic achieves the theoretical best compression for booleans. Adding additional encoding (lz4, zstd, etc...)
+/// will not improve anything and in fact it might worsen the encoding.
+[[nodiscard]] static size_t encode_bitmap(const util::BitMagic& sparse_map, Buffer& out, std::ptrdiff_t& pos) {
+    ARCTICDB_DEBUG(log::version(), "Encoding sparse map of count: {}", sparse_map.count());
+    bm::serializer<bm::bvector<> > bvs;
+    bm::serializer<bm::bvector<> >::buffer sbuf;
+    bvs.serialize(sparse_map, sbuf);
+    auto sz = sbuf.size();
+    auto total_sz = sz + util::combined_bit_magic_delimiters_size();
+    out.assert_size(pos + total_sz);
+
+    uint8_t* target = out.data() + pos;
+    util::write_magic<util::BitMagicStart>(target);
+    std::memcpy(target, sbuf.data(), sz);
+    target += sz;
+    util::write_magic<util::BitMagicEnd>(target);
+    pos = pos + static_cast<ptrdiff_t>(total_sz);
+    return total_sz;
+}
+
+void encode_sparse_map(
+    ColumnData& column_data,
+    std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+    Buffer& out,
+    std::ptrdiff_t& pos
+) {
+    if (column_data.bit_vector() != nullptr && column_data.bit_vector()->count() > 0)   {
+        ARCTICDB_DEBUG(log::codec(), "Sparse map count = {} pos = {}", column_data.bit_vector()->count(), pos);
+        const size_t sparse_bm_bytes = encode_bitmap(*column_data.bit_vector(), out, pos);
+        util::variant_match(variant_field, [&](auto field) {
+            field->mutable_ndarray()->set_sparse_map_bytes(static_cast<int>(sparse_bm_bytes));
+        });
+    }
+}
 } // namespace arcticdb
diff --git a/cpp/arcticdb/codec/codec.hpp b/cpp/arcticdb/codec/codec.hpp
index a22aada525..32269ab858 100644
--- a/cpp/arcticdb/codec/codec.hpp
+++ b/cpp/arcticdb/codec/codec.hpp
@@ -7,128 +7,26 @@
 
 #pragma once
 
-#include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/buffer.hpp>
-
+#include <arcticdb/codec/core.hpp>
 #include <arcticdb/codec/segment.hpp>
-#include <arcticdb/util/pb_util.hpp>
-#include <arcticdb/util/hash.hpp>
-#include <arcticdb/column_store/column_data.hpp>
 #include <arcticdb/column_store/memory_segment.hpp>
-
-#include <cstdlib>
-#include <cstdint>
-#include <memory>
-#include <variant>
+#include <arcticdb/entity/types.hpp>
 
 namespace arcticdb {
 
-using DescriptorMagic = util::MagicNum<'D','e','s','c'>;
-using EncodedMagic = util::MagicNum<'E','n','c','d'>;
-using StringPoolMagic = util::MagicNum<'S','t','r','p'>;
-using MetadataMagic = util::MagicNum<'M','e','t','a'>;
-using IndexMagic = util::MagicNum<'I','n','d','x'>;
-using ColumnMagic = util::MagicNum<'C','l','m','n'>;
-
-template<typename MagicType>
-void check_magic_in_place(const uint8_t* data) {
-    const auto magic = reinterpret_cast<const MagicType*>(data);
-    magic->check();
-}
-
-template<typename MagicType>
-void check_magic(const uint8_t*& data) {
-    check_magic_in_place<MagicType>(data);
-    data += sizeof(MagicType);
-}
-
-template<typename MagicType>
-void write_magic(Buffer& buffer, std::ptrdiff_t& pos) {
-    new (buffer.data() + pos) MagicType{};
-    pos += sizeof(MagicType);
-}
-
-//TODO  Provide tuning parameters to choose the encoding through encoding tags (also acting as config options)
-template<template<typename> class F, class TD>
-struct TypedBlockEncoderImpl;
-
-template<template<typename> class F, class TD>
-struct BlockEncoderHelper : public TypedBlockEncoderImpl<F, TD> {
-    using FieldType = F<TD>;
-    using TypeDescriptorTag = TD;
-    using DimTag = typename TD::DimensionTag;
-    using DataTag = typename TD::DataTypeTag;
-};
-
-size_t encode_bitmap(
-    const util::BitMagic &sparse_map,
-    Buffer &out,
-    std::ptrdiff_t &pos);
-
-template<typename TD>
-using BlockEncoder = BlockEncoderHelper<TypedBlockData, TD>;
-
-struct ColumnEncoder {
-    static std::pair<size_t, size_t> max_compressed_size(
-        const arcticdb::proto::encoding::VariantCodec &codec_opts,
-        ColumnData &column_data);
+using ShapesBlockTDT = TypeDescriptorTag<DataTypeTag<DataType::INT64>, DimensionTag<Dimension::Dim0>>;
 
-    template <typename EncodedFieldType>
-    void encode(
-        const arcticdb::proto::encoding::VariantCodec &codec_opts,
-        ColumnData &column_data,
-        EncodedFieldType &field,
-        Buffer &out,
-        std::ptrdiff_t &pos) {
-        column_data.type().visit_tag([&codec_opts, &column_data, &field, &out, &pos](auto type_desc_tag) {
-            using TDT = decltype(type_desc_tag);
-            using Encoder = BlockEncoder<TDT>;
-            ARCTICDB_TRACE(log::codec(), "Column data has {} blocks", column_data.num_blocks());
-
-            while (auto block = column_data.next<TDT>()) {
-                if constexpr(!is_empty_type(TDT::DataTypeTag::data_type)) {
-                    util::check(block->nbytes() > 0, "Zero-sized block");
-                }
-                Encoder::encode(codec_opts, *block, field, out, pos);
-            }
-        });
-
-        if (column_data.bit_vector() != nullptr && column_data.bit_vector()->count() > 0)   {
-            ARCTICDB_DEBUG(log::codec(), "Sparse map count = {} pos = {}", column_data.bit_vector()->count(), pos);
-            auto sparse_bm_bytes = encode_bitmap(*column_data.bit_vector(), out, pos);
-            field.mutable_ndarray()->set_sparse_map_bytes(static_cast<int>(sparse_bm_bytes));
-        }
-    }
-};
-
-
-Segment encode_v2(
-    SegmentInMemory&& in_mem_seg,
-    const arcticdb::proto::encoding::VariantCodec &codec_opts);
-
-Segment encode_v1(
-    SegmentInMemory&& in_mem_seg,
-    const arcticdb::proto::encoding::VariantCodec &codec_opts);
-
-
-inline Segment encode_dispatch(
+Segment encode_dispatch(
     SegmentInMemory&& in_mem_seg,
     const arcticdb::proto::encoding::VariantCodec &codec_opts,
-    EncodingVersion encoding_version) {
-    if(encoding_version == EncodingVersion::V2) {
-        return encode_v2(std::move(in_mem_seg), codec_opts);
-    } else {
-        return encode_v1(std::move(in_mem_seg), codec_opts);
-    }
-}
+    EncodingVersion encoding_version);
 
 Buffer decode_encoded_fields(
     const arcticdb::proto::encoding::SegmentHeader& hdr,
     const uint8_t* data,
     const uint8_t* begin ARCTICDB_UNUSED);
 
-SegmentInMemory decode_segment(
-    Segment&& segment);
+SegmentInMemory decode_segment(Segment&& segment);
 
 void decode_into_memory_segment(
     const Segment& segment,
@@ -142,7 +40,8 @@ std::size_t decode_field(
     const EncodedFieldType &field,
     const uint8_t *input,
     DataSink &data_sink,
-    std::optional<util::BitMagic>& bv);
+    std::optional<util::BitMagic>& bv,
+    arcticdb::EncodingVersion encoding_version);
 
 std::optional<google::protobuf::Any> decode_metadata_from_segment(
     const Segment& segment);
@@ -153,32 +52,7 @@ std::pair<std::optional<google::protobuf::Any>, StreamDescriptor> decode_metadat
 std::optional<std::tuple<google::protobuf::Any, arcticdb::proto::descriptors::TimeSeriesDescriptor, FieldCollection>> decode_timeseries_descriptor(
     Segment& segment);
 
-inline void hash_field(const arcticdb::proto::encoding::EncodedField &field, HashAccum &accum) {
-    auto &n = field.ndarray();
-    for(auto i = 0; i < n.shapes_size(); ++i) {
-        auto v = n.shapes(i).hash();
-        accum(&v);
-    }
-
-    for(auto j = 0; j < n.values_size(); ++j) {
-        auto v = n.values(j).hash();
-        accum(&v);
-    }
-}
-
-inline HashedValue hash_segment_header(const arcticdb::proto::encoding::SegmentHeader &hdr) {
-    HashAccum accum;
-    if (hdr.has_metadata_field()) {
-        hash_field(hdr.metadata_field(), accum);
-    }
-    for (int i = 0; i < hdr.fields_size(); ++i) {
-        hash_field(hdr.fields(i), accum);
-    }
-    if(hdr.has_string_pool_field()) {
-        hash_field(hdr.string_pool_field(), accum);
-    }
-    return accum.digest();
-}
+HashedValue hash_segment_header(const arcticdb::proto::encoding::SegmentHeader &hdr);
 } // namespace arcticdb
 
 #define ARCTICDB_SEGMENT_ENCODER_H_
diff --git a/cpp/arcticdb/codec/core.hpp b/cpp/arcticdb/codec/core.hpp
index 064caad662..5462c95300 100644
--- a/cpp/arcticdb/codec/core.hpp
+++ b/cpp/arcticdb/codec/core.hpp
@@ -7,6 +7,8 @@
 
 #pragma once
 
+
+#include <arcticdb/codec/encoded_field.hpp>
 #include <arcticdb/entity/protobufs.hpp>
 #include <arcticdb/util/buffer.hpp>
 #include <arcticdb/util/hash.hpp>
@@ -171,17 +173,24 @@ struct ShapeEncodingFromBlock {
  *  using Opts = VariantCodec::SomeCodecOptionType;
  * }
  * @tparam SE ShapeEncoding Same as VE but used for shape encoding. By default uses VE.
+ *
+ * @todo Try to replace this with arcticdb::detail::GenericBlockEncoder2
  */
-template<class BlockType, class TD, class EncoderType, class SE = ShapeEncodingFromBlock<EncoderType, EncoderType>>
+template<class BlockType, class TD, class EncoderType>
 struct GenericBlockEncoder {
     using Helper = CodecHelper<TD>;
     using T = typename Helper::T;
-    using ShapeEncoding = SE;
+    using ShapeEncoding = ShapeEncodingFromBlock<EncoderType, EncoderType>;
 
     GenericBlockEncoder() = delete;
     GenericBlockEncoder(GenericBlockEncoder &&enc) = delete;
 
-    static size_t max_compressed_size(const BlockType &block) {
+    static size_t max_compressed_size(const BlockType& block) {
+        if(block.nbytes() == 0) {
+            ARCTICDB_TRACE(log::codec(), "GenericBlockEncoder got empty block. Max compressed size is 0.");
+            return 0;
+        }
+
         Helper helper;
         helper.hasher_.reset(helper.seed);
         std::size_t block_row_count = block.row_count();
@@ -208,14 +217,19 @@ struct GenericBlockEncoder {
     template <typename EncodedFieldType>
     static void encode(
         const typename EncoderType::Opts &opts,
-        BlockType &block,
-        EncodedFieldType &field,
-        Buffer &out,
-        std::ptrdiff_t &pos
-        ) {
+        const BlockType& block,
+        EncodedFieldType& field,
+        Buffer& out,
+        std::ptrdiff_t& pos
+    ) {
         Helper helper;
         helper.hasher_.reset(helper.seed);
-        std::size_t block_row_count = block.row_count();
+        const std::size_t block_row_count = block.row_count();
+        auto *field_nd_array = field.mutable_ndarray();
+        if(block.nbytes() == 0) {
+            ARCTICDB_TRACE(log::codec(), "GenericBlockEncoder got empty block. There's nothing to encode");
+            return;
+        }
 
         if constexpr (Helper::dim == Dimension::Dim0) {
             // Only store data, no shapes since dimension is 0
@@ -229,35 +243,38 @@ struct GenericBlockEncoder {
             // doing copy + hash in one pass, this might have a negative effect on perf
             // since the hashing is path dependent. This is a toy example though so not critical
             auto t_out = reinterpret_cast<T *>(out.data() + pos);
-            auto *field_nd_array = field.mutable_ndarray();
             const auto total_items_count = field_nd_array->items_count() + block_row_count;
             field_nd_array->set_items_count(total_items_count);
             auto value_pb = field_nd_array->add_values();
-
-           const auto compressed_size = EncoderType::encode_block(opts, block.data(), helper_scalar_block, helper.hasher_, t_out,
-                                                           max_compressed_size, pos, *value_pb->mutable_codec());
-
+            const auto compressed_size = EncoderType::encode_block(opts,
+                block.data(),
+                helper_scalar_block,
+                helper.hasher_,
+                t_out,
+                max_compressed_size,
+                pos,
+                *value_pb->mutable_codec());
             helper_scalar_block.set_block_data(*value_pb, helper.hasher_.digest(), compressed_size);
             helper_scalar_block.set_version(*value_pb, EncoderType::VERSION);
         } else {
             auto helper_array_block = Helper::nd_array_block(block_row_count, block.shapes());
 
             ARCTICDB_TRACE(log::codec(), "Generic block encoder writing ndarray field of {} items", helper_array_block.item_count_);
-            std::size_t comp_data = EncoderType::max_compressed_size(helper_array_block.values_.bytes_);
-            std::size_t comp_shapes = ShapeEncoding::max_compressed_size(helper_array_block.shapes_.bytes_);
+            const std::size_t max_compressed_data_size = EncoderType::max_compressed_size(helper_array_block.values_.bytes_);
+            const std::size_t max_compressed_shapes_size = ShapeEncoding::max_compressed_size(helper_array_block.shapes_.bytes_);
+            const std::size_t helper_buffer_size = max_compressed_data_size + max_compressed_shapes_size;
 
-            helper.ensure_buffer(out, pos, comp_data + comp_shapes);
-            auto field_nd_array = field.mutable_ndarray();
+            helper.ensure_buffer(out, pos, helper_buffer_size);
             auto shape_pb = field_nd_array->add_shapes();
 
             // write shapes
-            auto s_out = reinterpret_cast<shape_t *>(out.data() + pos);
+            auto s_out = reinterpret_cast<shape_t*>(out.data() + pos);
             const auto shape_comp_size = ShapeEncoding::encode_block(
                 block.shapes(),
                 helper_array_block.shapes_,
                 helper.hasher_,
                 s_out,
-                comp_shapes,
+                max_compressed_shapes_size,
                 pos,
                 *shape_pb->mutable_codec());
 
@@ -266,14 +283,15 @@ struct GenericBlockEncoder {
             // write values
             auto value_pb = field_nd_array->add_values();
             auto t_out = reinterpret_cast<T *>(out.data() + pos);
-            const auto values_comp_size = EncoderType::encode_block(opts,
-                                                                        block.data(),
-                                                                        helper_array_block.values_,
-                                                                        helper.hasher_,
-                                                                        t_out,
-                                                                        comp_data,
-                                                                        pos,
-                                                                        *value_pb->mutable_codec());
+            const auto values_comp_size = EncoderType::encode_block(
+                opts,
+                block.data(),
+                helper_array_block.values_,
+                helper.hasher_,
+                t_out,
+                max_compressed_data_size,
+                pos,
+                *value_pb->mutable_codec());
 
             auto digest = helper.hasher_.digest();
             helper_array_block.update_field_size(*field_nd_array);
@@ -283,4 +301,66 @@ struct GenericBlockEncoder {
     }
 };
 
+/// @brief Encode a block based on encoding options.
+/// Just as arcticdb::detail::struct GenericBlockEncoder this class does not care about the specific content of the
+/// block and does not take advantage of specific patterns of data. The main difference between this and
+/// arcticdb::detail::struct GenericBlockEncoder is that the latter stores the shapes data at the beginning of each
+/// block, which is suboptimal. arcticdb::detail::GenericBlockEncoder2 does not care about dimensionality and
+/// does not encode the shapes of the block. For more information see comment above arcticdb::ColumnEncoder2
+template<class BlockType, class TD, class EncoderType>
+struct GenericBlockEncoderV2 {
+public:
+    using Helper = CodecHelper<TD>;
+    using T = typename Helper::T;
+
+    static size_t max_compressed_size(const BlockType &block) {
+        const auto uncompressed_size = block.nbytes();
+        if(uncompressed_size == 0) {
+            ARCTICDB_TRACE(log::codec(), "GenericBlockEncoderV2 got empty block. Max compressed size is 0.");
+            return 0;
+        }
+        const auto compressed = EncoderType::max_compressed_size(uncompressed_size);
+        ARCTICDB_TRACE(log::codec(), "Scalar block has {} bytes", compressed);
+        return compressed;
+    }
+
+    template <typename EncodedBlockType>
+    static void encode(
+        const typename EncoderType::Opts &opts,
+        const BlockType& block,
+        Buffer& out,
+        std::ptrdiff_t& pos,
+        EncodedBlockType* encoded_block
+    ) {
+        if(block.nbytes() == 0) {
+            ARCTICDB_TRACE(log::codec(), "GenericBlockEncoderV2 got empty block. There's nothing to encode.");
+            return;
+        }
+
+        Helper helper;
+        helper.hasher_.reset(helper.seed);
+        auto helper_scalar_block = BlockProtobufHelper{block.nbytes() / sizeof(T), block.nbytes()};
+        ARCTICDB_TRACE(log::codec(), "Generic block encode writing scalar of {} elements", block.row_count());
+
+        const auto uncompressed_size = helper_scalar_block.bytes_;
+        const std::size_t max_compressed_size = EncoderType::max_compressed_size(uncompressed_size);
+        helper.ensure_buffer(out, pos, max_compressed_size);
+
+        // doing copy + hash in one pass, this might have a negative effect on perf
+        // since the hashing is path dependent. This is a toy example though so not critical
+        auto t_out = reinterpret_cast<T*>(out.data() + pos);
+        const auto compressed_size = EncoderType::encode_block(
+            opts,
+            block.data(),
+            helper_scalar_block,
+            helper.hasher_,
+            t_out,
+            max_compressed_size,
+            pos,
+            *encoded_block->mutable_codec());
+        helper_scalar_block.set_block_data(*encoded_block, helper.hasher_.digest(), compressed_size);
+        helper_scalar_block.set_version(*encoded_block, EncoderType::VERSION);
+    }
+};
+
 } // namespace arcticdb::detail
diff --git a/cpp/arcticdb/codec/default_codecs.hpp b/cpp/arcticdb/codec/default_codecs.hpp
index 62ec7110a4..aa7b01f0cc 100644
--- a/cpp/arcticdb/codec/default_codecs.hpp
+++ b/cpp/arcticdb/codec/default_codecs.hpp
@@ -17,10 +17,15 @@ inline arcticdb::proto::encoding::VariantCodec default_lz4_codec() {
     lz4ptr->set_acceleration(1);
     return codec;
 }
+
 inline arcticdb::proto::encoding::VariantCodec default_passthrough_codec() {
     arcticdb::proto::encoding::VariantCodec codec;
     auto lz4ptr = codec.mutable_passthrough();
     lz4ptr->set_mark(true);
     return codec;
 }
+
+inline arcticdb::proto::encoding::VariantCodec default_shapes_codec() {
+    return codec::default_lz4_codec();
+}
 }
diff --git a/cpp/arcticdb/codec/encode_common.hpp b/cpp/arcticdb/codec/encode_common.hpp
new file mode 100644
index 0000000000..55f613dffd
--- /dev/null
+++ b/cpp/arcticdb/codec/encode_common.hpp
@@ -0,0 +1,213 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+/// @file encode_common.hpp Functions and classes used by both V1 and V2 encodings
+
+#pragma once
+
+#include <arcticdb/codec/segment.hpp>
+#include <arcticdb/entity/types.hpp>
+#include <arcticdb/entity/protobufs.hpp>
+#include <arcticdb/column_store/memory_segment.hpp>
+#include <arcticdb/codec/default_codecs.hpp>
+
+#include <cstddef>
+
+namespace arcticdb {
+
+    /// @brief This should be the block data type descriptor when the shapes array is encoded as a block
+    using ShapesBlockTDT = TypeDescriptorTag<DataTypeTag<DataType::INT64>, DimensionTag<Dimension::Dim0>>;
+
+    using ByteArrayTDT = TypeDescriptorTag<DataTypeTag<DataType::UINT8>, DimensionTag<Dimension::Dim1>>;
+
+    template<template<typename> class TypedBlock, class TD, EncodingVersion encoder_version>
+    struct TypedBlockEncoderImpl;
+
+    template<EncodingVersion v, typename VersionedColumnEncoder>
+    struct EncodingPolicyType {
+        static constexpr EncodingVersion version = v;
+        using ColumnEncoder = VersionedColumnEncoder;
+    };
+
+    template<typename EncodingPolicyType>
+    struct BytesEncoder {
+        using Encoder = TypedBlockEncoderImpl<TypedBlockData, ByteArrayTDT, EncodingPolicyType::version>;
+        using BytesBlock = TypedBlockData<ByteArrayTDT>;
+        using ShapesEncoder = TypedBlockEncoderImpl<TypedBlockData, ShapesBlockTDT, EncodingPolicyType::version>;
+
+        template<typename EncodedFieldType>
+        static void encode(
+            const ChunkedBuffer& data,
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            Buffer& out_buffer,
+            std::ptrdiff_t& pos,
+            EncodedFieldType* encoded_field
+        ) {
+            if constexpr(EncodingPolicyType::version == EncodingVersion::V1) {
+                const auto bytes_count = static_cast<shape_t>(data.bytes());
+                auto typed_block = BytesBlock(
+                    data.data(),
+                    &bytes_count,
+                    bytes_count,
+                    1u,
+                    data.block_and_offset(0).block_);
+                Encoder::encode(codec_opts, typed_block, *encoded_field, out_buffer, pos);
+            } else if constexpr(EncodingPolicyType::version == EncodingVersion::V2) {
+                // On Man's Mac build servers size_t and ssize_t are long rather than long long but the shape TDT
+                // expects int64 (long long).
+                const size_t row_count = 1;
+                const auto shapes_data = static_cast<ShapesBlockTDT::DataTypeTag::raw_type>(data.bytes());
+                auto shapes_block = TypedBlockData<ShapesBlockTDT>(&shapes_data,
+                    nullptr,
+                    sizeof(shape_t),
+                    row_count,
+                    data.block_and_offset(0).block_);
+                const auto bytes_count = static_cast<shape_t>(data.bytes());
+                auto data_block = BytesBlock(data.data(),
+                    &bytes_count,
+                    static_cast<shape_t>(bytes_count),
+                    row_count,
+                    data.block_and_offset(0).block_);
+                ShapesEncoder::encode_shapes(codec::default_shapes_codec(), shapes_block, *encoded_field, out_buffer, pos);
+                Encoder::encode_values(codec_opts, data_block, *encoded_field, out_buffer, pos);
+                auto* field_nd_array = encoded_field->mutable_ndarray();
+                const auto total_items_count = field_nd_array->items_count() + row_count;
+                field_nd_array->set_items_count(total_items_count);
+            } else {
+                static_assert(std::is_same_v<decltype(EncodingPolicyType::version), void>, "Unknown encoding version");
+            }
+        }
+
+        static size_t
+        max_compressed_size(const arcticdb::proto::encoding::VariantCodec& codec_opts, shape_t data_size) {
+            const shape_t shapes_bytes = sizeof(shape_t);
+            const auto values_block = BytesBlock(data_size, &data_size);
+            if constexpr(EncodingPolicyType::version == EncodingVersion::V1) {
+                const auto shapes_block = BytesBlock(shapes_bytes, &shapes_bytes);
+                return Encoder::max_compressed_size(codec_opts, values_block) +
+                       Encoder::max_compressed_size(codec_opts, shapes_block);
+            } else if constexpr(EncodingPolicyType::version == EncodingVersion::V2) {
+                const auto shapes_block = TypedBlockData<ShapesBlockTDT>(shapes_bytes, &shapes_bytes);
+                return Encoder::max_compressed_size(codec_opts, values_block) +
+                       ShapesEncoder::max_compressed_size(codec::default_shapes_codec(), shapes_block);
+            } else {
+                static_assert(std::is_same_v<decltype(EncodingPolicyType::version), void>, "Unknown encoding version");
+            }
+        }
+    };
+
+    struct SizeResult {
+        size_t max_compressed_bytes_;
+        size_t uncompressed_bytes_;
+        shape_t encoded_blocks_bytes_;
+    };
+
+    template<typename EncodingPolicyType>
+    void calc_metadata_size(
+        const SegmentInMemory& in_mem_seg,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        SizeResult& result
+    ) {
+        if(in_mem_seg.metadata()) {
+            const auto metadata_bytes = static_cast<shape_t>(in_mem_seg.metadata()->ByteSizeLong());
+            result.uncompressed_bytes_ += metadata_bytes + sizeof(shape_t);
+            result.max_compressed_bytes_ += BytesEncoder<EncodingPolicyType>::max_compressed_size(codec_opts, metadata_bytes);
+            ARCTICDB_TRACE(log::codec(), "Metadata requires {} max_compressed_bytes", result.max_compressed_bytes_);
+        }
+    }
+
+    template<typename EncodingPolicyType>
+    void calc_columns_size(
+        const SegmentInMemory& in_mem_seg,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        SizeResult& result
+    ) {
+        for (std::size_t c = 0; c < in_mem_seg.num_columns(); ++c) {
+            auto column_data = in_mem_seg.column_data(c);
+            const auto [uncompressed, required] = EncodingPolicyType::ColumnEncoder::max_compressed_size(codec_opts,
+                column_data);
+            result.uncompressed_bytes_ += uncompressed;
+            result.max_compressed_bytes_ += required;
+            ARCTICDB_TRACE(log::codec(),
+                "Column {} requires {} max_compressed_bytes, total {}",
+                c,
+                required,
+                result.max_compressed_bytes_);
+        }
+    }
+
+    template<typename EncodingPolicyType>
+    void calc_string_pool_size(
+        const SegmentInMemory& in_mem_seg,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        SizeResult& result
+    ) {
+        if(in_mem_seg.has_string_pool()) {
+            auto string_col = in_mem_seg.string_pool_data();
+            const auto [uncompressed, required] = EncodingPolicyType::ColumnEncoder::max_compressed_size(codec_opts,
+                string_col);
+            result.uncompressed_bytes_ += uncompressed;
+            result.max_compressed_bytes_ += required;
+            ARCTICDB_TRACE(log::codec(),
+                "String pool requires {} max_compressed_bytes, total {}",
+                required,
+                result.max_compressed_bytes_);
+        }
+    }
+
+    template<typename EncodingPolicyType>
+    void encode_metadata(
+        const SegmentInMemory& in_mem_seg,
+        arcticdb::proto::encoding::SegmentHeader& segment_header,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        Buffer& out_buffer,
+        std::ptrdiff_t& pos
+    ) {
+        if(in_mem_seg.metadata()) {
+            const auto bytes_count = static_cast<shape_t>(in_mem_seg.metadata()->ByteSizeLong());
+            ARCTICDB_TRACE(log::codec(), "Encoding {} bytes of metadata", bytes_count);
+            auto encoded_field = segment_header.mutable_metadata_field();
+
+            constexpr int max_stack_alloc = 1 << 11;
+            bool malloced{false};
+            uint8_t* meta_ptr{nullptr};
+            if(bytes_count > max_stack_alloc) {
+                meta_ptr = reinterpret_cast<uint8_t*>(malloc(bytes_count));
+                malloced = true;
+            } else {
+                meta_ptr = reinterpret_cast<uint8_t*>(alloca(bytes_count));
+            }
+            ChunkedBuffer meta_buffer;
+            meta_buffer.add_external_block(meta_ptr, bytes_count, 0u);
+            google::protobuf::io::ArrayOutputStream aos(&meta_buffer[0], static_cast<int>(bytes_count));
+            in_mem_seg.metadata()->SerializeToZeroCopyStream(&aos);
+            BytesEncoder<EncodingPolicyType>::encode(meta_buffer, codec_opts, out_buffer, pos, encoded_field);
+            ARCTICDB_DEBUG(log::codec(), "Encoded metadata to position {}", pos);
+            if(malloced)
+                free(meta_ptr);
+        } else {
+            ARCTICDB_DEBUG(log::codec(), "Not encoding any metadata");
+        }
+    }
+
+    template<typename EncodingPolicyType>
+    void encode_string_pool(
+        const SegmentInMemory& in_mem_seg,
+        arcticdb::proto::encoding::SegmentHeader& segment_header,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        Buffer& out_buffer,
+        std::ptrdiff_t& pos
+    ) {
+        if(in_mem_seg.has_string_pool()) {
+            ARCTICDB_TRACE(log::codec(), "Encoding string pool to position {}", pos);
+            auto* encoded_field = segment_header.mutable_string_pool_field();
+            auto col = in_mem_seg.string_pool_data();
+            EncodingPolicyType::ColumnEncoder::encode(codec_opts, col, encoded_field, out_buffer, pos);
+            ARCTICDB_TRACE(log::codec(), "Encoded string pool to position {}", pos);
+        }
+    }
+}
diff --git a/cpp/arcticdb/codec/encode_v1.cpp b/cpp/arcticdb/codec/encode_v1.cpp
new file mode 100644
index 0000000000..707e38515a
--- /dev/null
+++ b/cpp/arcticdb/codec/encode_v1.cpp
@@ -0,0 +1,152 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+#include <arcticdb/codec/encode_common.hpp>
+#include <arcticdb/codec/typed_block_encoder_impl.hpp>
+#include <arcticdb/column_store/memory_segment.hpp>
+
+namespace arcticdb {
+    void add_bitmagic_compressed_size(
+        const ColumnData& column_data,
+        size_t& max_compressed_bytes,
+        size_t& uncompressed_bytes
+    );
+
+    void encode_sparse_map(
+        ColumnData& column_data,
+        std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+        Buffer& out,
+        std::ptrdiff_t& pos
+    );
+
+    /// @brief Utility class used to encode and compute the max encoding size for regular data columns for V1 encoding
+    struct ColumnEncoderV1 {
+        static std::pair<size_t, size_t> max_compressed_size(
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            ColumnData& column_data);
+
+        static void encode(
+            const arcticdb::proto::encoding::VariantCodec &codec_opts,
+            ColumnData& column_data,
+            std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+            Buffer& out,
+            std::ptrdiff_t& pos);
+    };
+
+    std::pair<size_t, size_t> ColumnEncoderV1::max_compressed_size(
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        ColumnData& column_data
+    ) {
+        return column_data.type().visit_tag([&codec_opts, &column_data](auto type_desc_tag) {
+            size_t max_compressed_bytes = 0;
+            size_t uncompressed_bytes = 0;
+            using TDT = decltype(type_desc_tag);
+            using Encoder = TypedBlockEncoderImpl<TypedBlockData, TDT, EncodingVersion::V1>;
+            ARCTICDB_TRACE(log::codec(), "Column data has {} blocks", column_data.num_blocks());
+            while (auto block = column_data.next<TDT>()) {
+                const auto nbytes = block.value().nbytes();
+                if constexpr(must_contain_data(static_cast<TypeDescriptor>(type_desc_tag))) {
+                    util::check(nbytes > 0, "Zero-sized block");
+                }
+                uncompressed_bytes += nbytes;
+                // For the empty type the column will contain 0 size of user data however the encoder might need add some
+                // encoder specific data to the buffer, thus the uncompressed size will be 0 but the max_compressed_bytes
+                // might be non-zero.
+                max_compressed_bytes += Encoder::max_compressed_size(codec_opts, *block);
+            }
+            add_bitmagic_compressed_size(column_data, uncompressed_bytes, max_compressed_bytes);
+            return std::make_pair(uncompressed_bytes, max_compressed_bytes);
+        });
+    }
+
+    void ColumnEncoderV1::encode(
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        ColumnData& column_data,
+        std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+        Buffer& out,
+        std::ptrdiff_t& pos
+    ) {
+        column_data.type().visit_tag([&](auto type_desc_tag) {
+            using TDT = decltype(type_desc_tag);
+            using Encoder = TypedBlockEncoderImpl<TypedBlockData, TDT, EncodingVersion::V1>;
+            ARCTICDB_TRACE(log::codec(), "Column data has {} blocks", column_data.num_blocks());
+            while (auto block = column_data.next<TDT>()) {
+                if constexpr(must_contain_data(static_cast<TypeDescriptor>(type_desc_tag))) {
+                    util::check(block.value().nbytes() > 0, "Zero-sized block");
+                }
+                std::visit([&](auto field){
+                    Encoder::encode(codec_opts, block.value(), *field, out, pos);
+                }, variant_field);
+            }
+        });
+        encode_sparse_map(column_data, variant_field, out, pos);
+    }
+
+    using EncodingPolicyV1 = EncodingPolicyType<EncodingVersion::V1, ColumnEncoderV1>;
+
+    [[nodiscard]] static SizeResult max_compressed_size_v1(
+        const SegmentInMemory& in_mem_seg,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts
+    ) {
+        ARCTICDB_SAMPLE(GetSegmentCompressedSize, 0)
+        SizeResult result{};
+        calc_metadata_size<EncodingPolicyV1>(in_mem_seg, codec_opts, result);
+
+        if(in_mem_seg.row_count() > 0) {
+            calc_columns_size<EncodingPolicyV1>(in_mem_seg, codec_opts, result);
+            calc_string_pool_size<EncodingPolicyV1>(in_mem_seg, codec_opts, result);
+        }
+        ARCTICDB_TRACE(log::codec(), "Max compressed size {}", result.max_compressed_bytes_);
+        return result;
+    }
+
+    [[nodiscard]] Segment encode_v1(SegmentInMemory&& s, const arcticdb::proto::encoding::VariantCodec &codec_opts) {
+        /*
+         * This takes an in memory segment with all the metadata, column tensors etc., loops through each column
+         * and based on the type of the column, calls the typed block encoder for that column.
+         */
+        ARCTICDB_SAMPLE(EncodeSegment, 0)
+        auto in_mem_seg = std::move(s);
+        auto arena = std::make_unique<google::protobuf::Arena>();
+        auto segment_header = google::protobuf::Arena::CreateMessage<arcticdb::proto::encoding::SegmentHeader>(arena.get());
+        *segment_header->mutable_stream_descriptor() = in_mem_seg.descriptor().copy_to_proto();
+        segment_header->set_compacted(in_mem_seg.compacted());
+        std::ptrdiff_t pos = 0;
+        static auto block_to_header_ratio = ConfigsMap::instance()->get_int("Codec.EstimatedHeaderRatio", 75);
+        const auto preamble = in_mem_seg.num_blocks() * block_to_header_ratio;
+        auto [max_compressed_size, uncompressed_size, encoded_blocks_bytes] = max_compressed_size_v1(in_mem_seg, codec_opts);
+        ARCTICDB_TRACE(log::codec(), "Estimated max buffer requirement: {}", max_compressed_size);
+        auto out_buffer = std::make_shared<Buffer>(max_compressed_size, preamble);
+        ColumnEncoderV1 encoder;
+
+        ARCTICDB_TRACE(log::codec(), "Encoding descriptor: {}", segment_header->stream_descriptor().DebugString());
+        auto *tsd = segment_header->mutable_stream_descriptor();
+        tsd->set_in_bytes(uncompressed_size);
+
+        encode_metadata<EncodingPolicyV1>(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
+
+        if(in_mem_seg.row_count() > 0) {
+            ARCTICDB_TRACE(log::codec(), "Encoding fields");
+            for (std::size_t column_index = 0; column_index < in_mem_seg.num_columns(); ++column_index) {
+                auto column_data = in_mem_seg.column_data(column_index);
+                auto *encoded_field = segment_header->mutable_fields()->Add();
+                encoder.encode(codec_opts, column_data, encoded_field, *out_buffer, pos);
+                ARCTICDB_TRACE(log::codec(), "Encoded column {}: ({}) to position {}", column_index, in_mem_seg.descriptor().fields(column_index).name(), pos);
+            }
+            encode_string_pool<EncodingPolicyV1>(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
+        }
+        ARCTICDB_DEBUG(log::codec(), "Setting buffer bytes to {}", pos);
+        out_buffer->set_bytes(pos);
+        tsd->set_out_bytes(pos);
+        ARCTICDB_DEBUG(log::codec(), "Encoded header: {}", tsd->DebugString());
+        if(!segment_header->has_metadata_field())
+            ARCTICDB_DEBUG(log::codec(), "No metadata field");
+        ARCTICDB_DEBUG(log::codec(), "Block count {} header size {} ratio {}",
+            in_mem_seg.num_blocks(), segment_header->ByteSizeLong(),
+            in_mem_seg.num_blocks() ? segment_header->ByteSizeLong() / in_mem_seg.num_blocks() : 0);
+        return {std::move(arena), segment_header, std::move(out_buffer), in_mem_seg.descriptor().fields_ptr()};
+    }
+}
diff --git a/cpp/arcticdb/codec/encode_v2.cpp b/cpp/arcticdb/codec/encode_v2.cpp
new file mode 100644
index 0000000000..3a0cfee463
--- /dev/null
+++ b/cpp/arcticdb/codec/encode_v2.cpp
@@ -0,0 +1,343 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+#include <arcticdb/codec/encoded_field.hpp>
+#include <arcticdb/codec/encode_common.hpp>
+#include <arcticdb/codec/typed_block_encoder_impl.hpp>
+#include <arcticdb/codec/magic_words.hpp>
+#include <arcticdb/column_store/memory_segment.hpp>
+
+
+namespace arcticdb {
+    void add_bitmagic_compressed_size(
+        const ColumnData& column_data,
+        size_t& max_compressed_bytes,
+        size_t& uncompressed_bytes
+    );
+
+    void encode_sparse_map(
+        ColumnData& column_data,
+        std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+        Buffer& out,
+        std::ptrdiff_t& pos
+    );
+
+    template<typename MagicType>
+    static void write_magic(Buffer& buffer, std::ptrdiff_t& pos) {
+        new (buffer.data() + pos) MagicType{};
+        pos += sizeof(MagicType);
+    }
+
+    /// @brief Utility class used to encode and compute the max encoding size for regular data columns for V2 encoding
+    /// What differs this from the already existing ColumnEncoder is that ColumnEncoder encodes the shapes of
+    /// multidimensional data as part of each block. ColumnEncoder2 uses a better strategy and encodes the shapes for the
+    /// whole column upfront (before all blocks).
+    /// @note Although ArcticDB did not support multidimensional user data prior to creating ColumnEncoder2 some of the
+    /// internal data was multidimensional and used ColumnEncoder. More specifically: string pool and metadata.
+    /// @note This should be used for V2 encoding. V1 encoding can't use it as there is already data written the other
+    ///	way and it will be hard to distinguish both.
+    struct ColumnEncoderV2 {
+    public:
+        static void encode(
+            const arcticdb::proto::encoding::VariantCodec &codec_opts,
+            ColumnData& column_data,
+            std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+            Buffer& out,
+            std::ptrdiff_t& pos);
+        static std::pair<size_t, size_t> max_compressed_size(
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            ColumnData& column_data);
+    private:
+        static void encode_shapes(
+            const ColumnData& column_data,
+            std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+            Buffer& out,
+            std::ptrdiff_t& pos_in_buffer);
+        static void encode_blocks(
+            const arcticdb::proto::encoding::VariantCodec &codec_opts,
+            ColumnData& column_data,
+            std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+            Buffer& out,
+            std::ptrdiff_t& pos);
+    };
+
+
+    [[nodiscard]] static TypedBlockData<ShapesBlockTDT> create_shapes_typed_block(const ColumnData& column_data) {
+        static_assert(sizeof(ssize_t) == sizeof(int64_t));
+        const size_t row_count = column_data.shapes()->bytes() / sizeof(shape_t);
+        return {reinterpret_cast<const typename ShapesBlockTDT::DataTypeTag::raw_type*>(column_data.shapes()->data()),
+                nullptr,
+                column_data.shapes()->bytes(),
+                row_count,
+                nullptr};
+    }
+
+    void ColumnEncoderV2::encode(
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        ColumnData& column_data,
+        std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+        Buffer& out,
+        std::ptrdiff_t& pos
+    ) {
+        encode_shapes(column_data, variant_field, out, pos);
+        encode_blocks(codec_opts, column_data, variant_field, out, pos);
+        encode_sparse_map(column_data, variant_field, out, pos);
+    }
+
+    void ColumnEncoderV2::encode_shapes(
+        const ColumnData& column_data,
+        std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+        Buffer& out,
+        std::ptrdiff_t& pos_in_buffer
+    ) {
+        // There is no need to store the shapes for a column of empty type as they will be all 0. The type handler will
+        // assign 0 for the shape upon reading. There is one edge case - when we have None in the column, as it should not
+        // have shape at all (since it's not an array). This is handled by the sparse map.
+        if(column_data.type().dimension() != Dimension::Dim0 && !is_empty_type(column_data.type().data_type())) {
+            TypedBlockData<ShapesBlockTDT> shapes_block = create_shapes_typed_block(column_data);
+            util::variant_match(variant_field, [&](auto field){
+                using ShapesEncoder = TypedBlockEncoderImpl<TypedBlockData, ShapesBlockTDT, EncodingVersion::V2>;
+                ShapesEncoder::encode_shapes(codec::default_shapes_codec(), shapes_block, *field, out, pos_in_buffer);
+            });
+        }
+    }
+
+    void ColumnEncoderV2::encode_blocks(
+        const arcticdb::proto::encoding::VariantCodec &codec_opts,
+        ColumnData& column_data,
+        std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+        Buffer& out,
+        std::ptrdiff_t& pos
+    ) {
+        column_data.type().visit_tag([&](auto type_desc_tag) {
+            using TDT = decltype(type_desc_tag);
+            using Encoder = TypedBlockEncoderImpl<TypedBlockData, TDT, EncodingVersion::V2>;
+            ARCTICDB_TRACE(log::codec(), "Column data has {} blocks", column_data.num_blocks());
+            while (auto block = column_data.next<TDT>()) {
+                if constexpr(must_contain_data(static_cast<TypeDescriptor>(type_desc_tag))) {
+                    util::check(block.value().nbytes() > 0, "Zero-sized block");
+                }
+                util::variant_match(variant_field, [&](auto field) {
+                    Encoder::encode_values(codec_opts, block.value(), *field, out, pos);
+                });
+            }
+        });
+    }
+
+    std::pair<size_t, size_t> ColumnEncoderV2::max_compressed_size(
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        ColumnData& column_data
+    ) {
+        return column_data.type().visit_tag([&codec_opts, &column_data](auto type_desc_tag) {
+            size_t max_compressed_bytes = 0;
+            size_t uncompressed_bytes = 0;
+            using TDT = decltype(type_desc_tag);
+            using Encoder = TypedBlockEncoderImpl<TypedBlockData, TDT, EncodingVersion::V2>;
+            using ShapesEncoder = TypedBlockEncoderImpl<TypedBlockData, ShapesBlockTDT, EncodingVersion::V2>;
+            ARCTICDB_TRACE(log::codec(), "Column data has {} blocks", column_data.num_blocks());
+            const size_t shapes_byte_count = column_data.shapes()->bytes();
+            const TypedBlockData<ShapesBlockTDT> shapes_block = create_shapes_typed_block(column_data);
+            max_compressed_bytes += ShapesEncoder::max_compressed_size(codec::default_shapes_codec(), shapes_block);
+            uncompressed_bytes += shapes_byte_count;
+            while (auto block = column_data.next<TDT>()) {
+                const auto nbytes = block.value().nbytes();
+                if constexpr(must_contain_data(static_cast<TypeDescriptor>(type_desc_tag))) {
+                    util::check(nbytes > 0, "Zero-sized block");
+                }
+                uncompressed_bytes += nbytes;
+                // For the empty type the column will contain 0 size of user data however the encoder might need add some
+                // encoder specific data to the buffer, thus the uncompressed size will be 0 but the max_compressed_bytes
+                // might be non-zero.
+                max_compressed_bytes += Encoder::max_compressed_size(codec_opts, block.value());
+            }
+            add_bitmagic_compressed_size(column_data, uncompressed_bytes, max_compressed_bytes);
+            return std::make_pair(uncompressed_bytes, max_compressed_bytes);
+        });
+    }
+
+    using EncodingPolicyV2 = EncodingPolicyType<EncodingVersion::V2, ColumnEncoderV2>;
+
+    static void encode_field_descriptors(
+        const SegmentInMemory& in_mem_seg,
+        arcticdb::proto::encoding::SegmentHeader& segment_header,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        Buffer& out_buffer,
+        std::ptrdiff_t& pos
+    ) {
+        ARCTICDB_TRACE(log::codec(), "Encoding field descriptors to position {}", pos);
+        auto *encoded_field = segment_header.mutable_descriptor_field();
+        auto col = in_mem_seg.descriptor().fields().column_data();
+        ColumnEncoderV2::encode(codec_opts, col, encoded_field, out_buffer, pos);
+        ARCTICDB_TRACE(log::codec(), "Encoded field descriptors to position {}", pos);
+
+        write_magic<IndexMagic>(out_buffer, pos);
+        if (in_mem_seg.index_fields()) {
+            ARCTICDB_TRACE(log::codec(), "Encoding index fields descriptors to position {}", pos);
+            auto *index_field = segment_header.mutable_index_descriptor_field();
+            auto index_col = in_mem_seg.index_fields()->column_data();
+            ColumnEncoderV2::encode(codec_opts, index_col, index_field, out_buffer, pos);
+            ARCTICDB_TRACE(log::codec(), "Encoded index field descriptors to position {}", pos);
+        }
+    }
+
+    [[nodiscard]] size_t calc_column_blocks_size(const Column& col) {
+        size_t bytes = EncodedField::Size;
+        if(col.type().dimension() != entity::Dimension::Dim0)
+            bytes += sizeof(EncodedBlock);
+
+        bytes += sizeof(EncodedBlock) * col.num_blocks();
+        ARCTICDB_DEBUG(log::version(), "Encoded block size: {} + shapes({}) + {} * {} = {}",
+            EncodedField::Size,
+            col.type().dimension() != entity::Dimension::Dim0 ? sizeof(EncodedBlock) : 0u,
+            sizeof(EncodedBlock),
+            col.num_blocks(),
+            bytes);
+
+        return bytes;
+    }
+
+    [[nodiscard]] static size_t encoded_blocks_size(const SegmentInMemory& in_mem_seg) {
+        size_t bytes = 0;
+        for (std::size_t c = 0; c < in_mem_seg.num_columns(); ++c) {
+            const auto& col = in_mem_seg.column(position_t(c));
+            bytes += calc_column_blocks_size(col);
+        }
+        return bytes;
+    }
+
+    static void calc_encoded_blocks_size(
+        const SegmentInMemory& in_mem_seg,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        SizeResult& result
+    ) {
+        result.encoded_blocks_bytes_ = static_cast<shape_t>(encoded_blocks_size(in_mem_seg));
+        result.uncompressed_bytes_ += result.encoded_blocks_bytes_;
+        result.max_compressed_bytes_ += BytesEncoder<EncodingPolicyV2>::max_compressed_size(codec_opts, result.encoded_blocks_bytes_);
+    }
+
+    static void calc_stream_descriptor_fields_size(
+        const SegmentInMemory& in_mem_seg,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        SizeResult& result
+    ) {
+        auto segment_fields = in_mem_seg.descriptor().fields().column_data();
+        const auto [uncompressed, required] = ColumnEncoderV2::max_compressed_size(codec_opts, segment_fields);
+        result.uncompressed_bytes_ += uncompressed;
+        result.max_compressed_bytes_ += required;
+
+        // Calculate index fields size
+        if(in_mem_seg.index_fields()) {
+            auto index_field_data = in_mem_seg.index_fields()->column_data();
+            const auto [idx_uncompressed, idx_required] = ColumnEncoderV2::max_compressed_size(codec_opts, index_field_data);
+            result.uncompressed_bytes_ += idx_uncompressed;
+            result.max_compressed_bytes_ += idx_required;
+        }
+    }
+
+    [[nodiscard]] static SizeResult max_compressed_size_v2(
+        const SegmentInMemory& in_mem_seg,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts
+    ) {
+        ARCTICDB_SAMPLE(GetSegmentCompressedSize, 0)
+        SizeResult result{};
+        result.max_compressed_bytes_ += sizeof(MetadataMagic);
+        calc_metadata_size<EncodingPolicyV2>(in_mem_seg, codec_opts, result);
+        result.max_compressed_bytes_ += sizeof(DescriptorMagic);
+        result.max_compressed_bytes_ += sizeof(IndexMagic);
+        calc_stream_descriptor_fields_size(in_mem_seg, codec_opts, result);
+        result.max_compressed_bytes_ += sizeof(EncodedMagic);
+        calc_encoded_blocks_size(in_mem_seg, codec_opts, result);
+
+        // Calculate fields collection size
+        if(in_mem_seg.row_count() > 0) {
+            result.max_compressed_bytes_ += sizeof(ColumnMagic) * in_mem_seg.descriptor().field_count();
+            calc_columns_size<EncodingPolicyV2>(in_mem_seg, codec_opts, result);
+            result.max_compressed_bytes_ += sizeof(StringPoolMagic);
+            calc_string_pool_size<EncodingPolicyV2>(in_mem_seg, codec_opts, result);
+        }
+        ARCTICDB_TRACE(log::codec(), "Max compressed size {}", result.max_compressed_bytes_);
+        return result;
+    }
+
+    static void encode_encoded_fields(
+        arcticdb::proto::encoding::SegmentHeader& segment_header,
+        const arcticdb::proto::encoding::VariantCodec& codec_opts,
+        Buffer& out_buffer,
+        std::ptrdiff_t& pos,
+        const ChunkedBuffer& encoded_blocks_buffer
+    ) {
+
+        ARCTICDB_TRACE(log::codec(), "Encoding encoded blocks to position {}", pos);
+        auto encoded_field = segment_header.mutable_column_fields();
+        encoded_field->set_offset(static_cast<uint32_t>(pos));
+        write_magic<EncodedMagic>(out_buffer, pos);
+        BytesEncoder<EncodingPolicyV2>::encode(encoded_blocks_buffer, codec_opts, out_buffer, pos, encoded_field);
+        ARCTICDB_TRACE(log::codec(), "Encoded encoded blocks to position {}", pos);
+    }
+
+    [[nodiscard]] Segment encode_v2(SegmentInMemory&& s, const arcticdb::proto::encoding::VariantCodec &codec_opts) {
+        ARCTICDB_SAMPLE(EncodeSegment, 0)
+
+        auto in_mem_seg = std::move(s);
+        auto arena = std::make_unique<google::protobuf::Arena>();
+        auto segment_header = google::protobuf::Arena::CreateMessage<arcticdb::proto::encoding::SegmentHeader>(arena.get());
+        auto& seg_descriptor = in_mem_seg.descriptor();
+        *segment_header->mutable_stream_descriptor() = std::move(seg_descriptor.mutable_proto());
+        segment_header->set_compacted(in_mem_seg.compacted());
+        segment_header->set_encoding_version(static_cast<uint16_t>(EncodingVersion::V2));
+
+        std::ptrdiff_t pos = 0;
+        static auto block_to_header_ratio = ConfigsMap::instance()->get_int("Codec.EstimatedHeaderRatio", 75);
+        const auto preamble = in_mem_seg.num_blocks() * block_to_header_ratio;
+        auto [max_compressed_size, uncompressed_size, encoded_buffer_size] = max_compressed_size_v2(in_mem_seg, codec_opts);
+        ARCTICDB_TRACE(log::codec(), "Estimated max buffer requirement: {}", max_compressed_size);
+        auto out_buffer = std::make_shared<Buffer>(max_compressed_size + encoded_buffer_size, preamble);
+        ARCTICDB_TRACE(log::codec(), "Encoding descriptor: {}", segment_header->stream_descriptor().DebugString());
+        auto *tsd = segment_header->mutable_stream_descriptor();
+        tsd->set_in_bytes(uncompressed_size);
+
+        write_magic<MetadataMagic>(*out_buffer, pos);
+        encode_metadata<EncodingPolicyV2>(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
+        write_magic<DescriptorMagic>(*out_buffer, pos);
+        encode_field_descriptors(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
+
+        auto encoded_fields_buffer = ChunkedBuffer::presized(static_cast<size_t>(encoded_buffer_size));
+        auto encoded_field_pos = 0u;
+        ColumnEncoderV2 encoder;
+        if(in_mem_seg.row_count() > 0) {
+            ARCTICDB_TRACE(log::codec(), "Encoding fields");
+            for (std::size_t column_index = 0; column_index < in_mem_seg.num_columns(); ++column_index) {
+                write_magic<ColumnMagic>(*out_buffer, pos);
+                auto column_field = new(encoded_fields_buffer.data() + encoded_field_pos) EncodedField;
+                ARCTICDB_TRACE(log::codec(),"Beginning encoding of column {}: ({}) to position {}", column_index, in_mem_seg.descriptor().field(column_index).name(), pos);
+                auto column_data = in_mem_seg.column_data(column_index);
+                encoder.encode(codec_opts, column_data, column_field, *out_buffer, pos);
+                ARCTICDB_TRACE(log::codec(), "Encoded column {}: ({}) to position {}", column_index, in_mem_seg.descriptor().field(column_index).name(), pos);
+                encoded_field_pos += encoded_field_bytes(*column_field);
+                util::check(encoded_field_pos <= encoded_fields_buffer.bytes(),
+                    "Encoded field buffer overflow {} > {}",
+                    encoded_field_pos,
+                    encoded_fields_buffer.bytes());
+            }
+            auto field_here ARCTICDB_UNUSED = reinterpret_cast<EncodedField*>(encoded_fields_buffer.data());
+            write_magic<StringPoolMagic>(*out_buffer, pos);
+            encode_string_pool<EncodingPolicyV2>(in_mem_seg, *segment_header, codec_opts, *out_buffer, pos);
+        }
+
+        auto field_before ARCTICDB_UNUSED = reinterpret_cast<EncodedField*>(encoded_fields_buffer.data());
+        encode_encoded_fields(*segment_header, codec_opts, *out_buffer, pos, encoded_fields_buffer);
+        auto field ARCTICDB_UNUSED = reinterpret_cast<EncodedField*>(encoded_fields_buffer.data());
+        out_buffer->set_bytes(pos);
+        tsd->set_out_bytes(pos);
+
+        ARCTICDB_DEBUG(log::codec(), "Encoded header: {}", tsd->DebugString());
+        ARCTICDB_DEBUG(log::codec(), "Block count {} header size {} ratio {}",
+            in_mem_seg.num_blocks(), segment_header->ByteSizeLong(),
+            in_mem_seg.num_blocks() ? segment_header->ByteSizeLong() / in_mem_seg.num_blocks() : 0);
+        return {std::move(arena), segment_header, std::move(out_buffer), seg_descriptor.fields_ptr()};
+    }
+}
diff --git a/cpp/arcticdb/codec/encoded_field.hpp b/cpp/arcticdb/codec/encoded_field.hpp
index a0e6bb9c81..979014bd61 100644
--- a/cpp/arcticdb/codec/encoded_field.hpp
+++ b/cpp/arcticdb/codec/encoded_field.hpp
@@ -199,6 +199,10 @@ struct EncodedBlock {
         hash_ = hash;
     }
 
+    [[nodiscard]] uint64_t hash() const {
+        return hash_;
+    }
+
     uint32_t out_bytes() const {
         return out_bytes_;
     }
@@ -332,6 +336,12 @@ struct EncodedField {
             return is_shapes_ ? field_.shapes_count_ : field_.shapes_count_ + field_.values_count_;
         }
 
+        [[nodiscard]] const EncodedBlock& operator[](const size_t idx) const {
+            // Shape blocks are located before values blocks in the field. In case this is a collection of value blocks
+            // we have to skip all shape blocks. In case this is a collection of shapes we can start from 0 index.
+            const size_t shape_offset = !is_shapes_ * field_.shapes_count_;
+            return field_.blocks()[shape_offset + idx];
+        }
         const EncodedField &field_;
         bool is_shapes_;
     };
@@ -340,7 +350,7 @@ struct EncodedField {
         return type_;
     }
 
-    const EncodedBlock &shapes(size_t n) const {
+    const EncodedBlock& shapes(size_t n) const {
         util::check(n == 0, "Expected only one shape");
         util::check(shapes_count_ != 0, "No shape allocated");
         return blocks_[0];
diff --git a/cpp/arcticdb/codec/encoded_field_collection.hpp b/cpp/arcticdb/codec/encoded_field_collection.hpp
index 952830b0a4..ca3379ed0f 100644
--- a/cpp/arcticdb/codec/encoded_field_collection.hpp
+++ b/cpp/arcticdb/codec/encoded_field_collection.hpp
@@ -7,11 +7,8 @@
 
 #pragma once
 
-#include <column_store/chunked_buffer.hpp>
-#include <util/cursored_buffer.hpp>
-#include <util/buffer.hpp>
-#include <column_store/column_data.hpp>
-#include <codec/encoded_field.hpp>
+#include <arcticdb/codec/encoded_field.hpp>
+#include <arcticdb/util/buffer.hpp>
 
 namespace arcticdb {
 
diff --git a/cpp/arcticdb/codec/lz4.hpp b/cpp/arcticdb/codec/lz4.hpp
index b957574e7b..e465835a0e 100644
--- a/cpp/arcticdb/codec/lz4.hpp
+++ b/cpp/arcticdb/codec/lz4.hpp
@@ -45,9 +45,13 @@ struct Lz4BlockEncoder {
         int compressed_bytes = LZ4_compress_default(
             reinterpret_cast<const char *>(in),
             reinterpret_cast<char *>(out),
-            int(block_utils.bytes_), int(out_capacity));
+            int(block_utils.bytes_),
+            int(out_capacity));
 
-        util::check_arg(compressed_bytes >= 0, "expected compressed bytes >= 0, actual {}", compressed_bytes);
+        // Compressed bytes equal to 0 means error unless there is nothing to compress.
+        util::check_arg(compressed_bytes > 0 || (compressed_bytes == 0 && block_utils.bytes_ == 0),
+            "expected compressed bytes >= 0, actual {}",
+            compressed_bytes);
         ARCTICDB_TRACE(log::storage(), "Block of size {} compressed to {} bytes", block_utils.bytes_, compressed_bytes);
         hasher(in, block_utils.count_);
         pos += ssize_t(compressed_bytes);
@@ -56,9 +60,6 @@ struct Lz4BlockEncoder {
     }
 };
 
-template<template<typename> class F, class TD>
-using Lz4Encoder = GenericBlockEncoder<F<TD>, TD, Lz4BlockEncoder>;
-
 struct Lz4Decoder {
 /*
  * encoder_version is here to support multiple versions but won't be used before we have them
@@ -67,24 +68,36 @@ struct Lz4Decoder {
 #pragma GCC diagnostic ignored "-Wunused-parameter"
 
     template<typename T>
-    static void decode_block([[maybe_unused]] std::uint32_t encoder_version, const std::uint8_t *in, std::size_t in_bytes, T *t_out,
-                             std::size_t out_bytes) {
+    static void decode_block(
+        [[maybe_unused]] std::uint32_t encoder_version,
+        const std::uint8_t* in,
+        std::size_t in_bytes,
+        T* t_out,
+        std::size_t out_bytes
+    ) {
 
         ARCTICDB_TRACE(log::codec(), "Lz4 decoder reading block: {} {}", in_bytes, out_bytes);
-        int real_decomp = LZ4_decompress_safe(
-            reinterpret_cast<const char *>(in),
-            reinterpret_cast<char *>(t_out),
+        // Decompressed size < 0 means an error occurred in LZ4 during the decompression. In case it's  negative
+        // the specific value is somewhat random and does not mean anything. Decompressed size of 0 is allowed and means
+        // 0 bytes were passed for compression. In that case t_out is allowed to be null since it's not used at all.
+        const int decompressed_size = LZ4_decompress_safe(
+            reinterpret_cast<const char*>(in),
+            reinterpret_cast<char*>(t_out),
             int(in_bytes),
             int(out_bytes)
         );
-        codec::check<ErrorCode::E_DECODE_ERROR>(real_decomp > 0, "Error while decoding with lz4 at address {:x} with size {}. Code {}", uintptr_t(in), in_bytes, real_decomp);
-        codec::check<ErrorCode::E_DECODE_ERROR>(std::size_t(real_decomp) == out_bytes,
-                        "expected out_bytes == lz4 decompressed bytes, actual {} != {}",
-                        out_bytes,
-                        real_decomp);
+        util::check_arg(decompressed_size >= 0,
+            "Error while decoding with lz4 at address {:x} with size {}. Code {}",
+            uintptr_t(in),
+            in_bytes,
+            decompressed_size);
+        util::check_arg(std::size_t(decompressed_size) == out_bytes,
+            "expected out_bytes == lz4 decompressed bytes, actual {} != {}",
+            out_bytes,
+            decompressed_size);
     }
 
 #pragma GCC diagnostic pop
 };
 
-} // namespace arcticdb::detail
\ No newline at end of file
+} // namespace arcticdb::detail
diff --git a/cpp/arcticdb/codec/magic_words.hpp b/cpp/arcticdb/codec/magic_words.hpp
new file mode 100644
index 0000000000..c654c7e92c
--- /dev/null
+++ b/cpp/arcticdb/codec/magic_words.hpp
@@ -0,0 +1,17 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+#pragma once
+
+namespace arcticdb {
+    using DescriptorMagic = util::MagicNum<'D','e','s','c'>;
+    using EncodedMagic = util::MagicNum<'E','n','c','d'>;
+    using StringPoolMagic = util::MagicNum<'S','t','r','p'>;
+    using MetadataMagic = util::MagicNum<'M','e','t','a'>;
+    using IndexMagic = util::MagicNum<'I','n','d','x'>;
+    using ColumnMagic = util::MagicNum<'C','l','m','n'>;
+}
diff --git a/cpp/arcticdb/codec/passthrough.hpp b/cpp/arcticdb/codec/passthrough.hpp
index 5a52166104..0660a05651 100644
--- a/cpp/arcticdb/codec/passthrough.hpp
+++ b/cpp/arcticdb/codec/passthrough.hpp
@@ -18,7 +18,9 @@
 namespace arcticdb::detail {
 
 template<template<typename> class BlockType, class TD>
-struct PassthroughEncoder {
+struct PassthroughEncoderV1 {
+
+    using Opts = arcticdb::proto::encoding::VariantCodec::Passthrough;
 
     static size_t max_compressed_size(const BlockType<TD> &block ) {
         using Helper = CodecHelper<TD>;
@@ -33,7 +35,7 @@ struct PassthroughEncoder {
     }
 
     template <typename EncodedFieldType>
-    static void encode(BlockType<TD> &block, EncodedFieldType &field, Buffer &out, std::ptrdiff_t &pos) {
+    static void encode(const Opts&, const BlockType<TD>& block, EncodedFieldType& field, Buffer& out, std::ptrdiff_t& pos) {
         using namespace arcticdb::entity;
         using Helper = CodecHelper<TD>;
         using T = typename Helper::T;
@@ -70,14 +72,20 @@ struct PassthroughEncoder {
             T *t_out = out.ptr_cast<T>(pos, helper_array_block.values_.bytes_);
             encode_block(d, helper_array_block.values_, helper.hasher_, t_out, pos);
             auto field_nd_array = field.mutable_ndarray();
-            auto values_pb = field_nd_array->add_values();
+            // Important: In case V2 EncodedField is used shapes must be added before values.
             auto shapes_pb = field_nd_array->add_shapes();
+            auto values_pb = field_nd_array->add_values();
             helper_array_block.update_field_size(*field_nd_array);
-            helper_array_block.set_block_data(shapes_pb, values_pb, shape_hash, helper_array_block.shapes_.bytes_,
-                                      helper.hasher_.digest(), helper_array_block.values_.bytes_);
+            helper_array_block.set_block_data(
+				shapes_pb,
+				values_pb,
+				shape_hash,
+				helper_array_block.shapes_.bytes_,
+				helper.hasher_.digest(),
+				helper_array_block.values_.bytes_);
         }
     }
-
+private:
     template<class T>
     static void encode_block(const T *in, BlockProtobufHelper &block_utils, HashAccum &hasher, T *out, std::ptrdiff_t &pos) {
         memcpy(out, in, block_utils.bytes_);
@@ -86,6 +94,59 @@ struct PassthroughEncoder {
     }
 };
 
+/// @brief "Encoder" which stores the input in the output buffer without performing any transformation
+/// @note The difference between arcticdb::detail::PassthroughEncoder and arcticdb::detail::PassthroughEncoder2 is that
+/// the latter does not care about the shapes array.
+/// @see arcticdb::ColumnEncoder2 arcticdb::detail::GenericBlockEncoder2
+template<template<typename> class BlockType, class TD>
+struct PassthroughEncoderV2 {
+
+    using Opts = arcticdb::proto::encoding::VariantCodec::Passthrough;
+
+    static size_t max_compressed_size(const BlockType<TD> &block) {
+        return block.nbytes();
+    }
+
+    template <typename EncodedBlockType>
+    static void encode(
+        const Opts&,
+        const BlockType<TD> &block,
+        Buffer &out,
+        std::ptrdiff_t &pos,
+        EncodedBlockType* encoded_block
+    ) {
+        using namespace arcticdb::entity;
+        using Helper = CodecHelper<TD>;
+        using T = typename Helper::T;
+        Helper helper;
+        helper.hasher_.reset(helper.seed);
+        const T* d = block.data();
+        const size_t data_byte_size = block.nbytes();
+        helper.ensure_buffer(out, pos, data_byte_size);
+
+        // doing copy + hash in one pass, this might have a negative effect on perf
+        // since the hashing is path dependent. This is a toy example though so not critical
+        T *t_out = out.ptr_cast<T>(pos, data_byte_size);
+        encode_block(d, data_byte_size, helper.hasher_, t_out, pos);
+        encoded_block->set_in_bytes(data_byte_size);
+        encoded_block->set_out_bytes(data_byte_size);
+        encoded_block->set_hash(helper.hasher_.digest());
+    }
+private:
+    template<class T>
+    static void encode_block(
+        const T* in,
+        size_t in_byte_size,
+        HashAccum& hasher,
+        T* out,
+        std::ptrdiff_t& pos
+    ) {
+        memcpy(out, in, in_byte_size);
+        hasher(in, in_byte_size / sizeof(T));
+        pos += static_cast<ssize_t>(in_byte_size);
+    }
+};
+
 struct PassthroughDecoder {
     template<typename T>
     static void decode_block(const std::uint8_t *in, std::size_t in_bytes, T *t_out,
@@ -96,4 +157,4 @@ struct PassthroughDecoder {
     }
 };
 
-} // namespace arcticdb::detail
\ No newline at end of file
+} // namespace arcticdb::detail
diff --git a/cpp/arcticdb/codec/python_bindings.cpp b/cpp/arcticdb/codec/python_bindings.cpp
index 9e7db68b49..eeddaffd12 100644
--- a/cpp/arcticdb/codec/python_bindings.cpp
+++ b/cpp/arcticdb/codec/python_bindings.cpp
@@ -96,21 +96,6 @@ struct FieldEncodingResult {
     proto::encoding::EncodedField encoded_field_;
 };
 
-FieldEncodingResult py_encode_field(py::object &opts, DynamicFieldBuffer &field_buffer) {
-    proto::encoding::VariantCodec opts_cpp;
-    python_util::pb_from_python(opts, opts_cpp);
-
-    std::ptrdiff_t pos = 0;
-    proto::encoding::EncodedField encoded_field;
-    auto field = field_buffer.as_field();
-    const auto [uncompressed, required] = ColumnEncoder::max_compressed_size(opts_cpp, *field);
-    auto buffer = std::make_shared<Buffer>(required);
-    field->reset();
-    ColumnEncoder encoder;
-    encoder.encode(opts_cpp, *field, encoded_field, *buffer, pos);
-    return FieldEncodingResult{buffer, encoded_field};
-}
-
 Segment encode_segment(SegmentInMemory segment_in_memory, const py::object &opts, EncodingVersion encoding_version) {
     proto::encoding::VariantCodec opts_cpp;
     python_util::pb_from_python(opts, opts_cpp);
@@ -166,18 +151,6 @@ struct FieldDecodingResult {
     std::shared_ptr<Buffer> values_buffer_;
 };
 
-FieldDecodingResult py_decode_field(const TypeDescriptor &td, py::object &encoded_field, py::buffer &buffer) {
-    proto::encoding::EncodedField enc_field;
-    python_util::pb_from_python(encoded_field, enc_field);
-    auto info = buffer.request();
-    util::check_arg(info.ndim == 1, "only one dimension supported");
-
-    BufferPairDataSink ds;
-    std::optional<util::BitMagic> bv;
-    decode_field(td, enc_field, reinterpret_cast<std::uint8_t *>(info.ptr), ds, bv);
-    return FieldDecodingResult{ds.shapes(), ds.values()};
-}
-
 void register_codec(py::module &m) {
     py::class_<DynamicFieldBuffer>(m, "DynamicFieldBuffer")
         .def(py::init<TypeDescriptor, py::buffer, py::buffer>())
@@ -223,9 +196,6 @@ void register_codec(py::module &m) {
 
     m.def("encode_segment", &encode_segment);
     m.def("decode_segment", &decode_python_segment, py::return_value_policy::move);
-    m.def("encode_field", &py_encode_field, py::return_value_policy::move);
-    m.def("decode_field", &py_decode_field, py::return_value_policy::move);
 }
 
 } // namespace arcticdb
-
diff --git a/cpp/arcticdb/codec/segment.cpp b/cpp/arcticdb/codec/segment.cpp
index 6647546135..5eeab44b0d 100644
--- a/cpp/arcticdb/codec/segment.cpp
+++ b/cpp/arcticdb/codec/segment.cpp
@@ -12,6 +12,7 @@
 #include <arcticdb/util/pb_util.hpp>
 #include <arcticdb/util/dump_bytes.hpp>
 #include <arcticdb/codec/codec.hpp>
+#include <arcticdb/codec/magic_words.hpp>
 
 namespace arcticdb {
 namespace segment_size {
@@ -45,10 +46,11 @@ FieldCollection decode_fields(
         ARCTICDB_TRACE(log::codec(), "Decoding string pool");
         std::optional<util::BitMagic> bv;
         data += decode_field(FieldCollection::type(),
-                       hdr.descriptor_field(),
-                       data,
-                       fields,
-                       bv);
+            hdr.descriptor_field(),
+            data,
+            fields,
+            bv,
+            to_encoding_version(hdr.encoding_version()));
 
         ARCTICDB_TRACE(log::codec(), "Decoded string pool to position {}", data-begin);
     }
@@ -66,10 +68,11 @@ std::optional<FieldCollection> decode_index_fields(
         ARCTICDB_TRACE(log::codec(), "Decoding string pool");
         std::optional<util::BitMagic> bv;
         data += decode_field(FieldCollection::type(),
-                       hdr.index_descriptor_field(),
-                       data,
-                       fields,
-                       bv);
+            hdr.index_descriptor_field(),
+            data,
+            fields,
+            bv,
+            to_encoding_version(hdr.encoding_version()));
 
         ARCTICDB_TRACE(log::codec(), "Decoded string pool to position {}", data-begin);
         return std::make_optional<FieldCollection>(std::move(fields));
@@ -110,11 +113,11 @@ Segment Segment::from_bytes(const std::uint8_t* src, std::size_t readable_size,
         fields = fields_from_proto(seg_hdr->stream_descriptor());
     else {
         const auto* fields_ptr = src;
-        check_magic<MetadataMagic>(fields_ptr);
+        util::check_magic<MetadataMagic>(fields_ptr);
         if(seg_hdr->has_metadata_field())
             fields_ptr += encoding_sizes::field_compressed_size(seg_hdr->metadata_field());
 
-        check_magic<DescriptorMagic>(fields_ptr);
+        util::check_magic<DescriptorMagic>(fields_ptr);
         if(seg_hdr->has_descriptor_field() && seg_hdr->descriptor_field().has_ndarray())
             fields = decode_fields(*seg_hdr, fields_ptr);
     }
@@ -189,11 +192,11 @@ Segment Segment::from_buffer(std::shared_ptr<Buffer>&& buffer) {
     }
     else {
         const auto* fields_ptr = buffer->data() + preamble_size;
-        check_magic<MetadataMagic>(fields_ptr);
+        util::check_magic<MetadataMagic>(fields_ptr);
         if(seg_hdr->has_metadata_field())
             fields_ptr += encoding_sizes::field_compressed_size(seg_hdr->metadata_field());
 
-        check_magic<DescriptorMagic>(fields_ptr);
+        util::check_magic<DescriptorMagic>(fields_ptr);
         if(seg_hdr->has_descriptor_field() && seg_hdr->descriptor_field().has_ndarray())
             fields = decode_fields(*seg_hdr, fields_ptr);
     }
diff --git a/cpp/arcticdb/codec/segment.hpp b/cpp/arcticdb/codec/segment.hpp
index 889b44e9bd..8799755ee0 100644
--- a/cpp/arcticdb/codec/segment.hpp
+++ b/cpp/arcticdb/codec/segment.hpp
@@ -27,9 +27,16 @@ std::tuple<size_t, size_t> compressed(const arcticdb::proto::encoding::SegmentHe
 
 enum class EncodingVersion : uint16_t {
     V1 = 0,
-    V2 = 1
+    V2 = 1,
+    COUNT
 };
 
+template<typename T, typename = std::enable_if_t<std::is_integral_v<T>>>
+inline constexpr EncodingVersion to_encoding_version(T encoding_version) {
+    util::check(encoding_version >= 0 && encoding_version < uint16_t(EncodingVersion::COUNT), "Invalid encoding version");
+    return static_cast<EncodingVersion>(encoding_version);
+}
+
 static constexpr uint16_t HEADER_VERSION_V1 = 1;
 
 inline EncodingVersion encoding_version(const storage::LibraryDescriptor::VariantStoreConfig& cfg) {
@@ -284,9 +291,10 @@ struct formatter<arcticdb::EncodingVersion> {
             break;
         case arcticdb::EncodingVersion::V2:c = '2';
             break;
+        default:break;
         }
         return format_to(ctx.out(), "{:c}", c);
     }
 };
 
-} //namespace fmt
\ No newline at end of file
+} //namespace fmt
diff --git a/cpp/arcticdb/codec/typed_block_encoder_impl.hpp b/cpp/arcticdb/codec/typed_block_encoder_impl.hpp
new file mode 100644
index 0000000000..34ea13b2fb
--- /dev/null
+++ b/cpp/arcticdb/codec/typed_block_encoder_impl.hpp
@@ -0,0 +1,149 @@
+#include <arcticdb/codec/encoding_sizes.hpp>
+#include <arcticdb/codec/passthrough.hpp>
+#include <arcticdb/codec/zstd.hpp>
+#include <arcticdb/codec/lz4.hpp>
+#include <arcticdb/codec/encoded_field.hpp>
+#include <arcticdb/util/buffer.hpp>
+
+#include <type_traits>
+
+namespace arcticdb {
+    /// @todo Split this class. This class does too much. Supports encoding via V1 and V2 but in a clunky way. The
+    ///	interface for encoding is different (V2 requires EncodedBlock to be passed, thus encode_values and
+    /// encode_shapes were added).
+    template<template<typename> class TypedBlock, class TD, EncodingVersion encoder_version>
+    struct TypedBlockEncoderImpl {
+        using ShapesBlockTDT = TypeDescriptorTag<DataTypeTag<DataType::INT64>, DimensionTag<Dimension::Dim0>>;
+
+        static size_t max_compressed_size(
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            const TypedBlock<TD>& typed_block
+        ) {
+            return visit_encoder(codec_opts, [&](auto encoder_tag) {
+                return decltype(encoder_tag)::Encoder::max_compressed_size(typed_block);
+            });
+        }
+        /**
+         * Perform encoding of in memory field for storage
+         * @param[in] codec_opts Option used to dispatch to the appropriate encoder and configure it
+         * @param[in] typed_block The block which will be encoded
+         * @param[in, out] field description of the encoding operation
+         * @param[out] out output buffer to write the encoded values to. Must be resized if pos becomes > size
+         * @param[in, out] pos position in bytes in the buffer where to start writing.
+         *  Modified to reflect the position after the last byte written
+         */
+        template <typename EncodedFieldType>
+        static void encode(
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            const TypedBlock<TD>& typed_block,
+            EncodedFieldType& field,
+            Buffer& out,
+            std::ptrdiff_t& pos
+        ) {
+            static_assert(encoder_version == EncodingVersion::V1,
+                "Encoding of both shapes and values at the same time is allowed only in V1 encoding");
+            visit_encoder(codec_opts, [&](auto encoder_tag) {
+                decltype(encoder_tag)::Encoder::encode(get_opts(codec_opts, encoder_tag),
+                    typed_block,
+                    field,
+                    out,
+                    pos);
+            });
+        }
+
+        template<typename EncodedFieldType>
+        static void encode_values(
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            const TypedBlock<TD>& typed_block,
+            EncodedFieldType& field,
+            Buffer& out,
+            std::ptrdiff_t& pos
+        ) {
+            static_assert(encoder_version == EncodingVersion::V2,
+                "Encoding values separately from the shapes is allowed only in V2 encoding");
+            auto* ndarray = field.mutable_ndarray();
+            if(typed_block.nbytes() == 0) {
+                ARCTICDB_TRACE(log::codec(), "Encoder got values of size 0. Noting to encode.");
+                return;
+            }
+            auto* values_encoded_block = ndarray->add_values();
+            visit_encoder(codec_opts, [&](auto encoder_tag) {
+                decltype(encoder_tag)::Encoder::encode(get_opts(codec_opts, encoder_tag),
+                    typed_block,
+                    out,
+                    pos,
+                    values_encoded_block);
+            });
+            const auto existing_items_count = ndarray->items_count();
+            ndarray->set_items_count(existing_items_count + typed_block.row_count());
+        }
+
+        template<typename EncodedFieldType>
+        static void encode_shapes(
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            const TypedBlockData<ShapesBlockTDT>& typed_block,
+            EncodedFieldType& field,
+            Buffer& out,
+            std::ptrdiff_t& pos
+        ) {
+            static_assert(encoder_version == EncodingVersion::V2,
+                "Encoding shapes separately from the values is allowed only in V2 encoding");
+            if(typed_block.nbytes() == 0) {
+                ARCTICDB_TRACE(log::codec(), "Encoder got shapes of size 0. Noting to encode.");
+                return;
+            }
+            auto* ndarray = field.mutable_ndarray();
+            auto* shapes_encoded_block = ndarray->add_shapes();
+            visit_encoder(codec_opts, [&](auto encoder_tag) {
+                decltype(encoder_tag)::Encoder::encode(get_opts(codec_opts, encoder_tag),
+                    typed_block,
+                    out,
+                    pos,
+                    shapes_encoded_block);
+            });
+        }
+    private:
+        template<class EncoderType>
+        using BlockEncoder = std::conditional_t<encoder_version == EncodingVersion::V1,
+            arcticdb::detail::GenericBlockEncoder<TypedBlock<TD>, TD, EncoderType>,
+            arcticdb::detail::GenericBlockEncoderV2<TypedBlock<TD>, TD, EncoderType>>;
+
+        using ZstdEncoder = BlockEncoder<arcticdb::detail::ZstdBlockEncoder>;
+        using Lz4Encoder = BlockEncoder<arcticdb::detail::Lz4BlockEncoder>;
+
+        using PassthroughEncoder = std::conditional_t<encoder_version == EncodingVersion::V1,
+            arcticdb::detail::PassthroughEncoderV1<TypedBlock, TD>,
+            arcticdb::detail::PassthroughEncoderV2<TypedBlock, TD>>;
+
+        template<typename EncoderT>
+        struct EncoderTag {
+            using Encoder = EncoderT;
+        };
+
+        template<typename FunctorT>
+        static auto visit_encoder(const arcticdb::proto::encoding::VariantCodec& codec_opts, FunctorT&& f) {
+            switch (codec_opts.codec_case()) {
+                case arcticdb::proto::encoding::VariantCodec::kZstd:
+                    return f(EncoderTag<ZstdEncoder>());
+                case arcticdb::proto::encoding::VariantCodec::kLz4:
+                    return f(EncoderTag<Lz4Encoder>());
+                case arcticdb::proto::encoding::VariantCodec::kPassthrough :
+                    return f(EncoderTag<PassthroughEncoder>());
+                default:
+                    return f(EncoderTag<PassthroughEncoder>());
+            }
+        }
+
+        static auto get_opts(const arcticdb::proto::encoding::VariantCodec& codec_opts, EncoderTag<Lz4Encoder>) {
+            return codec_opts.lz4();
+        }
+
+        static auto get_opts(const arcticdb::proto::encoding::VariantCodec& codec_opts, EncoderTag<ZstdEncoder>) {
+            return codec_opts.zstd();
+        }
+
+        static auto get_opts(const arcticdb::proto::encoding::VariantCodec& codec_opts, EncoderTag<PassthroughEncoder>) {
+            return codec_opts.passthrough();
+        }
+    };
+}
diff --git a/cpp/arcticdb/codec/variant_encoded_field_collection.cpp b/cpp/arcticdb/codec/variant_encoded_field_collection.cpp
index f300110d8f..df3da27cd2 100644
--- a/cpp/arcticdb/codec/variant_encoded_field_collection.cpp
+++ b/cpp/arcticdb/codec/variant_encoded_field_collection.cpp
@@ -5,8 +5,9 @@
  * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
  */
 
-#include <codec/variant_encoded_field_collection.hpp>
-#include <codec/codec.hpp>
+#include <arcticdb/codec/variant_encoded_field_collection.hpp>
+#include <arcticdb/codec/codec.hpp>
+#include <arcticdb/codec/magic_words.hpp>
 
 namespace arcticdb {
 
@@ -14,7 +15,7 @@ VariantEncodedFieldCollection::VariantEncodedFieldCollection(const Segment& segm
     if(EncodingVersion(segment.header().encoding_version()) == EncodingVersion::V2) {
         const auto& hdr = segment.header();
         auto [begin, encoded_fields_ptr] = get_segment_begin_end(segment, segment.header());
-        check_magic<EncodedMagic>(encoded_fields_ptr);
+        util::check_magic<EncodedMagic>(encoded_fields_ptr);
         auto encoded_fields_buffer = decode_encoded_fields(hdr, encoded_fields_ptr, begin);
         fields_ = EncodedFieldCollection{std::move(encoded_fields_buffer)};
     } else {
@@ -23,4 +24,18 @@ VariantEncodedFieldCollection::VariantEncodedFieldCollection(const Segment& segm
     }
 }
 
-} //namespace arcticdb
\ No newline at end of file
+VariantField VariantEncodedFieldCollection::at(size_t pos) const {
+    if (is_proto_)
+        return &header_->fields(static_cast<int>(pos));
+    else
+        return &fields_.at(pos);
+}
+
+size_t VariantEncodedFieldCollection::size() const {
+    if (is_proto_)
+        return header_->fields_size();
+    else
+        return fields_.size();
+}
+
+} //namespace arcticdb
diff --git a/cpp/arcticdb/codec/variant_encoded_field_collection.hpp b/cpp/arcticdb/codec/variant_encoded_field_collection.hpp
index 91f3c1f36a..41288cab85 100644
--- a/cpp/arcticdb/codec/variant_encoded_field_collection.hpp
+++ b/cpp/arcticdb/codec/variant_encoded_field_collection.hpp
@@ -7,7 +7,7 @@
 
 #pragma once
 
-#include <codec/encoded_field_collection.hpp>
+#include <arcticdb/codec/encoded_field_collection.hpp>
 
 namespace arcticdb {
 
@@ -17,22 +17,9 @@ struct VariantEncodedFieldCollection {
     EncodedFieldCollection fields_;
     const arcticdb::proto::encoding::SegmentHeader *header_ = nullptr;
     bool is_proto_ = false;
-
     explicit VariantEncodedFieldCollection(const Segment &segment);
-
-    [[nodiscard]] VariantField at(size_t pos) const {
-        if (is_proto_)
-            return &header_->fields(static_cast<int>(pos));
-        else
-            return &fields_.at(pos);
-    }
-
-    [[nodiscard]] size_t size() const {
-        if (is_proto_)
-            return header_->fields_size();
-        else
-            return fields_.size();
-    }
+    [[nodiscard]] VariantField at(size_t pos) const;
+    [[nodiscard]] size_t size() const;
 };
 
-} // namespace arcticdb
\ No newline at end of file
+} // namespace arcticdb
diff --git a/cpp/arcticdb/codec/zstd.hpp b/cpp/arcticdb/codec/zstd.hpp
index b225b3a361..712c26543d 100644
--- a/cpp/arcticdb/codec/zstd.hpp
+++ b/cpp/arcticdb/codec/zstd.hpp
@@ -49,9 +49,6 @@ struct ZstdBlockEncoder {
     }
 };
 
-template<template<typename> class F, class TD>
-using ZstdEncoder = GenericBlockEncoder<F<TD>, TD, ZstdBlockEncoder>;
-
 struct ZstdDecoder {
 
 /*
@@ -78,4 +75,4 @@ struct ZstdDecoder {
 #pragma GCC diagnostic pop
 };
 
-} // namespace arcticdb::detail
\ No newline at end of file
+} // namespace arcticdb::detail
diff --git a/cpp/arcticdb/column_store/chunked_buffer.hpp b/cpp/arcticdb/column_store/chunked_buffer.hpp
index 1eaabad1a2..761330887b 100644
--- a/cpp/arcticdb/column_store/chunked_buffer.hpp
+++ b/cpp/arcticdb/column_store/chunked_buffer.hpp
@@ -49,6 +49,8 @@ class ChunkedBufferImpl {
     static_assert(DefaultBlockSize >= BlockType::MinSize);
 
   public:
+    constexpr static size_t block_size = DefaultBlockSize;
+
     struct Iterator {
         ChunkedBufferImpl* parent_;
         BlockType* block_ = nullptr;
diff --git a/cpp/arcticdb/column_store/column.cpp b/cpp/arcticdb/column_store/column.cpp
index 45b7632703..6a40d711ba 100644
--- a/cpp/arcticdb/column_store/column.cpp
+++ b/cpp/arcticdb/column_store/column.cpp
@@ -573,6 +573,21 @@ std::shared_ptr<Column> Column::truncate(const std::shared_ptr<Column>& column,
     return res;
 }
 
+
+void Column::set_empty_array(ssize_t row_offset, int dimension_count) {
+    ARCTICDB_SAMPLE(ColumnSetArray, RMTSF_Aggregate)
+    magic_.check();
+    util::check_arg(last_logical_row_ + 1 == row_offset, "set_array expected row {}, actual {} ", last_logical_row_ + 1, row_offset);
+    shapes_.ensure<shape_t>(dimension_count);
+    memset(shapes_.ptr(), 0, dimension_count * sizeof(shape_t));
+    shapes_.commit();
+    ++last_logical_row_;
+}
+
+void Column::set_type(TypeDescriptor td) {
+    type_ = td;
+}
+
 // Column private methods
 position_t Column::last_offset() const {
     return offsets_.empty() ? 0 : *offsets_.rbegin();
diff --git a/cpp/arcticdb/column_store/column.hpp b/cpp/arcticdb/column_store/column.hpp
index 11c523f9e5..d4c2439fbf 100644
--- a/cpp/arcticdb/column_store/column.hpp
+++ b/cpp/arcticdb/column_store/column.hpp
@@ -217,7 +217,11 @@ class Column {
         size_t expected_rows,
         bool presize,
         bool allow_sparse) :
-            data_(expected_rows * get_type_size(type.data_type()), presize),
+            // Array types are stored on disk as flat sequences. The python layer cannot work with this. We need to pass
+            // it pointers to an array type (at numpy arrays at the moment). When we allocate a column for an array we
+            // need to allocate space for one pointer per row. This also affects how we handle arrays to python as well.
+            // Check cpp/arcticdb/column_store/column_utils.hpp::array_at and cpp/arcticdb/column_store/column.hpp::Column
+            data_(expected_rows * (type.dimension() == Dimension::Dim0 ? get_type_size(type.data_type()) : sizeof(void*)), presize),
             type_(type),
             allow_sparse_(allow_sparse){
         ARCTICDB_TRACE(log::inmem(), "Creating column with descriptor {}", type);
@@ -306,7 +310,7 @@ class Column {
 
     template<class T, std::enable_if_t<std::is_integral_v<T> || std::is_floating_point_v<T>, int> = 0>
     inline void set_sparse_block(ssize_t row_offset, T *ptr, size_t rows_to_write) {
-        util::check(row_offset == 0, "Cannot write sparse column  with existing data");
+        util::check(row_offset == 0, "Cannot write sparse column with existing data");
         auto new_buffer = util::scan_floating_point_to_sparse(ptr, rows_to_write, sparse_map());
         std::swap(data_.buffer(), new_buffer);
     }
@@ -322,6 +326,14 @@ class Column {
         sparse_map_ = std::move(bitset);
     }
 
+    ChunkedBuffer&& release_buffer() {
+        return std::move(data_.buffer());
+    }
+
+    Buffer&& release_shapes() {
+        return std::move(shapes_.buffer());
+    }
+
     template<class T, template<class> class Tensor, std::enable_if_t<
             std::is_integral_v<T> || std::is_floating_point_v<T>,
             int> = 0>
@@ -345,7 +357,7 @@ class Column {
     template<class T, std::enable_if_t< std::is_integral_v<T> || std::is_floating_point_v<T>, int> = 0>
     void set_array(ssize_t row_offset, py::array_t<T>& val) {
         ARCTICDB_SAMPLE(ColumnSetArray, RMTSF_Aggregate)
-            magic_.check();
+        magic_.check();
         util::check_arg(last_logical_row_ + 1 == row_offset, "set_array expected row {}, actual {} ", last_logical_row_ + 1, row_offset);
         data_.ensure_bytes(val.nbytes());
         shapes_.ensure<shape_t>(val.ndim());
@@ -360,8 +372,11 @@ class Column {
         ++last_logical_row_;
     }
 
+    void set_empty_array(ssize_t row_offset, int dimension_count);
+    void set_type(TypeDescriptor td);
     ssize_t last_row() const;
 
+
     void check_magic() const;
 
     void unsparsify(size_t num_rows);
@@ -402,6 +417,7 @@ class Column {
 
     void set_shapes_buffer(size_t row_count);
 
+
     // The following two methods inflate (reduplicate) numpy string arrays that are potentially multi-dimensional,
     // i.e where the value is not a string but an array of strings
     void inflate_string_array(const TensorType<OffsetString::offset_t> &string_refs,
@@ -476,19 +492,6 @@ class Column {
         return orig_type_.value();
     }
 
-    void set_secondary_type(TypeDescriptor type) {
-        secondary_type_ = std::move(type);
-    }
-
-    bool has_secondary_type() const {
-        return secondary_type_.has_value();
-    }
-
-    const TypeDescriptor& secondary_type() const {
-        util::check(secondary_type_.has_value(), "Requesting secondary type in a column that does not have one");
-        return *secondary_type_;
-    }
-
     void compact_blocks() {
         data_.compact_blocks();
     }
@@ -545,7 +548,8 @@ class Column {
                 ndim,
                 type().data_type(),
                 get_type_size(type().data_type()),
-                reinterpret_cast<const T*>(data_.buffer().ptr_cast<uint8_t>(bytes_offset(idx), calc_elements(shape_ptr, ndim))));
+                reinterpret_cast<const T*>(data_.buffer().ptr_cast<uint8_t>(bytes_offset(idx), calc_elements(shape_ptr, ndim))),
+                ndim);
     }
 
     template<typename T>
@@ -665,8 +669,6 @@ class Column {
     bool inflated_ = false;
     bool allow_sparse_ = false;
     std::optional<util::BitMagic> sparse_map_;
-    // For types that are arrays or matrices, the member type
-    std::optional<TypeDescriptor> secondary_type_;
     util::MagicNum<'D', 'C', 'o', 'l'> magic_;
 }; //class Column
 
diff --git a/cpp/arcticdb/column_store/column_data.cpp b/cpp/arcticdb/column_store/column_data.cpp
new file mode 100644
index 0000000000..ac6aea363c
--- /dev/null
+++ b/cpp/arcticdb/column_store/column_data.cpp
@@ -0,0 +1,18 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+#include "column_data.hpp"
+
+namespace arcticdb {
+    bool ColumnData::current_tensor_is_empty() const {
+        return shape_pos_ < shapes_->bytes() && *shapes_->ptr_cast<shape_t>(shape_pos_, sizeof(shape_t)) == 0;
+    }
+
+    const Buffer* ColumnData::shapes() const noexcept {
+        return shapes_;
+    }
+}
diff --git a/cpp/arcticdb/column_store/column_data.hpp b/cpp/arcticdb/column_store/column_data.hpp
index 4b9c26c1f2..06d087659a 100644
--- a/cpp/arcticdb/column_store/column_data.hpp
+++ b/cpp/arcticdb/column_store/column_data.hpp
@@ -70,11 +70,12 @@ struct TypedBlockData {
         ValueType* ptr_;
     };
 
+    using TypeDescriptorTag = TDT;
     using raw_type = typename TDT::DataTypeTag::raw_type;
 
     ARCTICDB_MOVE_COPY_DEFAULT(TypedBlockData)
 
-    TypedBlockData(const raw_type *data, const shape_t *shapes, size_t nbytes, size_t row_count, MemBlock *block) :
+    TypedBlockData(const raw_type *data, const shape_t *shapes, size_t nbytes, size_t row_count, const MemBlock *block) :
         data_(data),
         shapes_(shapes),
         nbytes_(nbytes),
@@ -83,7 +84,7 @@ struct TypedBlockData {
         {}
 
 
-    TypedBlockData(size_t nbytes, shape_t* shapes) :
+    TypedBlockData(size_t nbytes, const shape_t* shapes) :
         data_(nullptr),
         shapes_(shapes),
         nbytes_(nbytes),
@@ -215,6 +216,9 @@ struct ColumnData {
         };
     }
 
+    /// @brief Get non-owning pointer to the shapes array for the column
+    [[nodiscard]] const Buffer* shapes() const noexcept;
+
   private:
     template<typename TDT>
     TypedBlockData<TDT> next_typed_block(MemBlock* block) {
@@ -222,23 +226,36 @@ struct ColumnData {
         const shape_t *shape_ptr = nullptr;
 
         constexpr auto dim = TDT::DimensionTag::value;
-        if constexpr (dim == Dimension::Dim0){
+        if constexpr (dim == Dimension::Dim0) {
             num_elements = block->bytes() / get_type_size(type_.data_type());
-        }  else {
+        } else {
             if (shapes_->empty()) {
                 num_elements = block->bytes() / get_type_size(type_.data_type());
             } else {
                 shape_ptr = shapes_->ptr_cast<shape_t>(shape_pos_, sizeof(shape_t));
                 size_t size = 0;
                 constexpr auto raw_type_sz = sizeof(typename TDT::DataTypeTag::raw_type);
-                while (size < block->bytes()) {
-                    if constexpr (dim == Dimension::Dim1)
+                // Blocks can contain empty tensors (empty arrays/matrices). They don't hold any data, however, they
+                // have assigned shapes. In case of an empty tensor the corresponding entries in the shapes array must
+                // be 0. The rationale is that the shapes array describes how many elements are there in a tensor. This
+                // way we can distinguish between [] (empty array) and None (no array at all). We assume that no block,
+                // besides the first, can start with empty tensors. This means that a block is exhausted when both are
+                // conditions are satisfied:
+                // i) The processed size becomes equal to the block size
+                // ii) All zero-sized shapes are parsed (i.e. the shape of the current tensor is not 0)
+                while (current_tensor_is_empty() || size < block->bytes()) {
+                    if constexpr (dim == Dimension::Dim1) {
                         size += next_shape() * raw_type_sz;
-                    else
-                        size += next_shape() * next_shape() * raw_type_sz;
+                    } else {
+                        const shape_t row_count = next_shape();
+                        const shape_t column_count = next_shape();
+                        util::check(
+                            row_count > 0 || (row_count == 0 && column_count == 0),
+                            "Tensor column count must be zero when the row count is 0");
+                        size += row_count * column_count * raw_type_sz;
+                    }
                     ++num_elements;
                 }
-
                 util::check(size == block->bytes(), "Element size vs block size overrun: {} > {}", size, block->bytes());
             }
         }
@@ -252,6 +269,10 @@ struct ColumnData {
         };
     }
 
+    /// @brief Check if the current tensor is of size 0
+    /// @note The size of the current tensor is written in #shapes_ under #shapes_pos_
+    [[nodiscard]] bool current_tensor_is_empty() const;
+
     const ChunkedBuffer* data_;
     const Buffer* shapes_;
     size_t pos_;
@@ -260,4 +281,4 @@ struct ColumnData {
     const util::BitMagic* bit_vector_;
 };
 
-}
\ No newline at end of file
+}
diff --git a/cpp/arcticdb/column_store/column_utils.hpp b/cpp/arcticdb/column_store/column_utils.hpp
index cadf95a0a3..71fc1192a9 100644
--- a/cpp/arcticdb/column_store/column_utils.hpp
+++ b/cpp/arcticdb/column_store/column_utils.hpp
@@ -18,18 +18,18 @@ namespace arcticdb::detail {
 inline py::array array_at(const SegmentInMemory& frame, std::size_t col_pos, py::object &anchor) {
     ARCTICDB_SAMPLE_DEFAULT(PythonOutputFrameArrayAt)
     if (frame.empty()) {
-        return visit_field(frame.field(col_pos), [] (auto &&tag) {
+        return visit_field(frame.field(col_pos), [] (auto tag) {
             using TypeTag = std::decay_t<decltype(tag)>;
             constexpr auto data_type = TypeTag::DataTypeTag::data_type;
             std::string dtype;
-            constexpr ssize_t esize = is_sequence_type(data_type) && is_fixed_string_type(data_type) ? 1 : get_type_size(data_type);
+            ssize_t esize = is_sequence_type(data_type) && is_fixed_string_type(data_type) ? 1 : get_type_size(data_type);
             if constexpr (is_sequence_type(data_type)) {
                 if constexpr (is_fixed_string_type(data_type)) {
                     dtype = data_type == DataType::ASCII_FIXED64 ? "<S0" : "<U0";
                 } else {
                     dtype = "O";
                 }
-            } else if constexpr(is_numeric_type(data_type) || is_bool_type(data_type)) {
+            } else if constexpr((is_numeric_type(data_type) || is_bool_type(data_type)) && tag.dimension() == Dimension::Dim0) {
                 constexpr auto dim = TypeTag::DimensionTag::value;
                 util::check(dim == Dimension::Dim0, "Only scalars supported, {}", data_type);
                 if constexpr (data_type == DataType::NANOSECONDS_UTC64) {
@@ -44,18 +44,30 @@ inline py::array array_at(const SegmentInMemory& frame, std::size_t col_pos, py:
                 } else {
                     dtype = fmt::format("{}{:d}", get_dtype_specifier(data_type), esize);
                 }
-            } else if constexpr (is_empty_type(data_type)) {
+            } else if constexpr (is_empty_type(data_type) || is_py_bool_type(data_type) || is_numpy_array(TypeDescriptor(tag))) {
                 dtype= "O";
+                // The python representation of multidimensional columns differs from the in-memory/on-storage. In memory,
+                // we hold all scalars in a contiguous buffer with the shapes buffer telling us how many elements are there
+                // per array. Each element is of size sizeof(DataTypeTag::raw_type). For the python representation the column
+                // is represented as an array of (numpy) arrays. Each nested arrays is represented as a pointer to the
+                // (numpy) array, thus the size of the element is not the size of the raw type, but the size of a pointer.
+                // This also affects how we allocate columns. Check cpp/arcticdb/column_store/column.hpp::Column and
+                // cpp/arcticdb/pipeline/column_mapping.hpp::sizeof_datatype
+                if constexpr(tag.dimension() > Dimension::Dim0) {
+                    esize = sizeof(PyObject*);
+                }
+            } else if constexpr(tag.dimension() == Dimension::Dim2) {
+                util::raise_rte("Read resulted in two dimensional type. This is not supported.");
             } else {
                 static_assert(!sizeof(data_type), "Unhandled data type");
             }
             return py::array{py::dtype{dtype}, py::array::ShapeContainer{0}, py::array::StridesContainer{esize}};
         });
     }
-    return visit_field(frame.field(col_pos), [&, frame=frame, col_pos=col_pos] (auto &&tag) {
+    return visit_field(frame.field(col_pos), [&, frame=frame, col_pos=col_pos] (auto tag) {
         using TypeTag = std::decay_t<decltype(tag)>;
         constexpr auto data_type = TypeTag::DataTypeTag::data_type;
-        const auto &buffer = frame.column(col_pos).data().buffer();
+        const auto& buffer = frame.column(col_pos).data().buffer();
         std::string dtype;
         ssize_t esize = get_type_size(data_type);
         if constexpr (is_sequence_type(data_type)) {
@@ -69,7 +81,7 @@ inline py::array array_at(const SegmentInMemory& frame, std::size_t col_pos, py:
             } else {
                 dtype = "O";
             }
-        } else if constexpr(is_numeric_type(data_type) || is_bool_type(data_type)) {
+        } else if constexpr((is_numeric_type(data_type) || is_bool_type(data_type)) && tag.dimension() == Dimension::Dim0) {
             constexpr auto dim = TypeTag::DimensionTag::value;
             util::check(dim == Dimension::Dim0, "Only scalars supported, {}", frame.field(col_pos));
             if constexpr (data_type == DataType::NANOSECONDS_UTC64) {
@@ -84,8 +96,20 @@ inline py::array array_at(const SegmentInMemory& frame, std::size_t col_pos, py:
             } else {
                 dtype = fmt::format("{}{:d}", get_dtype_specifier(data_type), esize);
             }
-        } else if constexpr (is_empty_type(data_type)) {
+        } else if constexpr (is_empty_type(data_type) || is_py_bool_type(data_type) || is_numpy_array(TypeDescriptor(tag))){
             dtype = "O";
+            // The python representation of multidimensional columns differs from the in-memory/on-storage. In memory,
+            // we hold all scalars in a contiguous buffer with the shapes buffer telling us how many elements are there
+            // per array. Each element is of size sizeof(DataTypeTag::raw_type). For the python representation the column
+            // is represented as an array of (numpy) arrays. Each nested arrays is represented as a pointer to the
+            // (numpy) array, thus the size of the element is not the size of the raw type, but the size of a pointer.
+            // This also affects how we allocate columns. Check cpp/arcticdb/column_store/column.hpp::Column and
+            // cpp/arcticdb/pipeline/column_mapping.hpp::sizeof_datatype
+            if constexpr(tag.dimension() > Dimension::Dim0) {
+                esize = sizeof(PyObject*);
+            }
+        } else if constexpr(tag.dimension() == Dimension::Dim2) {
+            util::raise_rte("Read resulted in two dimensional type. This is not supported.");
         } else {
             static_assert(!sizeof(data_type), "Unhandled data type");
         }
diff --git a/cpp/arcticdb/column_store/memory_segment.hpp b/cpp/arcticdb/column_store/memory_segment.hpp
index 9527a06c25..612b0b1de1 100644
--- a/cpp/arcticdb/column_store/memory_segment.hpp
+++ b/cpp/arcticdb/column_store/memory_segment.hpp
@@ -285,10 +285,6 @@ class SegmentInMemory {
         impl_->set_sparse_block(idx, std::move(buffer), std::move(bitset));
     }
 
-    void set_secondary_type(position_t idx, TypeDescriptor type) {
-        impl_->set_secondary_type(idx, type);
-    }
-
     void set_offset(ssize_t offset) {
         impl_->set_offset(offset);
     }
diff --git a/cpp/arcticdb/column_store/memory_segment_impl.hpp b/cpp/arcticdb/column_store/memory_segment_impl.hpp
index d70594757d..63b6213f75 100644
--- a/cpp/arcticdb/column_store/memory_segment_impl.hpp
+++ b/cpp/arcticdb/column_store/memory_segment_impl.hpp
@@ -495,10 +495,6 @@ class SegmentInMemoryImpl {
         column_unchecked(idx).set_sparse_block(std::move(buffer), std::move(shapes), std::move(bitset));
     }
 
-    void set_secondary_type(position_t idx, TypeDescriptor type) {
-        column_unchecked(idx).set_secondary_type(type);
-    }
-
     template<class T, std::enable_if_t<std::is_same_v<std::decay_t<T>, std::string>, int> = 0>
         void set_scalar(position_t idx, T val) {
         set_string(idx, val);
diff --git a/cpp/arcticdb/entity/native_tensor.hpp b/cpp/arcticdb/entity/native_tensor.hpp
index 87f89f6894..74cc9f99c6 100644
--- a/cpp/arcticdb/entity/native_tensor.hpp
+++ b/cpp/arcticdb/entity/native_tensor.hpp
@@ -31,37 +31,42 @@ inline ssize_t calc_elements(const shape_t* shape, ssize_t ndim) {
  * This typically stores the data of the numpy array backing a column of a pandas DataFrame.
  */
 struct NativeTensor {
-    static constexpr ssize_t MaxDimensions = 2;
+    static constexpr int MaxDimensions = 2;
     using StrideContainer = std::array<stride_t, MaxDimensions>;
 
+    /// @param ndim The dimension of the tensor as reported by the API ArcticDB used to read it
+    /// @param expanded_dim The dimension of the tensor as ArcticDB should perceive it. E.g. in case a column tensor
+    /// holds pointers to an array it will be reported to be of dim1 (the column is an array of pointers) while we think
+    /// of it as a dim2 tensor (a column is an array of arrays).
     NativeTensor(
-               ssize_t nbytes,
-               ssize_t ndim,
-               const stride_t *strides,
-               const shape_t *shapes,
-               DataType dt,
-               ssize_t elsize,
-               const void *ptr) :
+        int64_t nbytes,
+        int ndim,
+        const stride_t* strides,
+        const shape_t* shapes,
+        DataType dt,
+        stride_t elsize,
+        const void* ptr,
+        int expanded_dim
+    ) :
         nbytes_(nbytes),
         ndim_(ndim),
         dt_(dt),
         elsize_(elsize),
-        ptr(ptr) {
+        ptr(ptr),
+        expanded_dim_(expanded_dim){
         util::check(shapes != nullptr, "Unexpected null shapes ptr");
         if(shapes[0] == 0)
             ARCTICDB_DEBUG(log::version(), "Supplied tensor is empty");
-        
-        strides_[ndim - 1] = static_cast<ssize_t>(get_type_size(dt_));
-        for(ssize_t i = 0; i < std::min(MaxDimensions, ndim); ++i)
+
+        for (ssize_t i = 0; i < std::min(MaxDimensions, ndim); ++i)
             shapes_[i] = shapes[i];
 
         if(strides == nullptr) {
             strides_[ndim - 1] = static_cast<ssize_t>(get_type_size(dt_));
             if(ndim == 2)
                 strides_[0] = strides_[1] * shapes_[1];
-        }
-        else {
-            for(ssize_t i = 0; i < std::min(MaxDimensions, ndim); ++i)
+        } else {
+            for (ssize_t i = 0; i < std::min(MaxDimensions, ndim); ++i)
                 strides_[i] = strides[i];
         }
     }
@@ -71,7 +76,8 @@ struct NativeTensor {
     ndim_(other.ndim_),
     dt_(other.dt_),
     elsize_(other.elsize_),
-    ptr(other.ptr) {
+    ptr(other.ptr),
+    expanded_dim_(other.expanded_dim_){
         for (ssize_t i = 0; i < std::min(MaxDimensions, ndim_); ++i)
             shapes_[i] = other.shapes_[i];
 
@@ -87,6 +93,7 @@ struct NativeTensor {
         swap(left.dt_, right.dt_);
         swap(left.elsize_, right.elsize_);
         swap(left.ptr, right.ptr);
+        swap(left.expanded_dim_, right.expanded_dim_);
         for(ssize_t i = 0; i < MaxDimensions; ++i) {
             swap(left.shapes_[i], right.shapes_[i]);
             swap(left.strides_[i], right.strides_[i]);
@@ -99,22 +106,22 @@ struct NativeTensor {
         return *this;
     }
 
-    [[nodiscard]] ssize_t nbytes() const { return nbytes_; }
-    [[nodiscard]] ssize_t ndim() const { return ndim_; }
-    [[nodiscard]] stride_t strides(size_t pos) const { return strides_[pos]; }
-    [[nodiscard]] const stride_t* strides() const { return strides_.data(); };
-    [[nodiscard]] shape_t shape(size_t pos) const { return shapes_[pos]; }
-    [[nodiscard]] ssize_t elsize() const { return elsize_; }
-    [[nodiscard]] const shape_t* shape() const { return shapes_.data(); }
-    [[nodiscard]] DataType data_type() const { return dt_; }
+    [[nodiscard]] auto nbytes() const { return nbytes_; }
+    [[nodiscard]] auto ndim() const { return ndim_; }
+    [[nodiscard]] auto strides(size_t pos) const { return strides_[pos]; }
+    [[nodiscard]] const auto* strides() const { return strides_.data(); };
+    [[nodiscard]] auto shape(size_t pos) const { return shapes_[pos]; }
+    [[nodiscard]] auto elsize() const { return elsize_; }
+    [[nodiscard]] const auto* shape() const { return shapes_.data(); }
+    [[nodiscard]] auto data_type() const { return dt_; }
     [[nodiscard]] const void* data() const { magic_.check(); return ptr; }
-    [[nodiscard]] ssize_t extent(ssize_t dim) const { return shapes_[dim] * strides_[dim]; }
-
+    [[nodiscard]] auto extent(ssize_t dim) const { return shapes_[dim] * strides_[dim]; }
+    [[nodiscard]] auto expanded_dim() const { return expanded_dim_; }
     template<typename T>
     const T *ptr_cast(size_t pos) const {
-        bool dimension_condition = ndim() == 1;
-        bool elsize_condition = elsize_ != 0;
-        bool strides_condition = strides_[0] % elsize_ == 0;
+        const bool dimension_condition = ndim() == 1;
+        const bool elsize_condition = elsize_ != 0;
+        const bool strides_condition = strides_[0] % elsize_ == 0;
         util::warn(dimension_condition, "Cannot safely ptr_cast matrices in NativeTensor");
         util::warn(elsize_condition, "Cannot safely ptr_cast when elsize_ is zero in NativeTensor");
         util::warn(strides_condition,
@@ -128,20 +135,24 @@ struct NativeTensor {
     }
 
     // returns number of elements, not bytesize
-    ssize_t size() const {
+    [[nodiscard]] ssize_t size() const {
         return calc_elements(shape(), ndim());
     }
 
     NativeTensor &request() { return *this; }
 
     util::MagicNum<'T','n','s','r'> magic_;
-    ssize_t nbytes_;
-    ssize_t ndim_;
+    int64_t nbytes_;
+    int ndim_;
     StrideContainer strides_ = {};
     StrideContainer shapes_ = {};
     DataType dt_;
-    ssize_t elsize_;
+    stride_t elsize_;
     const void *ptr;
+    /// @note: when iterating strides and shapes we should use the ndim as it is the dimension reported by the
+    /// API providing the strides and shapes arrays, expanded_dim is what ArcticDB thinks of the tensor and using it
+    /// can lead to out of bounds reads from strides and shapes.
+    int expanded_dim_;
 };
 
 template <ssize_t> ssize_t byte_offset_impl(const stride_t* ) { return 0; }
@@ -183,8 +194,9 @@ struct TypedTensor : public NativeTensor {
         return *(static_cast<const T*>(NativeTensor::data()) + byte_offset(ssize_t(index)...) / itemsize());
     }
 
-    TypedTensor(const shape_t* shapes, ssize_t ndim, DataType dt,  ssize_t elsize, const T* data) :
-        NativeTensor(calc_elements(shapes, ndim) * itemsize(), ndim, nullptr, shapes, dt, elsize, data) {
+    /// @param expanded_dim @see NativeTensor::NativeTensor for information about the difference between ndim and expanded_dim
+    TypedTensor(const shape_t* shapes, ssize_t ndim, DataType dt,  ssize_t elsize, const T* data, ssize_t expanded_dim) :
+        NativeTensor(calc_elements(shapes, ndim) * itemsize(), ndim, nullptr, shapes, dt, elsize, data, expanded_dim) {
     }
 
     explicit TypedTensor(const NativeTensor& tensor) :
@@ -193,15 +205,16 @@ struct TypedTensor : public NativeTensor {
     }
 
     TypedTensor(const NativeTensor& tensor, ssize_t slice_num, ssize_t regular_slice_size, ssize_t nvalues) :
-            NativeTensor(
-                    nvalues * itemsize(),
-                    tensor.ndim(),
-                    tensor.strides(),
-                    tensor.shape(),
-                    tensor.data_type(),
-                    tensor.elsize(),
-                    nullptr
-                    ) {
+        NativeTensor(
+            nvalues * itemsize(),
+            tensor.ndim(),
+            tensor.strides(),
+            tensor.shape(),
+            tensor.data_type(),
+            tensor.elsize(),
+            nullptr,
+            tensor.expanded_dim()
+        ) {
 
         ssize_t stride_offset;
         if(ndim() > 1) {
@@ -214,8 +227,7 @@ struct TypedTensor : public NativeTensor {
             auto divisor = calc_elements(shape(), ndim()) / nvalues;
             shapes_[0] /= divisor;
             stride_offset = strides(0) * (shape(0));
-        }
-        else {
+        } else {
             shapes_[0] = nvalues;
             stride_offset = regular_slice_size * strides(0);
         }
@@ -236,4 +248,4 @@ py::array to_py_array(const TypedTensor<T>& tensor) {
 template<typename T>
 using TensorType = TypedTensor<T>;
 
-}//namespace arcticdb
\ No newline at end of file
+}//namespace arcticdb
diff --git a/cpp/arcticdb/entity/stream_descriptor.hpp b/cpp/arcticdb/entity/stream_descriptor.hpp
index d484fb6cd6..d02f1cba1b 100644
--- a/cpp/arcticdb/entity/stream_descriptor.hpp
+++ b/cpp/arcticdb/entity/stream_descriptor.hpp
@@ -221,6 +221,10 @@ struct StreamDescriptor {
         fields_->erase_field(field);
     }
 
+    FieldCollection& mutable_fields() {
+        return *fields_;
+    }
+
     [[nodiscard]] const Field& fields(size_t pos) const {
         return fields_->at(pos);
     }
diff --git a/cpp/arcticdb/entity/type_utils.hpp b/cpp/arcticdb/entity/type_utils.hpp
index a702962c08..13804582f7 100644
--- a/cpp/arcticdb/entity/type_utils.hpp
+++ b/cpp/arcticdb/entity/type_utils.hpp
@@ -15,6 +15,18 @@ inline bool trivially_compatible_types(entity::TypeDescriptor left, entity::Type
     if(left == right)
         return true;
 
+    // Multidimensional types are pointers
+    if(left.dimension() >= entity::Dimension::Dim1 && right.dimension() >= entity::Dimension::Dim1)
+        return true;
+
+    // Multidimensional types are pointer the empty type is pointer as well
+    if(left.dimension() >= entity::Dimension::Dim1 && is_empty_type(right.data_type()))
+        return true;
+    
+    // Multidimensional types are pointer the empty type is pointer as well
+    if(right.dimension() >= entity::Dimension::Dim1 && is_empty_type(left.data_type()))
+        return true;
+
     if(is_sequence_type(left.data_type()) && is_sequence_type(right.data_type())) {
         //TODO coercion of utf strings is not always safe, should allow safe conversion and reinstate the
         //stronger requirement for trivial conversion below.
@@ -24,12 +36,23 @@ inline bool trivially_compatible_types(entity::TypeDescriptor left, entity::Type
         return is_utf_type(slice_value_type(left.data_type()))  == is_utf_type(slice_value_type(right.data_type()));
     }
 
+    if(is_py_bool_type(right.data_type()) && is_py_bool_type(right.data_type())) {
+        return true;
+    }
+
     return false;
 }
 
 inline std::optional<entity::TypeDescriptor> has_valid_type_promotion(entity::TypeDescriptor source, entity::TypeDescriptor target) {
-    if(source.dimension() != target.dimension())
+
+    if(source.dimension() != target.dimension()) {
+        // Empty of dimension 0 means lack of any given type and can be promoted to anything (even if the dimensions don't
+        // match), e.g. empty type can become int or array of ints. Empty type of higher dimension is used to specify an
+        // empty array or an empty matrix, thus it cannot become any other type unless the dimensionality matches
+        if(is_empty_type(source.data_type()) && source.dimension() == entity::Dimension::Dim0)
+            return target;
         return std::nullopt;
+    }
 
     if (source == target)
         return target;
diff --git a/cpp/arcticdb/entity/types-inl.hpp b/cpp/arcticdb/entity/types-inl.hpp
index 06ab5925e6..d9d80c87e0 100644
--- a/cpp/arcticdb/entity/types-inl.hpp
+++ b/cpp/arcticdb/entity/types-inl.hpp
@@ -39,8 +39,10 @@ auto visit_dim(DataType dt, Callable &&c) {
         DT_CASE(UTF_FIXED64)
         DT_CASE(UTF_DYNAMIC64)
         DT_CASE(EMPTYVAL)
+        DT_CASE(PYBOOL8)
+        DT_CASE(PYBOOL64)
 #undef DT_CASE
-    default: util::raise_rte("Invalid dtype '{}' in visit dim", datatype_to_str(dt));
+        default: util::raise_rte("Invalid dtype '{}' in visit dim", datatype_to_str(dt));
     }
 }
 
@@ -66,6 +68,7 @@ auto visit_type(DataType dt, Callable &&c) {
         DT_CASE(UTF_FIXED64)
         DT_CASE(UTF_DYNAMIC64)
         DT_CASE(EMPTYVAL)
+        DT_CASE(PYBOOL8)
 #undef DT_CASE
     default: util::raise_rte("Invalid dtype '{}' in visit type", datatype_to_str(dt));
     }
@@ -76,13 +79,10 @@ auto visit_type(DataType dt, Callable &&c) {
 template<class Callable>
 auto TypeDescriptor::visit_tag(Callable &&callable) const {
     switch (dimension_) {
-#define DIM_CASE(__D__) case Dimension::__D__: \
-    return details::visit_dim<DimensionTag<Dimension::__D__>>(data_type_, callable)
-        DIM_CASE(Dim0);
-        DIM_CASE(Dim1);
-        DIM_CASE(Dim2);
-#undef DIM_CASE
-        default:throw std::invalid_argument(fmt::format("Invalid dimension %d", static_cast<uint32_t>(dimension_)));
+        case Dimension::Dim0: return details::visit_dim<DimensionTag<Dimension::Dim0>>(data_type_, callable);
+        case Dimension::Dim1: return details::visit_dim<DimensionTag<Dimension::Dim1>>(data_type_, callable);
+        case Dimension::Dim2: return details::visit_dim<DimensionTag<Dimension::Dim2>>(data_type_, callable);
+        default: throw std::invalid_argument(fmt::format("Invalid dimension %d", static_cast<uint32_t>(dimension_)));
     }
 }
 
diff --git a/cpp/arcticdb/entity/types.cpp b/cpp/arcticdb/entity/types.cpp
index dfc9a74798..5da45658bd 100644
--- a/cpp/arcticdb/entity/types.cpp
+++ b/cpp/arcticdb/entity/types.cpp
@@ -36,10 +36,11 @@ std::string_view datatype_to_str(const DataType dt) {
         TO_STR(UTF_FIXED64)
         TO_STR(UTF_DYNAMIC64)
         TO_STR(EMPTYVAL)
+        TO_STR(PYBOOL8)
+        TO_STR(PYBOOL64)
 #undef TO_STR
         default:return std::string_view("UNKNOWN");
     }
 }
 
 } // namespace arcticdb
-
diff --git a/cpp/arcticdb/entity/types.hpp b/cpp/arcticdb/entity/types.hpp
index ad942d8405..876bd69947 100644
--- a/cpp/arcticdb/entity/types.hpp
+++ b/cpp/arcticdb/entity/types.hpp
@@ -84,9 +84,9 @@ using VersionId = uint64_t;
 using SignedVersionId = int64_t;
 using GenerationId = VersionId;
 using timestamp = int64_t;
-using shape_t = ssize_t;
-using stride_t = ssize_t;
-using position_t = ssize_t;
+using shape_t = int64_t;
+using stride_t = int64_t;
+using position_t = int64_t;
 
 /** The VariantId holds int64 (NumericId) but is also used to store sizes up to uint64, so needs safe conversion */
 inline NumericId safe_convert_to_numeric_id(uint64_t input) {
@@ -133,7 +133,9 @@ enum class ValueType : uint8_t {
     // For instance, Pandas empty series whose types has not been specified is mapping to EMPTY.
     // When data is appended, the column type is inferred from the data and the column is promoted to the inferred type.
     EMPTY = 13,
-    COUNT = 14 // Not a real value type, should not be added to proto descriptor. Used to count the number of items in the enum
+    /// Nullable booleans
+    PYBOOL = 14,
+    COUNT // Not a real value type, should not be added to proto descriptor. Used to count the number of items in the enum
 };
 
 // Sequence types are composed of more than one element
@@ -193,6 +195,16 @@ constexpr SizeBits get_size_bits(uint8_t size) {
     }
 }
 
+[[nodiscard]] constexpr int get_byte_count(SizeBits size_bits) {
+    switch(size_bits) {
+        case SizeBits::S8: return 1;
+        case SizeBits::S16: return 2;
+        case SizeBits::S32: return 4;
+        case SizeBits::S64: return 8;
+        default: util::raise_rte("Unknown size bits");
+    }
+}
+
 namespace detail{
 
 constexpr uint8_t combine_val_bits(ValueType v, SizeBits b = SizeBits::UNKNOWN_SIZE_BITS) {
@@ -219,6 +231,8 @@ enum class DataType : uint8_t {
     UTF_FIXED64 = detail::combine_val_bits(ValueType::UTF8_FIXED, SizeBits::S64),
     UTF_DYNAMIC64 = detail::combine_val_bits(ValueType::UTF_DYNAMIC, SizeBits::S64),
     EMPTYVAL = detail::combine_val_bits(ValueType::EMPTY, SizeBits::S64),
+    PYBOOL8 = detail::combine_val_bits(ValueType::PYBOOL, SizeBits::S8),
+    PYBOOL64 = detail::combine_val_bits(ValueType::PYBOOL, SizeBits::S64),
     UNKNOWN = 0,
 };
 
@@ -272,6 +286,10 @@ constexpr bool is_bool_type(DataType dt) {
     return slice_value_type(dt) == ValueType::BOOL;
 }
 
+constexpr bool is_py_bool_type(DataType dt) {
+    return slice_value_type(dt) == ValueType::PYBOOL;
+}
+
 constexpr bool is_unsigned_type(DataType dt) {
     return slice_value_type(dt) == ValueType::UINT;
 }
@@ -399,6 +417,8 @@ DATA_TYPE_TAG(ASCII_DYNAMIC64, std::uint64_t)
 DATA_TYPE_TAG(UTF_FIXED64, std::uint64_t)
 DATA_TYPE_TAG(UTF_DYNAMIC64, std::uint64_t)
 DATA_TYPE_TAG(EMPTYVAL, std::uint64_t)
+DATA_TYPE_TAG(PYBOOL8, uint8_t)
+DATA_TYPE_TAG(PYBOOL64, std::uint64_t)
 #undef DATA_TYPE_TAG
 
 enum class Dimension : uint8_t {
@@ -464,11 +484,11 @@ struct TypeDescriptor {
         return !(*this == o);
     }
 
-    [[nodiscard]] DataType data_type() const {
+    [[nodiscard]] constexpr DataType data_type() const {
         return data_type_;
     }
 
-    [[nodiscard]] Dimension dimension() const {
+    [[nodiscard]] constexpr Dimension dimension() const {
         return dimension_;
     }
 
@@ -483,8 +503,38 @@ struct TypeDescriptor {
 
         return output;
     }
+
+    void set_size_bits(SizeBits new_size_bits) {
+        data_type_ = combine_data_type(slice_value_type(data_type_), new_size_bits);
+    }
+
+    [[nodiscard]] constexpr int get_type_byte_size() const {
+        return get_byte_count(slice_bit_size(data_type_));
+    }
 };
 
+/// @brief Check if the type must contain data
+/// Some types are allowed not to have any data, e.g. empty arrays or the empty type (which by design denotes the
+/// lack of data).
+/// @return true if the type must contain data, false it it's allowed for the type to have 0 bytes of data
+constexpr bool must_contain_data(TypeDescriptor td) {
+    return !(is_empty_type(td.data_type()) || td.dimension() > Dimension::Dim0);
+}
+
+/// @biref Check if type descriptor corresponds to numpy array type
+/// @important Be sure to match this with the type handler registry in: cpp/arcticdb/python/python_module.cpp#register_type_handlers
+constexpr bool is_numpy_array(TypeDescriptor td) {
+    return (is_numeric_type(td.data_type()) || is_bool_type(td.data_type()) || is_empty_type(td.data_type())) &&
+           (td.dimension() == Dimension::Dim1);
+}
+
+constexpr bool is_pyobject_type(TypeDescriptor td) {
+    return is_dynamic_string_type(slice_value_type(td.data_type())) ||
+           slice_value_type(td.data_type()) == ValueType::PYBOOL ||
+           is_numpy_array(td);
+}
+
+
 inline void set_data_type(DataType data_type, TypeDescriptor& type_desc) {
     type_desc.data_type_ = data_type;
 }
@@ -502,6 +552,14 @@ struct TypeDescriptorTag {
     explicit constexpr operator TypeDescriptor() const {
         return TypeDescriptor{DataTypeTag::data_type, DimensionTag::value};
     }
+
+    [[nodiscard]] constexpr Dimension dimension() const {
+        return DimensionTag::value;
+    }
+
+    [[nodiscard]] constexpr DataType data_type() const {
+        return DataTypeTag::data_type;
+    }
 };
 
 inline DataType get_data_type(const arcticdb::proto::descriptors::TypeDescriptor& type_desc) {
@@ -679,6 +737,10 @@ struct Field {
         return type_;
     }
 
+    [[nodiscard]] TypeDescriptor* mutable_type_desc() {
+        return &type_;
+    }
+
     TypeDescriptor& mutable_type() {
         return type_;
     }
diff --git a/cpp/arcticdb/log/log.hpp b/cpp/arcticdb/log/log.hpp
index f6742d99a4..f68795ce29 100644
--- a/cpp/arcticdb/log/log.hpp
+++ b/cpp/arcticdb/log/log.hpp
@@ -126,4 +126,4 @@ inline std::unordered_map<std::string, spdlog::logger*> get_loggers_by_name() {
     };
 }
 
-} //namespace arcticdb::log
\ No newline at end of file
+} //namespace arcticdb::log
diff --git a/cpp/arcticdb/pipeline/column_mapping.hpp b/cpp/arcticdb/pipeline/column_mapping.hpp
index 656e576379..3279ea7493 100644
--- a/cpp/arcticdb/pipeline/column_mapping.hpp
+++ b/cpp/arcticdb/pipeline/column_mapping.hpp
@@ -12,7 +12,11 @@ namespace arcticdb {
 inline std::size_t sizeof_datatype(const TypeDescriptor &td) {
     return td.visit_tag([](auto dt) {
         using RawType = typename std::decay_t<decltype(dt)>::DataTypeTag::raw_type;
-        return sizeof(RawType);
+        // Array types are stored on disk as flat sequences. The python layer cannot work with this. We need to pass
+        // it pointers to an array type (at numpy arrays at the moment). When we allocate a column for an array we
+        // need to allocate space for one pointer per row. This also affects how we handle arrays to python as well.
+        // Check cpp/arcticdb/column_store/column_utils.hpp::array_at and cpp/arcticdb/column_store/column.hpp
+        return dt.dimension() == Dimension::Dim0 ? sizeof(RawType) : sizeof(void*);
     });
 }
 
@@ -155,4 +159,4 @@ struct StaticColumnMappingIterator {
 };
 
 
-} // namespace arcticdb
\ No newline at end of file
+} // namespace arcticdb
diff --git a/cpp/arcticdb/pipeline/frame_utils.hpp b/cpp/arcticdb/pipeline/frame_utils.hpp
index 38c3912869..d1e99efd06 100644
--- a/cpp/arcticdb/pipeline/frame_utils.hpp
+++ b/cpp/arcticdb/pipeline/frame_utils.hpp
@@ -21,6 +21,7 @@
 #include <arcticdb/pipeline/string_pool_utils.hpp>
 #include <util/flatten_utils.hpp>
 #include <arcticdb/entity/timeseries_descriptor.hpp>
+#include <arcticdb/entity/type_utils.hpp>
 
 namespace arcticdb {
 
@@ -90,11 +91,12 @@ RawType* flatten_tensor(
     return reinterpret_cast<RawType*>(flattened_buffer->data());
 }
 
-template<typename Tensor, typename Aggregator>
+
+template<typename Aggregator>
 std::optional<convert::StringEncodingError> aggregator_set_data(
-    const TypeDescriptor &type_desc,
-    Tensor &tensor,
-    Aggregator &agg,
+    const TypeDescriptor& type_desc,
+    const entity::NativeTensor& tensor,
+    Aggregator& agg,
     size_t col,
     size_t rows_to_write,
     size_t row,
@@ -102,16 +104,18 @@ std::optional<convert::StringEncodingError> aggregator_set_data(
     size_t regular_slice_size,
     bool sparsify_floats
 ) {
-    return type_desc.visit_tag([&](auto &&tag) {
+    return type_desc.visit_tag([&](auto tag) {
         using RawType = typename std::decay_t<decltype(tag)>::DataTypeTag::raw_type;
         constexpr auto dt = std::decay_t<decltype(tag)>::DataTypeTag::data_type;
 
         util::check(type_desc.data_type() == tensor.data_type(), "Type desc {} != {} tensor type", type_desc.data_type(),
                     tensor.data_type());
         util::check(type_desc.data_type() == dt, "Type desc {} != {} static type", type_desc.data_type(), dt);
-        auto c_style = util::is_cstyle_array<RawType>(tensor);
+        const auto c_style = util::is_cstyle_array<RawType>(tensor);
         std::optional<ChunkedBuffer> flattened_buffer;
         if constexpr (is_sequence_type(dt)) {
+            normalization::check<ErrorCode::E_UNIMPLEMENTED_INPUT_TYPE>(tag.dimension() == Dimension::Dim0,
+                "Multidimensional string types are not supported.");
             ARCTICDB_SUBSAMPLE_AGG(SetDataString)
             if (is_fixed_string_type(dt)) {
                 // deduplicate the strings
@@ -167,7 +171,7 @@ std::optional<convert::StringEncodingError> aggregator_set_data(
                     }
                 }
             }
-        } else if constexpr (is_numeric_type(dt) || is_bool_type(dt)) {
+        } else if constexpr ((is_numeric_type(dt) || is_bool_type(dt)) && tag.dimension() == Dimension::Dim0) {
             auto ptr = tensor.template ptr_cast<RawType>(row);
             if (sparsify_floats) {
                 if constexpr (is_floating_point_type(dt)) {
@@ -190,7 +194,102 @@ std::optional<convert::StringEncodingError> aggregator_set_data(
                     agg.set_array(col, t);
                 }
             }
-        }  else if constexpr (!is_empty_type(dt)) {
+        } else if constexpr(is_py_bool_type(dt)) {
+            normalization::check<ErrorCode::E_UNIMPLEMENTED_INPUT_TYPE>(tag.dimension() == Dimension::Dim0,
+                "Multidimensional nullable booleans are not supported");
+            auto data = const_cast<void *>(tensor.data());
+            auto ptr_data = reinterpret_cast<PyObject **>(data);
+            ptr_data += row;
+
+            if (!c_style)
+                ptr_data = flatten_tensor<PyObject*>(flattened_buffer, rows_to_write, tensor, slice_num, regular_slice_size);
+
+            util::BitSet bitset = util::scan_object_type_to_sparse(ptr_data, rows_to_write);
+
+            const auto num_values = bitset.count();
+            auto bool_buffer = ChunkedBuffer::presized(num_values * sizeof(uint8_t));
+            auto en = bitset.first();
+            auto en_end = bitset.end();
+            auto bool_ptr = bool_buffer.ptr_cast<uint8_t>(0u, num_values);
+            while (en < en_end) {
+                *bool_ptr = static_cast<uint8_t>(PyObject_IsTrue(ptr_data[*en]));
+                ++en;
+                ++bool_ptr;
+            }
+            if(bitset.count() > 0)
+                agg.set_sparse_block(col, std::move(bool_buffer), std::move(bitset));
+
+        } else if constexpr(is_numpy_array(TypeDescriptor(tag))) {
+            auto data = const_cast<void*>(tensor.data());
+            const auto ptr_data = reinterpret_cast<PyObject**>(data) + row;
+
+            util::BitSet values_bitset = util::scan_object_type_to_sparse(ptr_data, rows_to_write);
+            util::check(!values_bitset.empty(),
+                "Empty bit set means empty colum and should be processed by the empty column code path.");
+            if constexpr (is_empty_type(dt)) {
+                // If we have a column of type {EMPTYVAL, Dim1} and all values of the bitset are set to 1 this means
+                // that we have a column full of empty arrays. In this case there is no need to proceed further and
+                // store anything on disc. Empty arrays can be reconstructed given the type descriptor. However, if
+                // there is at least one "missing" value this means that we're mixing empty arrays and None values.
+                // In that case we need to save the bitset so that we can distinguish empty array from None during the
+                // read.
+                if(values_bitset.size() == values_bitset.count()) {
+                    Column arr_col{TypeDescriptor{DataType::EMPTYVAL, Dimension::Dim2}, true};
+                    agg.set_sparse_block(col, arr_col.release_buffer(), arr_col.release_shapes(), std::move(values_bitset));
+                    return std::optional<convert::StringEncodingError>();
+                }
+            }
+
+            ssize_t last_logical_row{0};
+            const auto column_type_descriptor = TypeDescriptor{tensor.data_type(), Dimension::Dim2};
+            TypeDescriptor secondary_type = type_desc;
+            Column arr_col{column_type_descriptor, true};
+            for (auto en = values_bitset.first(); en < values_bitset.end(); ++en) {
+                const auto arr_pos = *en;
+                const auto row_tensor = convert::obj_to_tensor(ptr_data[arr_pos]);
+                const auto row_type_descriptor = TypeDescriptor{row_tensor.data_type(), Dimension::Dim1};
+                const std::optional<TypeDescriptor>& common_type = has_valid_common_type(row_type_descriptor, secondary_type);
+                normalization::check<ErrorCode::E_COLUMN_SECONDARY_TYPE_MISMATCH>(
+                    common_type.has_value(),
+                    "Numpy arrays in the same column must be of compatible types {} {}",
+                    datatype_to_str(secondary_type.data_type()),
+                    datatype_to_str(row_type_descriptor.data_type()));
+                secondary_type = *common_type;
+                // TODO: If the input array contains unexpected elements such as None, NaN, string the type
+                //  descriptor will have data_type == BYTES_DYNAMIC64. TypeDescriptor::visit_tag does not have a
+                //  case for it and it will throw exception which is not meaningful. Adding BYTES_DYNAMIC64 in
+                //  TypeDescriptor::visit_tag leads to a bunch of compilation errors spread all over the code.
+                normalization::check<ErrorCode::E_UNIMPLEMENTED_COLUMN_SECONDARY_TYPE>(
+                    is_numeric_type(row_type_descriptor.data_type()) || is_empty_type(row_type_descriptor.data_type()),
+                    "Numpy array type {} is not implemented. Only dense int and float arrays are supported.",
+                    datatype_to_str(row_type_descriptor.data_type())
+                );
+                row_type_descriptor.visit_tag([&arr_col, &row_tensor, &last_logical_row] (auto tdt) {
+                    using ArrayDataTypeTag = typename decltype(tdt)::DataTypeTag;
+                    using ArrayType = typename ArrayDataTypeTag::raw_type;
+                    if constexpr(is_empty_type(ArrayDataTypeTag::data_type)) {
+                        arr_col.set_empty_array(last_logical_row, row_tensor.ndim());
+                    } else if constexpr(is_numeric_type(ArrayDataTypeTag::data_type)) {
+                        if(row_tensor.nbytes()) {
+                            TypedTensor<ArrayType> typed_tensor{row_tensor};
+                            arr_col.set_array(last_logical_row, typed_tensor);
+                        } else {
+                            arr_col.set_empty_array(last_logical_row, row_tensor.ndim());
+                        }
+                    } else {
+                        normalization::raise<ErrorCode::E_UNIMPLEMENTED_COLUMN_SECONDARY_TYPE>(
+                            "Numpy array type is not implemented. Only dense int and float arrays are supported.");
+                    }
+                });
+                last_logical_row++;
+            }
+            arr_col.set_type(TypeDescriptor{secondary_type.data_type(), column_type_descriptor.dimension()});
+            agg.set_sparse_block(col, arr_col.release_buffer(), arr_col.release_shapes(), std::move(values_bitset));
+        } else if constexpr(tag.dimension() == Dimension::Dim2) {
+            normalization::raise<ErrorCode::E_UNIMPLEMENTED_INPUT_TYPE>(
+                "Trying to add matrix of base type {}. Matrix types are not supported.",
+                datatype_to_str(tag.data_type()));
+        } else if constexpr(!is_empty_type(dt)) {
             static_assert(!sizeof(dt), "Unknown data type");
         }
         return std::optional<convert::StringEncodingError>();
diff --git a/cpp/arcticdb/pipeline/input_tensor_frame.hpp b/cpp/arcticdb/pipeline/input_tensor_frame.hpp
index e706699148..ea237e979e 100644
--- a/cpp/arcticdb/pipeline/input_tensor_frame.hpp
+++ b/cpp/arcticdb/pipeline/input_tensor_frame.hpp
@@ -60,31 +60,31 @@ struct InputTensorFrame {
     bool has_index() const { return desc.index().field_count() != 0ULL; }
 
     void set_index_range() {
-            // Fill index range
-            // Note RowCountIndex will normally have an index field count of 0
-            if(num_rows == 0) {
-                index_range.start_  = IndexValue{0};
-                index_range.end_ = IndexValue{0};
-            } else if (desc.index().field_count() == 1) {
-                visit_field(desc.field(0), [&](auto &&tag) {
-                    using DT = std::decay_t<decltype(tag)>;
-                    using RawType = typename DT::DataTypeTag::raw_type;
-                    if constexpr (std::is_integral_v<RawType> || std::is_floating_point_v<RawType>) {
-                        util::check(static_cast<bool>(index_tensor), "Got null index tensor in set_index_range");
-                        util::check(index_tensor->nbytes() > 0, "Empty index tensor");
-                        auto &tensor = index_tensor.value();
-                        auto start_t = tensor.ptr_cast<RawType>(0);
-                        auto end_t = tensor.ptr_cast<RawType>(static_cast<size_t>(tensor.shape(0) - 1));
-                        index_range.start_  = IndexValue(static_cast<timestamp>(*start_t));
-                        index_range.end_ = IndexValue(static_cast<timestamp>(*end_t));
-                    } else
-                        throw std::runtime_error("Unsupported non-integral index type");
+        // Fill index range
+        // Note RowCountIndex will normally have an index field count of 0
+        if(num_rows == 0) {
+            index_range.start_  = IndexValue{0};
+            index_range.end_ = IndexValue{0};
+        } else if (desc.index().field_count() == 1) {
+            visit_field(desc.field(0), [&](auto &&tag) {
+                using DT = std::decay_t<decltype(tag)>;
+                using RawType = typename DT::DataTypeTag::raw_type;
+                if constexpr (std::is_integral_v<RawType> || std::is_floating_point_v<RawType>) {
+                    util::check(static_cast<bool>(index_tensor), "Got null index tensor in set_index_range");
+                    util::check(index_tensor->nbytes() > 0, "Empty index tensor");
+                    auto &tensor = index_tensor.value();
+                    auto start_t = tensor.ptr_cast<RawType>(0);
+                    auto end_t = tensor.ptr_cast<RawType>(static_cast<size_t>(tensor.shape(0) - 1));
+                    index_range.start_  = IndexValue(static_cast<timestamp>(*start_t));
+                    index_range.end_ = IndexValue(static_cast<timestamp>(*end_t));
+                } else
+                    throw std::runtime_error("Unsupported non-integral index type");
                 });
-            } else {
-                index_range.start_  = IndexValue{0};
-                index_range.end_ = IndexValue{static_cast<timestamp>(num_rows) - 1};
-            }
+        } else {
+            index_range.start_  = IndexValue{0};
+            index_range.end_ = IndexValue{static_cast<timestamp>(num_rows) - 1};
         }
+    }
 };
 
 } //namespace arcticdb::pipelines
diff --git a/cpp/arcticdb/pipeline/python_output_frame.cpp b/cpp/arcticdb/pipeline/python_output_frame.cpp
index e8996757e1..797ef7da36 100644
--- a/cpp/arcticdb/pipeline/python_output_frame.cpp
+++ b/cpp/arcticdb/pipeline/python_output_frame.cpp
@@ -39,11 +39,21 @@ PythonOutputFrame::~PythonOutputFrame() {
             auto column_data = column->data();
             column_data.type().visit_tag([&](auto type_desc_tag) {
                 using TDT = decltype(type_desc_tag);
-                if constexpr (is_dynamic_string_type(TDT::DataTypeTag::data_type)) {
-                    while (auto block = column_data.next<TDT>()) {
-                        for (auto item : *block) {
-                          util::check(reinterpret_cast<PyObject *>(item) != nullptr, "Can't delete null item");
-                          Py_DECREF(reinterpret_cast<PyObject *>(item));
+                constexpr auto td = TypeDescriptor(type_desc_tag);
+                if constexpr (is_pyobject_type(TypeDescriptor(type_desc_tag))) {
+                    if constexpr(is_numpy_array(td)) {
+                        auto it = column_data.buffer().iterator(sizeof(PyObject*));
+                        while(!it.finished()) {
+                            util::check(reinterpret_cast<PyObject*>(it.value()) != nullptr, "Can't delete null item");
+                            Py_DECREF(reinterpret_cast<PyObject*>(it.value()));
+                            it.next();
+                        }
+                    } else {
+                        while (auto block = column_data.next<TDT>()) {
+                            for (auto item : *block) {
+                                util::check(reinterpret_cast<PyObject *>(item) != nullptr, "Can't delete null item");
+                                Py_DECREF(reinterpret_cast<PyObject *>(item));
+                            }
                         }
                     }
                 }
diff --git a/cpp/arcticdb/pipeline/read_frame.cpp b/cpp/arcticdb/pipeline/read_frame.cpp
index ba158adffb..393eb5b5f7 100644
--- a/cpp/arcticdb/pipeline/read_frame.cpp
+++ b/cpp/arcticdb/pipeline/read_frame.cpp
@@ -28,7 +28,7 @@
     #include <arcticdb/util/third_party/robin_hood.hpp>
 #endif
 #include <arcticdb/codec/variant_encoded_field_collection.hpp>
-
+#include <arcticdb/util/magic_num.hpp>
 #include <google/protobuf/util/message_differencer.h>
 #include <folly/SpinLock.h>
 #include <folly/gen/Base.h>
@@ -54,6 +54,15 @@ void mark_index_slices(
         column_groups).value();
 }
 
+StreamDescriptor modify_output_sizes(StreamDescriptor&& desc) {
+    for(auto& field : desc.mutable_fields()) {
+        if(is_py_bool_type(field.type().data_type())) {
+            field.mutable_type_desc()->set_size_bits(SizeBits::S64);
+        }
+    }
+    return std::move(desc);
+}
+
 StreamDescriptor get_filtered_descriptor(StreamDescriptor&& descriptor, const std::shared_ptr<FieldCollection>& filter_columns) {
     // We assume here that filter_columns_ will always contain the index.
 
@@ -61,10 +70,10 @@ StreamDescriptor get_filtered_descriptor(StreamDescriptor&& descriptor, const st
     auto index = stream::index_type_from_descriptor(desc);
     return util::variant_match(index, [&desc, &filter_columns] (const auto& idx) {
         if(filter_columns) {
-            return StreamDescriptor{index_descriptor(desc.id(), idx, *filter_columns)};
+            return modify_output_sizes(StreamDescriptor{index_descriptor(desc.id(), idx, *filter_columns)});
         }
         else {
-            return StreamDescriptor{index_descriptor(desc.id(), idx, *desc.fields_ptr())};
+            return modify_output_sizes(StreamDescriptor{index_descriptor(desc.id(), idx, *desc.fields_ptr())});
         }
     });
 }
@@ -104,7 +113,7 @@ size_t get_index_field_count(const SegmentInMemory& frame) {
 const uint8_t* skip_heading_fields(const arcticdb::proto::encoding::SegmentHeader & hdr, const uint8_t*& data) {
     const auto has_magic_numbers = EncodingVersion(hdr.encoding_version()) == EncodingVersion::V2;
     if(has_magic_numbers)
-        check_magic<MetadataMagic>(data);
+        util::check_magic<MetadataMagic>(data);
 
     if (hdr.has_metadata_field()) {
         auto metadata_size = encoding_sizes::ndarray_field_compressed_size(hdr.metadata_field().ndarray());
@@ -113,7 +122,7 @@ const uint8_t* skip_heading_fields(const arcticdb::proto::encoding::SegmentHeade
     }
 
     if(has_magic_numbers)
-        check_magic<DescriptorMagic>(data);
+        util::check_magic<DescriptorMagic>(data);
 
     if(hdr.has_descriptor_field()) {
         auto descriptor_field_size = encoding_sizes::ndarray_field_compressed_size(hdr.descriptor_field().ndarray());
@@ -122,7 +131,7 @@ const uint8_t* skip_heading_fields(const arcticdb::proto::encoding::SegmentHeade
     }
 
     if(has_magic_numbers)
-        check_magic<IndexMagic>(data);
+        util::check_magic<IndexMagic>(data);
 
     if(hdr.has_index_descriptor_field()) {
         auto index_fields_size = encoding_sizes::ndarray_field_compressed_size(hdr.index_descriptor_field().ndarray());
@@ -141,13 +150,14 @@ void decode_string_pool(const arcticdb::proto::encoding::SegmentHeader & hdr, co
 
         // Note that this will decode the entire string pool into a ChunkedBuffer with exactly 1 chunk
         if(EncodingVersion(hdr.encoding_version()) == EncodingVersion::V2)
-            check_magic<StringPoolMagic>(data);
+            util::check_magic<StringPoolMagic>(data);
 
         data += decode_field(string_pool_descriptor().type(),
                        hdr.string_pool_field(),
                        data,
                        context.string_pool(),
-                       bv);
+                       bv,
+                       to_encoding_version(hdr.encoding_version()));
 
         ARCTICDB_TRACE(log::codec(), "Decoded string pool to position {}", data - begin);
     }
@@ -160,7 +170,8 @@ void decode_index_field_impl(
         const uint8_t*& data,
         const uint8_t *begin ARCTICDB_UNUSED,
         const uint8_t* end ARCTICDB_UNUSED,
-        PipelineContextRow &context) {
+        PipelineContextRow &context,
+        EncodingVersion encoding_version) {
     if (get_index_field_count(frame)) {
         if (!context.fetch_index()) {
             // not selected, skip decompression
@@ -186,7 +197,7 @@ void decode_index_field_impl(
                         context.descriptor().fields(0).type(), frame_field_descriptor.type());
 
             std::optional<util::BitMagic> bv;
-            data += decode_field(frame_field_descriptor.type(), field, data, sink, bv);
+            data += decode_field(frame_field_descriptor.type(), field, data, sink, bv, encoding_version);
             util::check(!bv, "Unexpected sparse vector in index field");
             ARCTICDB_DEBUG(log::codec(), "Decoded index column {} to position {}", 0, data - begin);
         }
@@ -194,14 +205,16 @@ void decode_index_field_impl(
 }
 
 void decode_index_field(
-        SegmentInMemory &frame,
-        VariantField variant_field,
-        const uint8_t*& data,
-        const uint8_t *begin ARCTICDB_UNUSED,
-        const uint8_t* end ARCTICDB_UNUSED,
-        PipelineContextRow &context) {
-    util::variant_match(variant_field, [&] (auto field) {
-        decode_index_field_impl(frame, *field, data, begin, end, context);
+    SegmentInMemory& frame,
+    VariantField variant_field,
+    const uint8_t*& data,
+    const uint8_t* begin ARCTICDB_UNUSED,
+    const uint8_t* end ARCTICDB_UNUSED,
+    PipelineContextRow& context,
+    EncodingVersion encoding_version
+) {
+    util::variant_match(variant_field, [&](auto field) {
+        decode_index_field_impl(frame, *field, data, begin, end, context, encoding_version);
     });
 }
 
@@ -212,9 +225,10 @@ void decode_or_expand_impl(
     const EncodedFieldType& encoded_field_info,
     const TypeDescriptor& type_descriptor,
     size_t dest_bytes,
-    std::shared_ptr<BufferHolder> buffers) {
-    if(auto handler = TypeHandlerRegistry::instance()->get_handler(type_descriptor.data_type()); handler) {
-        handler->handle_type(data, dest, VariantField{&encoded_field_info}, type_descriptor, dest_bytes, buffers);
+    std::shared_ptr<BufferHolder> buffers,
+	EncodingVersion encding_version) {
+    if(auto handler = TypeHandlerRegistry::instance()->get_handler(type_descriptor); handler) {
+        handler->handle_type(data, dest, VariantField{&encoded_field_info}, type_descriptor, dest_bytes, std::move(buffers), encding_version);
     } else {
         std::optional<util::BitMagic> bv;
         if (encoded_field_info.has_ndarray() && encoded_field_info.ndarray().sparse_map_bytes() > 0) {
@@ -222,7 +236,7 @@ void decode_or_expand_impl(
             const auto bytes = encoding_sizes::data_uncompressed_size(ndarray);
             ChunkedBuffer sparse{bytes};
             SliceDataSink sparse_sink{sparse.data(), bytes};
-            data += decode_field(type_descriptor, encoded_field_info, data, sparse_sink, bv);
+            data += decode_field(type_descriptor, encoded_field_info, data, sparse_sink, bv, encding_version);
             type_descriptor.visit_tag([dest, dest_bytes, &bv, &sparse](const auto tdt) {
                 using TagType = decltype(tdt);
                 using RawType = typename TagType::DataTypeTag::raw_type;
@@ -238,7 +252,7 @@ void decode_or_expand_impl(
                     util::default_initialize<TagType>(dest + bytes, dest_bytes - bytes);
                 });
             }
-            data += decode_field(type_descriptor, encoded_field_info, data, sink, bv);
+            data += decode_field(type_descriptor, encoded_field_info, data, sink, bv, encding_version);
         }
     }
 }
@@ -265,9 +279,11 @@ void decode_or_expand(
     const VariantField& variant_field,
     const TypeDescriptor& type_descriptor,
     size_t dest_bytes,
-    std::shared_ptr<BufferHolder> buffers) {
-    util::variant_match(variant_field, [&data, dest, &type_descriptor, dest_bytes, buffers] (auto field) {
-        decode_or_expand_impl(data, dest, *field, type_descriptor, dest_bytes, buffers);
+    std::shared_ptr<BufferHolder> buffers,
+    EncodingVersion encoding_version
+) {
+    util::variant_match(variant_field, [&](auto field) {
+        decode_or_expand_impl(data, dest, *field, type_descriptor, dest_bytes, buffers, encoding_version);
     });
 }
 
@@ -306,6 +322,7 @@ bool remaining_fields_empty(IteratorType it, const PipelineContextRow& context)
         if(!is_empty_type(field.type().data_type())) {
             return false;
         }
+        it.advance();
     }
     return true;
 }
@@ -314,7 +331,7 @@ void decode_into_frame_static(
     SegmentInMemory &frame,
     PipelineContextRow &context,
     Segment &&s,
-    std::shared_ptr<BufferHolder> buffers) {
+    const std::shared_ptr<BufferHolder>& buffers) {
     auto seg = std::move(s);
     ARCTICDB_SAMPLE_DEFAULT(DecodeIntoFrame)
     const uint8_t *data = seg.buffer().data();
@@ -326,12 +343,15 @@ void decode_into_frame_static(
     context.set_descriptor(StreamDescriptor{ std::make_shared<StreamDescriptor::Proto>(std::move(*hdr.mutable_stream_descriptor())), seg.fields_ptr() });
     context.set_compacted(hdr.compacted());
     ARCTICDB_DEBUG(log::version(), "Num fields: {}", seg.header().fields_size());
-    const bool has_magic_nums = EncodingVersion(hdr.encoding_version()) == EncodingVersion::V2;
+    const EncodingVersion encoding_version = EncodingVersion(hdr.encoding_version());
+    const bool has_magic_nums = encoding_version == EncodingVersion::V2;
+    VariantEncodedFieldCollection fields(seg);
 
-    if (data != end) {
-        VariantEncodedFieldCollection fields(seg);
+    // data == end in case we have empty data types (e.g. {EMPTYVAL, Dim0}, {EMPTYVAL, Dim1}) for which we store nothing
+    // in storage as they can be reconstructed in the type handler on the read path.
+    if (data != end || fields.size() > 0) {
         auto index_field = fields.at(0u);
-        decode_index_field(frame, index_field, data, begin, end, context);
+        decode_index_field(frame, index_field, data, begin, end, context, encoding_version);
 
         StaticColumnMappingIterator it(context, index_fieldcount);
         if(it.invalid())
@@ -340,7 +360,7 @@ void decode_into_frame_static(
         while (it.has_next()) {
             advance_skipped_cols(data, static_cast<ssize_t>(it.prev_col_offset()), it.source_col(), it.first_slice_col_offset(), index_fieldcount, fields, hdr);
             if(has_magic_nums)
-                check_magic_in_place<ColumnMagic>(data);
+                util::check_magic_in_place<ColumnMagic>(data);
 
             auto encoded_field = fields.at(it.source_field_pos());
             util::check(it.source_field_pos() < size_t(fields.size()), "Field index out of range: {} !< {}", it.source_field_pos(), fields.size());
@@ -354,24 +374,30 @@ void decode_into_frame_static(
                         field_name,
                         it.dest_col(),
                         m.frame_field_descriptor_.name());
-
             util::check(data != end || remaining_fields_empty(it, context), "Reached end of input block with {} fields to decode", it.remaining_fields());
-            decode_or_expand(data, buffer.data() + m.offset_bytes_, encoded_field, m.source_type_desc_,  m.dest_bytes_, buffers);
+            decode_or_expand(data, buffer.data() + m.offset_bytes_, encoded_field, m.source_type_desc_,  m.dest_bytes_, buffers, encoding_version);
             ARCTICDB_TRACE(log::codec(), "Decoded column {} to position {}", field_name, data - begin);
 
             it.advance();
 
             if(it.at_end_of_selected()) {
-                advance_skipped_cols(data, static_cast<ssize_t>(it.prev_col_offset()), it.last_slice_col_offset(), it.first_slice_col_offset(), it.index_fieldcount(), fields, hdr);
+                advance_skipped_cols(data,
+                    static_cast<ssize_t>(it.prev_col_offset()),
+                    it.last_slice_col_offset(),
+                    it.first_slice_col_offset(),
+                    it.index_fieldcount(),
+                    fields,
+                    hdr);
                 break;
             } else {
                 if(has_magic_nums)
-                    check_magic_in_place<ColumnMagic>(data);
+                    util::check_magic_in_place<ColumnMagic>(data);
             }
         }
 
         decode_string_pool(hdr, data, begin, end, context);
     }
+    ARCTICDB_TRACE(log::codec(), "Frame decoded into static schema");
 }
 
 void decode_into_frame_dynamic(
@@ -389,12 +415,13 @@ void decode_into_frame_dynamic(
     data = skip_heading_fields(hdr, data);
     context.set_descriptor(StreamDescriptor{std::make_shared<StreamDescriptor::Proto>(std::move(*hdr.mutable_stream_descriptor())), seg.fields_ptr()});
     context.set_compacted(hdr.compacted());
-    const bool has_magic_numbers = EncodingVersion(hdr.encoding_version()) == EncodingVersion::V2;
+    const EncodingVersion encdoing_version = EncodingVersion(hdr.encoding_version());
+    const bool has_magic_numbers = encdoing_version == EncodingVersion::V2;
 
     if (data != end) {
         VariantEncodedFieldCollection fields(seg);
         auto index_field = fields.at(0u);
-        decode_index_field(frame, index_field, data, begin, end, context);
+        decode_index_field(frame, index_field, data, begin, end, context, encdoing_version);
 
         auto field_count = context.slice_and_key().slice_.col_range.diff() + index_fieldcount;
         for (auto field_col = index_fieldcount; field_col < field_count; ++field_col) {
@@ -413,14 +440,14 @@ void decode_into_frame_dynamic(
                 util::check(static_cast<bool>(has_valid_type_promotion(m.source_type_desc_, m.dest_type_desc_)), "Can't promote type {} to type {} in field {}",
                             m.source_type_desc_, m.dest_type_desc_, m.frame_field_descriptor_.name());
 
-                    m.dest_type_desc_.visit_tag([&buffer, &m, &data, encoded_field, buffers] (auto dest_desc_tag) {
+                    m.dest_type_desc_.visit_tag([&buffer, &m, &data, encoded_field, buffers, encdoing_version] (auto dest_desc_tag) {
                         using DestinationType =  typename decltype(dest_desc_tag)::DataTypeTag::raw_type;
-                        m.source_type_desc_.visit_tag([&buffer, &m, &data, &encoded_field, &buffers] (auto src_desc_tag ) {
+                        m.source_type_desc_.visit_tag([&buffer, &m, &data, &encoded_field, &buffers, encdoing_version] (auto src_desc_tag ) {
                             using SourceType =  typename decltype(src_desc_tag)::DataTypeTag::raw_type;
                             if constexpr(std::is_arithmetic_v<SourceType> && std::is_arithmetic_v<DestinationType>) {
                                 const auto src_bytes = sizeof_datatype(m.source_type_desc_) * m.num_rows_;
                                 Buffer tmp_buf{src_bytes};
-                                decode_or_expand(data, tmp_buf.data(), encoded_field, m.source_type_desc_, src_bytes, buffers);
+                                decode_or_expand(data, tmp_buf.data(), encoded_field, m.source_type_desc_, src_bytes, buffers, encdoing_version);
                                 auto src_ptr = reinterpret_cast<SourceType *>(tmp_buf.data());
                                 auto dest_ptr = reinterpret_cast<DestinationType *>(buffer.data() + m.offset_bytes_);
                                 for (auto i = 0u; i < m.num_rows_; ++i) {
@@ -440,7 +467,7 @@ void decode_into_frame_dynamic(
                             "Reached end of input block with {} fields to decode",
                             field_count - field_col);
 
-                decode_or_expand(data, buffer.data() + m.offset_bytes_, encoded_field, m.source_type_desc_, m.dest_bytes_, buffers);
+                decode_or_expand(data, buffer.data() + m.offset_bytes_, encoded_field, m.source_type_desc_, m.dest_bytes_, buffers, encdoing_version);
             }
             ARCTICDB_TRACE(log::codec(), "Decoded column {} to position {}", frame.field(dst_col).name(), data - begin);
         }
diff --git a/cpp/arcticdb/pipeline/read_frame.hpp b/cpp/arcticdb/pipeline/read_frame.hpp
index 07459a5518..72b2b42f56 100644
--- a/cpp/arcticdb/pipeline/read_frame.hpp
+++ b/cpp/arcticdb/pipeline/read_frame.hpp
@@ -81,7 +81,7 @@ void decode_into_frame_static(
     SegmentInMemory &frame,
     PipelineContextRow &context,
     Segment &&seg,
-    const std::shared_ptr<BufferHolder> buffers
+    const std::shared_ptr<BufferHolder>& buffers
     );
 
 void decode_into_frame_dynamic(
@@ -102,4 +102,4 @@ size_t get_index_field_count(const SegmentInMemory& frame);
 StreamDescriptor get_filtered_descriptor(
         const StreamDescriptor& desc,
         const std::shared_ptr<FieldCollection>& filter_columns);
-} // namespace  arcticdb::pipelines
\ No newline at end of file
+} // namespace  arcticdb::pipelines
diff --git a/cpp/arcticdb/pipeline/write_frame.cpp b/cpp/arcticdb/pipeline/write_frame.cpp
index e7089936b5..e33f24f632 100644
--- a/cpp/arcticdb/pipeline/write_frame.cpp
+++ b/cpp/arcticdb/pipeline/write_frame.cpp
@@ -150,7 +150,7 @@ write_frame(
     // Write the keys of the slices into an index segment
     ARCTICDB_SUBSAMPLE_DEFAULT(WriteIndex)
     return std::move(fut_slice_keys).thenValue([frame = std::move(frame), key = std::move(key), &store](auto&& slice_keys) mutable {
-        return index::write_index(std::move(frame), std::move(slice_keys), key, store);
+        return index::write_index(std::move(frame), std::forward<decltype(slice_keys)>(slice_keys), key, store);
     });
 }
 
diff --git a/cpp/arcticdb/processing/aggregation.cpp b/cpp/arcticdb/processing/aggregation.cpp
index ab431f245f..f3cd86cbe6 100644
--- a/cpp/arcticdb/processing/aggregation.cpp
+++ b/cpp/arcticdb/processing/aggregation.cpp
@@ -117,6 +117,16 @@ namespace
     struct OutputType<DataTypeTag<DataType::EMPTYVAL>, void> {
         using type = ScalarTagType<DataTypeTag<DataType::EMPTYVAL>>;
     };
+
+    template<>
+    struct OutputType<DataTypeTag<DataType::PYBOOL8>, void> {
+        using type = ScalarTagType<DataTypeTag<DataType::PYBOOL8>>;
+    };
+
+    template<>
+    struct OutputType<DataTypeTag<DataType::PYBOOL64>, void> {
+        using type = ScalarTagType<DataTypeTag<DataType::PYBOOL64>>;
+    };
 }
 
 /**********************
diff --git a/cpp/arcticdb/python/python_handlers.cpp b/cpp/arcticdb/python/python_handlers.cpp
index ea812c94db..cadf3e58ef 100644
--- a/cpp/arcticdb/python/python_handlers.cpp
+++ b/cpp/arcticdb/python/python_handlers.cpp
@@ -6,46 +6,204 @@
  */
 #include <python/python_handlers.hpp>
 #include <python/gil_lock.hpp>
+#include <arcticdb/codec/slice_data_sink.hpp>
+#include <arcticdb/codec/encoding_sizes.hpp>
 #include <arcticdb/codec/codec.hpp>
 
 namespace arcticdb {
 
-void EmptyHandler::handle_type(
-    const uint8_t *&input,
-    uint8_t *dest,
-    const VariantField &variant_field,
-    const entity::TypeDescriptor &,
-    size_t dest_bytes,
-    std::shared_ptr<BufferHolder>
-) {
-    ARCTICDB_SAMPLE(HandleEmpty, 0)
-    util::check(dest != nullptr, "Got null destination pointer");
-    const size_t num_rows = dest_bytes / get_type_size(DataType::EMPTYVAL);
-    static_assert(get_type_size(DataType::EMPTYVAL) == sizeof(PyObject *));
-    auto ptr_dest = reinterpret_cast<const PyObject **>(dest);
-    py::none none = {};
-    for (auto row = 0u; row < num_rows; ++row) {
-        none.inc_ref();
-        *ptr_dest++ = none.ptr();
+    /// @brief Generate numpy.dtype object from ArcticDB type descriptor
+    /// The dtype is used as type specifier for numpy arrays stored as column elements
+    /// @note There is special handling for ArcticDB's empty type
+    /// When numpy creates an empty array its type is float64. We want to mimic this because:
+    /// i) There is no equivalent to empty value
+    /// ii) We want input dataframes to be exact match of the output and that includes the type
+    [[nodiscard]] static inline py::dtype generate_python_dtype(const TypeDescriptor& td, stride_t type_byte_size) {
+        if(is_empty_type(td.data_type())) {
+            return py::dtype{"f8"};
+        }
+        return py::dtype{fmt::format("{}{:d}", get_dtype_specifier(td.data_type()), type_byte_size)};
+    }
+
+    /// @important This calls pybind's initialize array function which is NOT thread safe. Moreover, numpy arrays can
+    /// be created only by the thread holindg the GIL. In practice we can get away with allocating arrays only from
+    /// a sigle thread (enve if it's not the one holding the GIL). This, however, is not guranteed to work.
+    /// @todo Allocate numpy arrays only from the thread holding the GIL
+    [[nodiscard]] static inline PyObject* initialize_array(
+        const pybind11::dtype& descr,
+        const shape_t shapes,
+        const stride_t strides,
+        const void* source_ptr,
+        std::shared_ptr<Column> owner,
+        std::mutex& creation_mutex
+    ) {
+        std::lock_guard creation_guard{creation_mutex};
+        // TODO: Py capsule can take only void ptr as input. We need a better way to handle destruction
+        //  Allocating shared ptr on the heap is sad.
+        auto* object = new std::shared_ptr<Column>(std::move(owner));
+        auto arr = py::array(descr, {shapes}, {strides}, source_ptr, py::capsule(object, [](void* obj){
+            delete reinterpret_cast<std::shared_ptr<Column>*>(obj);
+        }));
+        return arr.release().ptr();
     }
 
-    util::variant_match(variant_field, [&input](const auto &field) {
-        using EncodedFieldType = std::decay_t<decltype(*field)>;
-        if constexpr (std::is_same_v<EncodedFieldType, arcticdb::EncodedField>)
-            check_magic<ColumnMagic>(input);
-
-        if (field->encoding_case() == EncodedFieldType::kNdarray) {
-            const auto &ndarray_field = field->ndarray();
-            const auto num_blocks = ndarray_field.values_size();
-            util::check(num_blocks <= 1, "Unexpected number of empty type blocks: {}", num_blocks);
-            for (auto block_num = 0; block_num < num_blocks; ++block_num) {
-                const auto &block_info = ndarray_field.values(block_num);
-                input += block_info.out_bytes();
+    static inline const PyObject** fill_with_none(const PyObject** dest, size_t count) {
+        auto none = py::none();
+        std::generate(dest, dest + count, [&none]() { return none.inc_ref().ptr(); });
+        return dest + count;
+    }
+
+    void EmptyHandler::handle_type(
+        const uint8_t*& input,
+        uint8_t* dest,
+        const VariantField& variant_field,
+        const entity::TypeDescriptor&,
+        size_t dest_bytes,
+        std::shared_ptr<BufferHolder>,
+        EncodingVersion
+    ) {
+        ARCTICDB_SAMPLE(HandleEmpty, 0)
+        util::check(dest != nullptr, "Got null destination pointer");
+        const size_t num_rows = dest_bytes / get_type_size(DataType::EMPTYVAL);
+        static_assert(get_type_size(DataType::EMPTYVAL) == sizeof(PyObject *));
+        auto ptr_dest = reinterpret_cast<const PyObject **>(dest);
+        fill_with_none(ptr_dest, num_rows);
+
+        util::variant_match(variant_field, [&input](const auto &field) {
+            using EncodedFieldType = std::decay_t<decltype(*field)>;
+            if constexpr (std::is_same_v<EncodedFieldType, arcticdb::EncodedField>)
+                util::check_magic<ColumnMagic>(input);
+
+            if (field->encoding_case() == EncodedFieldType::kNdarray) {
+                const auto &ndarray_field = field->ndarray();
+                const auto num_blocks = ndarray_field.values_size();
+                util::check(num_blocks <= 1, "Unexpected number of empty type blocks: {}", num_blocks);
+                for (auto block_num = 0; block_num < num_blocks; ++block_num) {
+                    const auto &block_info = ndarray_field.values(block_num);
+                    input += block_info.out_bytes();
+                }
+            } else {
+                util::raise_error_msg("Unsupported encoding {}", *field);
+            }
+        });
+    }
+
+    void BoolHandler::handle_type(
+            const uint8_t*& data,
+            uint8_t* dest,
+            const VariantField& encoded_field_info,
+            const entity::TypeDescriptor& type_descriptor,
+            size_t dest_bytes,
+            std::shared_ptr<BufferHolder>,
+            EncodingVersion encding_version
+    ) {
+        std::visit([&](const auto& field){
+            ARCTICDB_SAMPLE(HandleBool, 0)
+            util::check(dest != nullptr, "Got null destination pointer");
+            util::check(field->has_ndarray(), "Bool handler expected array");
+            ARCTICDB_DEBUG(log::version(), "Bool handler got encoded field: {}", field->DebugString());
+            size_t row_count = dest_bytes / get_type_size(DataType::PYBOOL64);
+            auto ptr_dest = reinterpret_cast<const PyObject**>(dest);
+
+            if(!field->ndarray().sparse_map_bytes()) {
+                ARCTICDB_DEBUG(log::version(), "Bool handler has no values");
+                fill_with_none(ptr_dest, row_count);
+                return;
             }
-        } else {
-            util::raise_error_msg("Unsupported encoding {}", *field);
-        }
-    });
-}
 
-} //namespace arcticdb
\ No newline at end of file
+            std::optional<util::BitSet> bv;
+            const auto& ndarray = field->ndarray();
+            const auto bytes = encoding_sizes::data_uncompressed_size(ndarray);
+            auto sparse = ChunkedBuffer::presized(bytes);
+            SliceDataSink sparse_sink{sparse.data(), bytes};
+            data += decode_field(type_descriptor, *field, data, sparse_sink, bv, encding_version);
+            const auto num_bools = bv->count();
+            auto ptr_src = sparse.template ptr_cast<uint8_t>(0, num_bools * sizeof(uint8_t));
+            auto last_row = 0u;
+            const auto ptr_begin = ptr_dest;
+
+            for(auto en = bv->first(); en < bv->end(); ++en) {
+                const auto current_pos = *en;
+                ptr_dest = fill_with_none(ptr_dest, current_pos - last_row);
+                last_row = current_pos;
+                auto py_bool = py::bool_(static_cast<bool>(*ptr_src++));
+                *ptr_dest++ = py_bool.release().ptr();
+                ++last_row;
+                util::check(ptr_begin + last_row == ptr_dest, "Boolean pointer out of alignment {} != {}", last_row, uintptr_t(ptr_begin + last_row));
+            }
+            fill_with_none(ptr_dest, row_count - last_row);
+            util::check(ptr_begin + last_row == ptr_dest, "Boolean pointer out of alignment at end: {} != {}", last_row, uintptr_t(ptr_begin + last_row));
+        }, encoded_field_info);
+    }
+
+    std::mutex ArrayHandler::initialize_array_mutex;
+
+    void ArrayHandler::handle_type(
+        const uint8_t*& data,
+        uint8_t* dest,
+        const VariantField& encoded_field_info,
+        const entity::TypeDescriptor& type_descriptor,
+        size_t dest_bytes,
+        std::shared_ptr<BufferHolder> buffers,
+		EncodingVersion encoding_version
+    ) {
+        util::variant_match(encoded_field_info, [&](auto field){
+            ARCTICDB_SAMPLE(HandleArray, 0)
+            util::check(field->has_ndarray(), "Expected ndarray in array object handler");
+            const auto row_count = dest_bytes / sizeof(PyObject*);
+
+            auto ptr_dest = reinterpret_cast<const PyObject**>(dest);
+            if(!field->ndarray().sparse_map_bytes()) {
+                log::version().info("Array handler has no values");
+                fill_with_none(ptr_dest, row_count);
+                return;
+            }
+            std::shared_ptr<Column> column = buffers->get_buffer(type_descriptor, true);
+            column->check_magic();
+            log::version().info("Column got buffer at {}", uintptr_t(column.get()));
+            auto bv = std::make_optional(util::BitSet{});
+            data += decode_field(type_descriptor, *field, data, *column, bv, encoding_version);
+
+            auto last_row = 0u;
+            ARCTICDB_SUBSAMPLE(InitArrayAcquireGIL, 0)
+            const auto strides = static_cast<stride_t>(get_type_size(type_descriptor.data_type()));
+            const py::dtype py_dtype = generate_python_dtype(type_descriptor, strides);
+            type_descriptor.visit_tag([&] (auto tdt) {
+                const auto& blocks = column->blocks();
+                if(blocks.empty())
+                    return;
+
+                auto block_it = blocks.begin();
+                const auto* shapes = column->shape_ptr();
+                auto block_pos = 0u;
+                const auto* ptr_src = (*block_it)->data();
+                constexpr stride_t stride = static_cast<TypeDescriptor>(tdt).get_type_byte_size();
+                for (auto en = bv->first(); en < bv->end(); ++en) {
+                    const shape_t shape = shapes ? *shapes : 0;
+                    const auto offset = *en;
+                    ptr_dest = fill_with_none(ptr_dest, offset - last_row);
+                    last_row = offset;
+                    *ptr_dest++ = initialize_array(py_dtype,
+                        shape,
+                        stride,
+                        ptr_src + block_pos,
+                        column,
+                        initialize_array_mutex);
+                    block_pos += shape * stride;
+                    if(shapes) {
+                        ++shapes;
+                    }
+                    if(block_it != blocks.end() && block_pos == (*block_it)->bytes() && ++block_it != blocks.end()) {
+                        ptr_src = (*block_it)->data();
+                        block_pos = 0;
+                    }
+
+                    ++last_row;
+                }
+            });
+
+            ARCTICDB_SUBSAMPLE(ArrayIncNones, 0)
+            fill_with_none(ptr_dest, row_count - last_row);
+        });
+    }
+}
diff --git a/cpp/arcticdb/python/python_handlers.hpp b/cpp/arcticdb/python/python_handlers.hpp
index c12876570c..f6f750791b 100644
--- a/cpp/arcticdb/python/python_handlers.hpp
+++ b/cpp/arcticdb/python/python_handlers.hpp
@@ -12,9 +12,44 @@
 namespace arcticdb {
     namespace py = pybind11;
     struct EmptyHandler {
+        /// @see arcticdb::ITypeHandler
+        void handle_type(
+            const uint8_t*& data,
+            uint8_t* dest,
+            const VariantField& encoded_field,
+            const entity::TypeDescriptor& type_descriptor,
+            size_t dest_bytes,
+            std::shared_ptr<BufferHolder> buffers,
+            EncodingVersion encding_version);
+    };
 
-        EmptyHandler() = default;
+    struct BoolHandler {
+        /// @see arcticdb::ITypeHandler
+        void handle_type(
+            const uint8_t *&data,
+            uint8_t *dest,
+            const VariantField &encoded_field,
+            const entity::TypeDescriptor &type_descriptor,
+            size_t dest_bytes,
+            std::shared_ptr<BufferHolder> buffers,
+            EncodingVersion encding_version);
+    };
+
+    struct DecimalHandler {
+        void handle_type(
+                const uint8_t*& data,
+                uint8_t* dest,
+                const VariantField& encoded_field,
+                const entity::TypeDescriptor& type_descriptor,
+                size_t dest_bytes,
+                std::shared_ptr<BufferHolder> buffers
+        );
+
+        std::shared_ptr<std::mutex> mutex_ = std::make_shared<std::mutex>();
+        std::shared_ptr<py::object> decimal_ = std::make_shared<py::object>(py::module_::import("decimal").attr("Decimal"));
+    };
 
+    struct ArrayHandler {
         /// @see arcticdb::ITypeHandler
         void handle_type(
             const uint8_t*& data,
@@ -22,6 +57,8 @@ namespace arcticdb {
             const VariantField& encoded_field,
             const entity::TypeDescriptor& type_descriptor,
             size_t dest_bytes,
-            std::shared_ptr<BufferHolder> buffers);
+            std::shared_ptr<BufferHolder> buffers,
+            EncodingVersion encding_version);
+        static std::mutex initialize_array_mutex;
     };
 } //namespace arcticdb
diff --git a/cpp/arcticdb/python/python_module.cpp b/cpp/arcticdb/python/python_module.cpp
index 64e63a0807..ab536dae94 100644
--- a/cpp/arcticdb/python/python_module.cpp
+++ b/cpp/arcticdb/python/python_module.cpp
@@ -289,7 +289,14 @@ void register_metrics(py::module && m){
 /// Register handling of non-trivial types. For more information @see arcticdb::TypeHandlerRegistry and
 /// @see arcticdb::ITypeHandler
 void register_type_handlers() {
-    arcticdb::TypeHandlerRegistry::instance()->register_handler(arcticdb::DataType::EMPTYVAL, arcticdb::EmptyHandler());
+    using namespace arcticdb;
+    TypeHandlerRegistry::instance()->register_handler(TypeDescriptor{DataType::EMPTYVAL, Dimension::Dim0}, arcticdb::EmptyHandler());
+    constexpr std::array<DataType, 5> allowed_array_types = {DataType::INT64, DataType::FLOAT64, DataType::EMPTYVAL, DataType::FLOAT32, DataType::INT32};
+    for(const DataType& data_type : allowed_array_types) {
+        TypeHandlerRegistry::instance()->register_handler(TypeDescriptor{data_type, Dimension::Dim1}, arcticdb::ArrayHandler());
+    }
+    TypeHandlerRegistry::instance()->register_handler(TypeDescriptor{DataType::PYBOOL64, Dimension::Dim0}, arcticdb::BoolHandler());
+    TypeHandlerRegistry::instance()->register_handler(TypeDescriptor{DataType::PYBOOL8, Dimension::Dim0}, arcticdb::BoolHandler());
 }
 
 PYBIND11_MODULE(arcticdb_ext, m) {
diff --git a/cpp/arcticdb/python/python_to_tensor_frame.cpp b/cpp/arcticdb/python/python_to_tensor_frame.cpp
index dc1acae214..bdaa656421 100644
--- a/cpp/arcticdb/python/python_to_tensor_frame.cpp
+++ b/cpp/arcticdb/python/python_to_tensor_frame.cpp
@@ -24,20 +24,63 @@ bool is_unicode(PyObject *obj) {
     return PyUnicode_Check(obj);
 }
 
+bool is_py_boolean(PyObject* obj) {
+    return PyBool_Check(obj);
+}
+
 std::variant<StringEncodingError, PyStringWrapper> pystring_to_buffer(PyObject *obj, bool is_owned) {
     if(is_unicode(obj)) {
         return StringEncodingError(
-                fmt::format("Unexpected unicode in Python object with type {}", obj->ob_type->tp_name));
+            fmt::format("Unexpected unicode in Python object with type {}", obj->ob_type->tp_name));
     }
     char *buffer;
     ssize_t length;
     if (PYBIND11_BYTES_AS_STRING_AND_SIZE(obj, &buffer, &length)) {
         return StringEncodingError(fmt::format("Unable to extract string contents from Python object with type {}",
-                                               obj->ob_type->tp_name));
+            obj->ob_type->tp_name));
     }
     return PyStringWrapper(buffer, length, is_owned ? obj : nullptr);
 }
 
+[[nodiscard]] static inline bool is_py_array(PyObject* obj) {
+    const auto& api = pybind11::detail::npy_api::get();
+    return api.PyArray_Check_(obj);
+}
+
+std::tuple<ValueType, uint8_t, ssize_t> determine_python_object_type(PyObject* obj) {
+    if(is_py_boolean(obj))
+        return {ValueType::PYBOOL, 1, 1};
+
+    return {ValueType::BYTES, 8, 1};
+}
+
+/// @brief Determine the type for column composed of arrays
+/// In case column is composed of arrays all arrays must have the same element type. This iterates until if finds the
+/// first non-empty array and returns its type.
+/// @todo We will iterate over all arrays in a column in aggregator_set_data anyways, so this is redundant, however
+///     the type is determined at the point when obj_to_tensor is called. We need to make it possible to change the
+///     the column type in aggregator_set_data in order not to iterate all arrays twice.
+std::tuple<ValueType, uint8_t, ssize_t> determine_python_array_type(PyObject** begin, PyObject** end) {
+        auto none = py::none{};
+        while(begin != end) {
+        if(none.ptr() == *begin) {
+            ++begin;
+            continue;
+        }
+        const auto arr = pybind11::detail::array_proxy(*begin);
+        normalization::check<ErrorCode::E_UNIMPLEMENTED_COLUMN_SECONDARY_TYPE>(arr->nd == 1, "Only one dimensional arrays are supported in columns.");
+        const auto descr = pybind11::detail::array_descriptor_proxy(arr->descr);
+        const ssize_t element_count = arr->dimensions[0];
+        if(element_count != 0) {
+            ValueType value_type = get_value_type(descr->kind);
+            const uint8_t value_bytes = static_cast<uint8_t>(descr->elsize);
+            return {value_type, value_bytes, 2};
+        }
+        begin++;
+    }
+    return {ValueType::EMPTY, 8, 2};
+}
+
 std::variant<StringEncodingError, PyStringWrapper> py_unicode_to_buffer(PyObject *obj, std::optional<ScopedGILLock>& scoped_gil_lock) {
     if(!is_unicode(obj)) {
         return StringEncodingError(
@@ -66,11 +109,10 @@ std::variant<StringEncodingError, PyStringWrapper> py_unicode_to_buffer(PyObject
 }
 
 NativeTensor obj_to_tensor(PyObject *ptr) {
-//    ARCTICDB_SAMPLE(ObjToTensor, RMTSF_Aggregate)
     auto& api = pybind11::detail::npy_api::get();
     util::check(api.PyArray_Check_(ptr), "Expected Python array");
-    auto arr = pybind11::detail::array_proxy(ptr);
-    auto descr = pybind11::detail::array_descriptor_proxy(arr->descr);
+    const auto arr = pybind11::detail::array_proxy(ptr);
+    const auto descr = pybind11::detail::array_descriptor_proxy(arr->descr);
     auto ndim = arr->nd;
     ssize_t size = ndim == 1 ? arr->dimensions[0] : arr->dimensions[0] * arr->dimensions[1];
     // In Pandas < 2, empty series dtype is `"float"`, but as of Pandas 2.0, empty series dtype is `"object"`
@@ -78,21 +120,26 @@ NativeTensor obj_to_tensor(PyObject *ptr) {
     // See: https://github.com/man-group/ArcticDB/pull/1049
     auto val_type = (size == 0 && descr->kind == 'O') ? ValueType::EMPTY : get_value_type(descr->kind);
     auto val_bytes = static_cast<uint8_t>(descr->elsize);
-    auto c_style = arr->strides[0] == val_bytes;
+    const int64_t element_count = ndim == 1 ? int64_t(arr->dimensions[0]) : int64_t(arr->dimensions[0]) * int64_t(arr->dimensions[1]);
+    const auto c_style = arr->strides[0] == val_bytes;
 
-    if (is_sequence_type(val_type)) {
+    if(is_empty_type(val_type)) {
+        val_bytes = 8;
+        val_type = ValueType::EMPTY;
+    } else if (is_sequence_type(val_type)) {
         // wide type always is 64bits
         val_bytes = 8;
 
         // If Numpy has type 'O' then get_value_type above will return type 'BYTES'
-        // If there is no value and we can't deduce a type then leave it that way,
+        // If there is no value, and we can't deduce a type then leave it that way,
         // otherwise try to work out whether it was a bytes (string) type or unicode
-        if (!is_fixed_string_type(val_type) && size > 0) {
+        if (!is_fixed_string_type(val_type) && element_count > 0) {
             auto none = py::none{};
             auto obj = reinterpret_cast<PyObject **>(arr->data);
             bool empty = false;
             bool all_nans = false;
             PyObject *sample = *obj;
+            PyObject** current_object = obj;
             // Arctic allows both None and NaN to represent a string with no value. We have 3 options:
             // * In case all values are None we can mark this column segment as EmptyType and avoid allocating storage
             //      memory for it
@@ -109,6 +156,7 @@ NativeTensor obj_to_tensor(PyObject *ptr) {
                 PyObject** current_object = obj;
                 const auto* end = obj + size;
                 while(current_object < end) {
+
                     if(*current_object == none.ptr()) {
                         all_nans = false;
                     } else if(is_py_nan(*current_object)) {
@@ -123,22 +171,29 @@ NativeTensor obj_to_tensor(PyObject *ptr) {
                 if(current_object != end)
                     sample = *current_object;
             }
-            if (empty) {
+            if (empty && descr->kind == 'O') {
                 val_type = ValueType::EMPTY;
             } else if(all_nans || is_unicode(sample)){
                 val_type = ValueType::UTF_DYNAMIC;
             } else if (PYBIND11_BYTES_CHECK(sample)) {
                 val_type = ValueType::ASCII_DYNAMIC;
+            } else if(is_py_array(sample)) {
+                std::tie(val_type, val_bytes, ndim) = determine_python_array_type(current_object, current_object + element_count);
+            } else {
+                std::tie(val_type, val_bytes, ndim) = determine_python_object_type(sample);
             }
         }
     }
 
     // When processing empty collections, the size bits have to be `SizeBits::S64`,
     // and we can't use `val_bytes` to get this information since some dtype have another `elsize` than 8.
-    SizeBits size_bits = val_type == ValueType::EMPTY ? SizeBits::S64 : get_size_bits(val_bytes);
-    auto dt = combine_data_type(val_type, size_bits);
-    ssize_t nbytes = size * descr->elsize;
-    return {nbytes, ndim, arr->strides, arr->dimensions, dt, descr->elsize, arr->data};
+    const SizeBits size_bits = is_empty_type(val_type) ? SizeBits::S64 : get_size_bits(val_bytes);
+    const auto dt = combine_data_type(val_type, size_bits);
+    const int64_t nbytes = element_count * descr->elsize;
+    const void* data = nbytes ? arr->data : nullptr;
+    const std::array<stride_t, 2> strides = {arr->strides[0], arr->nd > 1 ? arr->strides[1] : 0};
+    const std::array<shape_t, 2> shapes = {arr->dimensions[0], arr->nd > 1 ? arr->dimensions[1] : 0};
+    return {nbytes, arr->nd, strides.data(), shapes.data(), dt, descr->elsize, data, ndim};
 }
 
 InputTensorFrame py_ndf_to_frame(
@@ -198,8 +253,12 @@ InputTensorFrame py_ndf_to_frame(
 
     for (auto i = 0u; i < col_vals.size(); ++i) {
         auto tensor = obj_to_tensor(col_vals[i].ptr());
-        res.num_rows = std::max(res.num_rows, tensor.shape(0));
-        res.desc.add_field(scalar_field(tensor.data_type(), col_names[i]));
+        res.num_rows = std::max(res.num_rows, static_cast<ssize_t>(tensor.shape(0)));
+        if(tensor.expanded_dim() == 1) {
+            res.desc.add_field(scalar_field(tensor.data_type(), col_names[i]));
+        } else if(tensor.expanded_dim() == 2) {
+            res.desc.add_field(FieldRef{TypeDescriptor{tensor.data_type(), Dimension::Dim1}, col_names[i]});
+        }
         res.field_tensors.push_back(std::move(tensor));
     }
 
@@ -228,11 +287,11 @@ InputTensorFrame py_none_to_frame() {
 
     res.set_sorted(sorted);
 
-    ssize_t strides = 8;
-    ssize_t shapes = 1;
-
-    auto tensor = NativeTensor{8, 1, &strides, &shapes, DataType::UINT64, 8, none_char};
-    res.num_rows = std::max(res.num_rows, tensor.shape(0));
+    const stride_t strides = 8;
+    const shape_t shapes = 1;
+    constexpr int ndim = 1;
+    auto tensor = NativeTensor{8, ndim, &strides, &shapes, DataType::UINT64, 8, none_char, ndim};
+    res.num_rows = std::max(res.num_rows, static_cast<ssize_t>(tensor.shape(0)));
     res.desc.add_field(scalar_field(tensor.data_type(), col_name));
     res.field_tensors.push_back(std::move(tensor));
 
diff --git a/cpp/arcticdb/stream/aggregator.hpp b/cpp/arcticdb/stream/aggregator.hpp
index 4a2f21b26d..cb695e337c 100644
--- a/cpp/arcticdb/stream/aggregator.hpp
+++ b/cpp/arcticdb/stream/aggregator.hpp
@@ -224,6 +224,14 @@ class Aggregator {
         segment_.set_sparse_block(pos, val,  rows_to_write);
     }
 
+    void set_sparse_block(position_t idx, ChunkedBuffer&& buffer, util::BitSet&& bitset) {
+        segment_.set_sparse_block(idx, std::move(buffer), std::move(bitset));
+    }
+
+    void set_sparse_block(position_t idx, ChunkedBuffer&& buffer, Buffer&& shapes, util::BitSet&& bitset) {
+        segment_.set_sparse_block(idx, std::move(buffer), std::move(shapes), std::move(bitset));
+    }
+
     void set_string_at(position_t col, position_t row, const char* val, size_t size) {
         segment_.set_string_at(col, row, val, size);
     }
diff --git a/cpp/arcticdb/util/bitset.hpp b/cpp/arcticdb/util/bitset.hpp
index de052c810e..f23fdd45f9 100644
--- a/cpp/arcticdb/util/bitset.hpp
+++ b/cpp/arcticdb/util/bitset.hpp
@@ -22,6 +22,11 @@ using BitMagicEnd = SmallMagicNum<'M', 'e'>;
 using BitSetSizeType = bm::bvector<>::size_type;
 using BitIndex = bm::bvector<>::rs_index_type;
 
+/// @brief Get the combined size of the magic words used as delimiters for the sparse bitmaps
+/// When sparse bitmaps are encoded we use two different magic words to mark the start and the end of the bitmap
+[[nodiscard]] inline constexpr size_t combined_bit_magic_delimiters_size() {
+    return sizeof(BitMagicStart) + sizeof(BitMagicEnd);
+}
 } // namespace util
 
 constexpr bm::bvector<>::size_type bv_size(uint64_t val) {
diff --git a/cpp/arcticdb/util/error_code.hpp b/cpp/arcticdb/util/error_code.hpp
index 20c5d46dfb..429da610c2 100644
--- a/cpp/arcticdb/util/error_code.hpp
+++ b/cpp/arcticdb/util/error_code.hpp
@@ -32,7 +32,8 @@ enum class ErrorCategory : detail::BaseType {
     SORTING = 6,
     USER_INPUT = 7,
     COMPATIBILITY = 8,
-    CODEC = 9,
+    /// Exceptions resulting in failure to encode or decode data while reading/writing from/to disc
+    CODEC = 9
     // NEW CATEGORIES MUST ALSO BE ADDED TO python_module.cpp:register_error_code_ecosystem
 };
 
@@ -59,11 +60,13 @@ inline std::unordered_map<ErrorCategory, const char*> get_error_category_names()
     ERROR_CODE(1002, E_ASSERTION_FAILURE) \
     ERROR_CODE(1003, E_RUNTIME_ERROR) \
     ERROR_CODE(1004, E_STORED_CONFIG_ERROR) \
-    ERROR_CODE(2000, E_INCOMPATIBLE_OBJECTS) \
+    ERROR_CODE(2000, E_INCOMPATIBLE_OBJECTS)\
     ERROR_CODE(2001, E_UNIMPLEMENTED_INPUT_TYPE) \
     ERROR_CODE(2002, E_UPDATE_NOT_SUPPORTED) \
     ERROR_CODE(2003, E_INCOMPATIBLE_INDEX)  \
     ERROR_CODE(2004, E_WRONG_SHAPE) \
+    ERROR_CODE(2005, E_COLUMN_SECONDARY_TYPE_MISMATCH) \
+    ERROR_CODE(2006, E_UNIMPLEMENTED_COLUMN_SECONDARY_TYPE) \
     ERROR_CODE(3000, E_NO_SUCH_VERSION)  \
     ERROR_CODE(3001, E_NO_SYMBOL_DATA)  \
     ERROR_CODE(3010, E_UNREADABLE_SYMBOL_LIST)  \
@@ -71,7 +74,7 @@ inline std::unordered_map<ErrorCategory, const char*> get_error_category_names()
     ERROR_CODE(4001, E_COLUMN_DOESNT_EXIST)  \
     ERROR_CODE(4002, E_UNSUPPORTED_COLUMN_TYPE)  \
     ERROR_CODE(4003, E_UNSUPPORTED_INDEX_TYPE)   \
-    ERROR_CODE(4004, E_OPERATION_NOT_SUPPORTED_WITH_PICKLED_DATA)  \
+    ERROR_CODE(4004, E_OPERATION_NOT_SUPPORTED_WITH_PICKLED_DATA) \
     ERROR_CODE(5000, E_KEY_NOT_FOUND) \
     ERROR_CODE(5001, E_DUPLICATE_KEY) \
     ERROR_CODE(5002, E_SYMBOL_NOT_FOUND) \
@@ -80,8 +83,12 @@ inline std::unordered_map<ErrorCategory, const char*> get_error_category_names()
     ERROR_CODE(7000, E_INVALID_USER_ARGUMENT) \
     ERROR_CODE(7001, E_INVALID_DECIMAL_STRING)   \
     ERROR_CODE(7002, E_INVALID_CHAR_IN_SYMBOL) \
-    ERROR_CODE(8000, E_UNRECOGNISED_COLUMN_STATS_VERSION) \
-    ERROR_CODE(9000, E_DECODE_ERROR)
+    ERROR_CODE(8000, E_UNRECOGNISED_COLUMN_STATS_VERSION)   \
+    ERROR_CODE(9000, E_DECODE_ERROR) \
+    ERROR_CODE(9001, E_UNKNOWN_CODEC) \
+    ERROR_CODE(9002, E_ZSDT_ENCODING) \
+    ERROR_CODE(9003, E_LZ4_ENCODING)  \
+    ERROR_CODE(9004, E_INPUT_TOO_LARGE)
 
 enum class ErrorCode : detail::BaseType {
 #define ERROR_CODE(code, Name, ...) Name = code,
diff --git a/cpp/arcticdb/util/flatten_utils.hpp b/cpp/arcticdb/util/flatten_utils.hpp
index 286fb1d663..1d85873c12 100644
--- a/cpp/arcticdb/util/flatten_utils.hpp
+++ b/cpp/arcticdb/util/flatten_utils.hpp
@@ -90,8 +90,9 @@ class FlattenHelperImpl {
             if (dim == 1) {
                 *dest = *(advance_func_(src, sd, i));
                 dest++;
-            } else
+            } else {
                 flatten(dest, advance_func_(src, sd, i), dim - 1);
+            }
         }
     }
 };
@@ -109,12 +110,11 @@ class FlattenHelper {
         if(has_funky_strides(array_)) {
             FlattenHelperImpl<T, Tensor, stride_advance_conservative<T>> flh{array_};
             flh.flatten(dest, src, array_.ndim());
-        }
-        else {
+        } else {
             FlattenHelperImpl<T, Tensor, stride_advance_optimistic<T>> flh{array_};
             flh.flatten(dest, src, array_.ndim());
         }
     }
 };
 
-} //namespace arcticdb
\ No newline at end of file
+} //namespace arcticdb
diff --git a/cpp/arcticdb/util/magic_num.hpp b/cpp/arcticdb/util/magic_num.hpp
index eb769875a6..07ad429d0f 100644
--- a/cpp/arcticdb/util/magic_num.hpp
+++ b/cpp/arcticdb/util/magic_num.hpp
@@ -25,7 +25,10 @@ struct MagicNum {
         magic_ = ~magic_;
     }
 
-    void check() const { util::check(magic_ == Magic, "Magic number failure, expected {} got {}", Magic, magic_); }
+    void check() const {
+        std::string_view expected(reinterpret_cast<const char*>(&Magic), 4);
+        util::check(magic_ == Magic, "Magic number failure, expected {}({}) got {}({})", Magic, expected, magic_);
+    }
 
   private:
     volatile uint64_t magic_ = Magic;
@@ -43,16 +46,24 @@ struct SmallMagicNum {
 
     [[nodiscard]] uint16_t magic() const { return magic_; }
 
-    void check() const { util::check(magic_ == Magic, "Magic number failure, expected {} got {}", Magic, magic_); }
+    void check() const {
+        std::string_view expected(reinterpret_cast<const char*>(&Magic), 2);
+        util::check(magic_ == Magic, "Magic number failure, expected {}({}) got {}", Magic, expected, magic_);
+    }
 
 private:
     volatile uint16_t magic_ = Magic;
 };
 
 template <typename MagicNumType>
-void check_magic(const uint8_t*& pos) {
+void check_magic_in_place(const uint8_t*& pos) {
     const auto magic_num = reinterpret_cast<const MagicNumType*>(pos);
     magic_num->check();
+}
+
+template <typename MagicNumType>
+void check_magic(const uint8_t*& pos) {
+    check_magic_in_place<MagicNumType>(pos);
     pos += sizeof(MagicNumType);
 }
 
@@ -65,4 +76,3 @@ void write_magic(uint8_t*& pos) {
 
 
 } // namespace arcticdb
-
diff --git a/cpp/arcticdb/util/preconditions.hpp b/cpp/arcticdb/util/preconditions.hpp
index b551735f63..708227f3f4 100644
--- a/cpp/arcticdb/util/preconditions.hpp
+++ b/cpp/arcticdb/util/preconditions.hpp
@@ -127,11 +127,11 @@ namespace compatibility {
 }
 
 namespace codec {
-template<ErrorCode code>
-constexpr auto check = util::detail::Check<code, ErrorCategory::CODEC>{};
+    template<ErrorCode code>
+    constexpr auto check = util::detail::Check<code, ErrorCategory::CODEC>{};
 
-template<ErrorCode code>
-constexpr auto raise = check<code>.raise;
+    template<ErrorCode code>
+    constexpr auto raise = check<code>.raise;
 }
 
 // TODO Change legacy codes to internal::
diff --git a/cpp/arcticdb/util/sparse_utils.cpp b/cpp/arcticdb/util/sparse_utils.cpp
index 433db9f57a..1c6b92a330 100644
--- a/cpp/arcticdb/util/sparse_utils.cpp
+++ b/cpp/arcticdb/util/sparse_utils.cpp
@@ -6,18 +6,17 @@
 */
 
 #include <arcticdb/util/sparse_utils.hpp>
-#include <arcticdb/python/python_to_tensor_frame.hpp>
+#include <pybind11/pybind11.h>
 
 namespace arcticdb::util {
 
-void scan_object_type_to_sparse(
+util::BitSet scan_object_type_to_sparse(
     const PyObject* const* ptr,
-    size_t rows_to_write,
-    util::BitMagic& bitset) {
-
-    namespace py = pybind11;
+    size_t rows_to_write
+) {
+    util::BitSet bitset;
     auto scan_ptr = ptr;
-    py::none none;
+    pybind11::none none;
     util::BitSet::bulk_insert_iterator inserter(bitset);
     for (size_t idx = 0; idx < rows_to_write; ++idx, ++scan_ptr) {
         if(*scan_ptr != none.ptr())
@@ -25,7 +24,7 @@ void scan_object_type_to_sparse(
     }
     inserter.flush();
     bitset.resize(bv_size(rows_to_write));
+    return bitset;
 }
 
 } //namespace arcticdb::util
-
diff --git a/cpp/arcticdb/util/sparse_utils.hpp b/cpp/arcticdb/util/sparse_utils.hpp
index 3eb00a136d..de4456034e 100644
--- a/cpp/arcticdb/util/sparse_utils.hpp
+++ b/cpp/arcticdb/util/sparse_utils.hpp
@@ -86,10 +86,9 @@ void default_initialize(uint8_t* data, size_t bytes) {
     }
 }
 
-void scan_object_type_to_sparse(
+[[nodiscard]] util::BitSet scan_object_type_to_sparse(
     const PyObject* const* ptr,
-    size_t rows_to_write,
-    util::BitMagic& bitset);
+    size_t rows_to_write);
 
 template <typename RawType>
 ChunkedBuffer scan_floating_point_to_sparse(
diff --git a/cpp/arcticdb/util/type_handler.cpp b/cpp/arcticdb/util/type_handler.cpp
index 152705f4da..1bf4734e3f 100644
--- a/cpp/arcticdb/util/type_handler.cpp
+++ b/cpp/arcticdb/util/type_handler.cpp
@@ -21,13 +21,21 @@ void TypeHandlerRegistry::init() {
 std::shared_ptr<TypeHandlerRegistry> TypeHandlerRegistry::instance_;
 std::once_flag TypeHandlerRegistry::init_flag_;
 
-std::shared_ptr<TypeHandler> TypeHandlerRegistry::get_handler(entity::DataType data_type) const {
-    auto it = handlers_.find(data_type);
+std::shared_ptr<TypeHandler> TypeHandlerRegistry::get_handler(const util::TypeDescriptor& type_descriptor) const {
+    auto it = handlers_.find(type_descriptor);
     return it == std::end(handlers_) ? std::shared_ptr<TypeHandler>{} : it->second;
 }
 
-void TypeHandlerRegistry::register_handler(entity::DataType data_type, TypeHandler&& handler) {
-     handlers_.try_emplace(data_type, std::make_shared<TypeHandler>(std::move(handler)));
+void TypeHandlerRegistry::register_handler(const util::TypeDescriptor& type_descriptor, TypeHandler&& handler) {
+     handlers_.try_emplace(type_descriptor, std::make_shared<TypeHandler>(std::move(handler)));
 }
 
-} //namespace arcticc
+size_t TypeHandlerRegistry::Hasher::operator()(const util::TypeDescriptor descriptor) const {
+    static_assert(sizeof(descriptor) == sizeof(uint16_t), "Cannot compute util::TypeDescriptor's hash. The size is wrong.");
+    static_assert(sizeof(decltype(descriptor.data_type())) == 1);
+    static_assert(sizeof(decltype(descriptor.dimension())) == 1);
+    const std::hash<uint16_t> hasher;
+    return hasher(uint16_t(descriptor.data_type()) << 8 | uint16_t(descriptor.dimension()));
+}
+
+} //namespace arcticdb
diff --git a/cpp/arcticdb/util/type_handler.hpp b/cpp/arcticdb/util/type_handler.hpp
index d2bcc60318..a696b0c9ec 100644
--- a/cpp/arcticdb/util/type_handler.hpp
+++ b/cpp/arcticdb/util/type_handler.hpp
@@ -34,8 +34,9 @@ struct ITypeHandler {
             const VariantField& encoded_field_info,
             const entity::TypeDescriptor& type_descriptor,
             size_t dest_bytes,
-            std::shared_ptr<BufferHolder> buffers) {
-            folly::poly_call<0>(*this, source, dest, encoded_field_info, type_descriptor, dest_bytes, buffers);
+            std::shared_ptr<BufferHolder> buffers,
+            EncodingVersion encoding_version) {
+            folly::poly_call<0>(*this, source, dest, encoded_field_info, type_descriptor, dest_bytes, buffers, encoding_version);
         }
     };
 
@@ -55,10 +56,13 @@ class TypeHandlerRegistry {
     static void init();
     static std::shared_ptr<TypeHandlerRegistry> instance();
 
-    std::shared_ptr<TypeHandler> get_handler(entity::DataType data_type) const;
-    void register_handler(entity::DataType data_type, TypeHandler&& handler);
+    std::shared_ptr<TypeHandler> get_handler(const util::TypeDescriptor& type_descriptor) const;
+    void register_handler(const util::TypeDescriptor& type_descriptor, TypeHandler&& handler);
 private:
-    std::unordered_map<entity::DataType, std::shared_ptr<TypeHandler>> handlers_;
+    struct Hasher {
+        size_t operator()(const util::TypeDescriptor val) const;
+    };
+    std::unordered_map<util::TypeDescriptor, std::shared_ptr<TypeHandler>, Hasher> handlers_;
 };
 
 }
diff --git a/cpp/arcticdb/version/schema_checks.hpp b/cpp/arcticdb/version/schema_checks.hpp
index 9349b1f119..e2478de6ea 100644
--- a/cpp/arcticdb/version/schema_checks.hpp
+++ b/cpp/arcticdb/version/schema_checks.hpp
@@ -66,10 +66,10 @@ inline bool columns_match(const StreamDescriptor &left, const StreamDescriptor &
         const TypeDescriptor &left_type = left.fields(i).type();
         const TypeDescriptor &right_type = right.fields(i).type();
 
-        const bool valid_type_promotion = has_valid_type_promotion(left_type, right_type).has_value();
+        const bool valid_common_type = has_valid_common_type(left_type, right_type).has_value();
         const bool trivial_type_compatibility = trivially_compatible_types(left_type, right_type);
 
-        if (!trivial_type_compatibility and !valid_type_promotion)
+        if (!trivial_type_compatibility and !valid_common_type)
             return false;
     }
     return true;
diff --git a/cpp/proto/arcticc/pb2/descriptors.proto b/cpp/proto/arcticc/pb2/descriptors.proto
index bf6efb2263..89d08b7d4c 100644
--- a/cpp/proto/arcticc/pb2/descriptors.proto
+++ b/cpp/proto/arcticc/pb2/descriptors.proto
@@ -31,6 +31,7 @@ message TypeDescriptor {
         DYNAMIC_STRING = 11; // offset reference to deduplicated string pool
         DYNAMIC_UTF8 = 12;
         EMPTY = 13;
+        PYBOOL = 14; // Nullable boolean
     }
 
     enum SizeBits {
diff --git a/cpp/proto/arcticc/pb2/encoding.proto b/cpp/proto/arcticc/pb2/encoding.proto
index 14285d8dad..4e50d66998 100644
--- a/cpp/proto/arcticc/pb2/encoding.proto
+++ b/cpp/proto/arcticc/pb2/encoding.proto
@@ -116,6 +116,3 @@ message DictionaryEncodedField {
     NDArrayEncodedField values = 1; // aka level in pandas multi index
     NDArrayEncodedField positions = 2; // aka label in pandas multi index
 }
-
-
-
diff --git a/python/arcticdb/version_store/_normalization.py b/python/arcticdb/version_store/_normalization.py
index c545479bd6..1219319f97 100644
--- a/python/arcticdb/version_store/_normalization.py
+++ b/python/arcticdb/version_store/_normalization.py
@@ -240,6 +240,8 @@ def _to_primitive(arr, arr_name, dynamic_strings, string_max_len=None, coerce_co
             casted_arr = coerce_string_column_to_fixed_length_array(arr, type(sample), string_max_len)
     elif dynamic_strings and sample is None:  # arr is entirely empty
         return arr
+    elif isinstance(sample, bool) or isinstance(sample, np.ndarray):
+        return arr
     else:
         raise ArcticDbNotYetImplemented(
             "Support for arbitrary objects in an array is not implemented apart from string, unicode, Timestamp. "


Test patch:
diff --git a/cpp/arcticdb/codec/test/test_codec.cpp b/cpp/arcticdb/codec/test/test_codec.cpp
index 8fbf249178..8ae381e36a 100644
--- a/cpp/arcticdb/codec/test/test_codec.cpp
+++ b/cpp/arcticdb/codec/test/test_codec.cpp
@@ -13,133 +13,384 @@
 #include <arcticdb/util/random.h>
 #include <arcticdb/stream/row_builder.hpp>
 #include <arcticdb/stream/aggregator.hpp>
+#include <arcticdb/codec/typed_block_encoder_impl.hpp>
 
 #include <gtest/gtest.h>
 
-using namespace arcticdb;
-
-TEST(FieldEncoderTest, PassthroughDim0) {
+namespace arcticdb {
+    struct ColumnEncoderV1 {
+        static std::pair<size_t, size_t> max_compressed_size(
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            ColumnData& column_data);
+
+        static void encode(
+            const arcticdb::proto::encoding::VariantCodec &codec_opts,
+            ColumnData& column_data,
+            std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+            Buffer& out,
+            std::ptrdiff_t& pos);
+    };
 
-    using TD = TypeDescriptorTag<
-        DataTypeTag<DataType::FLOAT64>,
-        DimensionTag<Dimension::Dim0>
-    >;
-    using Encoder = BlockEncoder<TD>;
+    struct ColumnEncoderV2 {
+    public:
+        static void encode(
+            const arcticdb::proto::encoding::VariantCodec &codec_opts,
+            ColumnData& column_data,
+            std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+            Buffer& out,
+            std::ptrdiff_t& pos);
+        static std::pair<size_t, size_t> max_compressed_size(
+            const arcticdb::proto::encoding::VariantCodec& codec_opts,
+            ColumnData& column_data);
+    private:
+        static void encode_shapes(
+            const ColumnData& column_data,
+            std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+            Buffer& out,
+            std::ptrdiff_t& pos_in_buffer);
+        static void encode_blocks(
+            const arcticdb::proto::encoding::VariantCodec &codec_opts,
+            ColumnData& column_data,
+            std::variant<EncodedField*, arcticdb::proto::encoding::EncodedField*> variant_field,
+            Buffer& out,
+            std::ptrdiff_t& pos);
+    };
 
-    std::vector<double> v{0.1, 0.2, 0.3};
-    const shape_t *shape = nullptr;
-    using Field = Encoder::FieldType;
+    size_t calc_column_blocks_size(const Column& col);
+}
 
-    Field f(v.data(), shape, v.size() * sizeof(double), v.size(), nullptr);
+using namespace arcticdb;
 
-    arcticdb::proto::encoding::EncodedField field;
-    arcticdb::proto::encoding::VariantCodec opt;
+using EncoginVersions = ::testing::Types<
+    std::integral_constant<EncodingVersion, EncodingVersion::V1>,
+    std::integral_constant<EncodingVersion, EncodingVersion::V2>>;
+
+class FieldEncoderTestDim0Base : public testing::Test {
+protected:
+    using ValuesTypeDescriptorTag = TypeDescriptorTag<DataTypeTag<DataType::FLOAT64>, DimensionTag<Dimension::Dim0>>;
+    static constexpr TypeDescriptor type_descriptor = static_cast<TypeDescriptor>(ValuesTypeDescriptorTag());
+    static constexpr std::array<double, 3> values = {0.1, 0.2, 0.3};
+    static constexpr size_t values_byte_size = values.size() * sizeof(decltype(values)::value_type);
+    static constexpr size_t expected_bytes = values.size() * sizeof(ValuesTypeDescriptorTag::DataTypeTag::raw_type);
+    static_assert(std::is_same_v<ValuesTypeDescriptorTag::DataTypeTag::raw_type, decltype(values)::value_type>);
+    arcticdb::proto::encoding::VariantCodec passthorugh_encoding_options;
+};
 
-    Buffer out{Encoder::max_compressed_size(opt, f)};
+template<typename EncodedFieldType>
+class FieldEncoderTestDim0 : public FieldEncoderTestDim0Base{};
+
+using EncodedFieldsType = ::testing::Types<arcticdb::proto::encoding::EncodedField, EncodedField>;
+TYPED_TEST_SUITE(FieldEncoderTestDim0, EncodedFieldsType);
+
+TYPED_TEST(FieldEncoderTestDim0, Passthrough_v1) {
+    using Encoder = TypedBlockEncoderImpl<TypedBlockData, typename TestFixture::ValuesTypeDescriptorTag, EncodingVersion::V1>;
+    const TypedBlockData<typename TestFixture::ValuesTypeDescriptorTag> values_block(
+        TestFixture::values.data(),
+        nullptr,
+        TestFixture::values_byte_size,
+        TestFixture::values.size(),
+        nullptr);
+    TypeParam encoded_field;
+    Buffer out{Encoder::max_compressed_size(TestFixture::passthorugh_encoding_options, values_block)};
     std::ptrdiff_t pos = 0;
+    Encoder::encode(TestFixture::passthorugh_encoding_options, values_block, encoded_field, out, pos);
+    auto& nd = encoded_field.ndarray();
+    ASSERT_EQ(nd.items_count(), TestFixture::values.size());
+    ASSERT_EQ(nd.shapes_size(), 0);
+    const auto& vals = nd.values();
+    ASSERT_EQ(vals[0].in_bytes(), TestFixture::expected_bytes);
+    ASSERT_EQ(vals[0].out_bytes(), TestFixture::expected_bytes);
+    ASSERT_NE(0, vals[0].hash());
+    ASSERT_EQ(pos, TestFixture::expected_bytes);
+}
 
-    Encoder::encode(opt, f, field, out, pos);
-
-    auto &nd = field.ndarray();
-    ASSERT_EQ(nd.items_count(), 3);
-    ASSERT_FALSE(nd.shapes_size() > 0);
-    auto &vals = nd.values();
+TYPED_TEST(FieldEncoderTestDim0, Passthrough_v2) {
+    using Encoder = TypedBlockEncoderImpl<TypedBlockData, typename TestFixture::ValuesTypeDescriptorTag, EncodingVersion::V2>;
+    const TypedBlockData<typename TestFixture::ValuesTypeDescriptorTag> values_block(
+        TestFixture::values.data(),
+        nullptr,
+        TestFixture::values_byte_size,
+        TestFixture::values.size(),
+        nullptr);
+    TypeParam encoded_field;
+    Buffer out{Encoder::max_compressed_size(TestFixture::passthorugh_encoding_options, values_block)};
+    std::ptrdiff_t pos = 0;
+    Encoder::encode_values(TestFixture::passthorugh_encoding_options, values_block, encoded_field, out, pos);
+    auto& nd = encoded_field.ndarray();
+    ASSERT_EQ(nd.items_count(), TestFixture::values.size());
+    ASSERT_EQ(nd.shapes_size(), 0);
+    const auto& vals = nd.values();
+    ASSERT_EQ(vals[0].in_bytes(), TestFixture::expected_bytes);
+    ASSERT_EQ(vals[0].out_bytes(), TestFixture::expected_bytes);
+    ASSERT_NE(0, vals[0].hash());
+    ASSERT_EQ(pos, TestFixture::expected_bytes);
+}
 
-    auto expected_bytes = v.size() * sizeof(TD::DataTypeTag::raw_type);
-    ASSERT_EQ(vals[0].in_bytes(), expected_bytes);
-    ASSERT_EQ(vals[0].out_bytes(), expected_bytes);
+template<typename EncodingVersionConstant>
+class FieldEncoderTestFromColumnDim0 : public FieldEncoderTestDim0Base{};
+
+/// @brief Cartesian product between the type of the encoded field and the encoding version.
+/// (EncodedField, arcticdb::proto::encoding::EncodedField) x (EncodingVersion::V1, EncodingVersion::V2)
+using FieldVersionT = ::testing::Types<
+    std::pair<arcticdb::proto::encoding::EncodedField, ColumnEncoderV1>,
+	std::pair<arcticdb::proto::encoding::EncodedField, ColumnEncoderV2>,
+	std::pair<EncodedField, ColumnEncoderV1>,
+	std::pair<EncodedField, ColumnEncoderV2>>;
+TYPED_TEST_SUITE(FieldEncoderTestFromColumnDim0, FieldVersionT);
+
+TYPED_TEST(FieldEncoderTestFromColumnDim0, Passthrough) {
+    using EncodedFieldType = typename TypeParam::first_type;
+    using ColumnEncoder = typename TypeParam::second_type;
+
+    ChunkedBuffer values_buffer;
+    values_buffer.ensure(TestFixture::values_byte_size);
+    memcpy(values_buffer.ptr_cast<uint8_t>(0, TestFixture::values_byte_size),
+        TestFixture::values.data(),
+        TestFixture::values_byte_size);
+    Buffer shapes_buffer;
+    ColumnData column_data(&values_buffer, &shapes_buffer, TestFixture::type_descriptor, nullptr);
+    EncodedFieldType field;
+    std::ptrdiff_t pos = 0;
+    const auto [_, max_compressed_size] = ColumnEncoder::max_compressed_size(TestFixture::passthorugh_encoding_options,
+        column_data);
+    Buffer out(max_compressed_size);
+    column_data.reset();
+    ColumnEncoder::encode(TestFixture::passthorugh_encoding_options, column_data, &field, out, pos);
+    auto& nd = field.ndarray();
+    ASSERT_EQ(nd.items_count(), TestFixture::values.size());
+    ASSERT_EQ(nd.shapes_size(), 0);
+    const auto& vals = nd.values();
+    ASSERT_EQ(vals[0].in_bytes(), TestFixture::expected_bytes);
+    ASSERT_EQ(vals[0].out_bytes(), TestFixture::expected_bytes);
     ASSERT_NE(0, vals[0].hash());
-    ASSERT_EQ(pos , expected_bytes);
+    ASSERT_EQ(pos, TestFixture::expected_bytes);
 }
 
-TEST(FieldEncoderTest, PassthroughDim0_Dynamic) {
-    TypeDescriptor td{DataType::FLOAT64, Dimension::Dim0};
-    std::vector<double> v{0.1};
-    ChunkedBuffer cbuf;
-    cbuf.ensure(sizeof(double));
-    memcpy(cbuf.ptr_cast<uint8_t>(0, sizeof(double)), v.data(), sizeof(double));
-    Buffer shapes;
-    ColumnData f(&cbuf, &shapes, td, nullptr);
+class FieldEncoderTestDim1 : public testing::Test {
+protected:
+    using ValuesTypeDescriptorTag = TypeDescriptorTag<DataTypeTag<DataType::FLOAT64>, DimensionTag<Dimension::Dim1>>;
+    static constexpr std::array<double, 5> values = {0.1, 0.2, 0.3, 0.4, 0.5};
+    static constexpr size_t values_byte_size = values.size() * sizeof(decltype(values)::value_type);
+    static constexpr std::array<shape_t, 2> shapes = {2, 3};
+    static constexpr size_t shapes_byte_size = shapes.size() * sizeof(decltype(shapes)::value_type);
+    static constexpr size_t values_expected_bytes =
+        values.size() * sizeof(ValuesTypeDescriptorTag::DataTypeTag::raw_type);
+    static_assert(std::is_same_v<ValuesTypeDescriptorTag::DataTypeTag::raw_type, decltype(values)::value_type>);
+    arcticdb::proto::encoding::VariantCodec passthorugh_encoding_options;
+};
 
+TEST_F(FieldEncoderTestDim1, PassthroughV1ProtoField) {
+    using Encoder = TypedBlockEncoderImpl<TypedBlockData, ValuesTypeDescriptorTag, EncodingVersion::V1>;
+    const TypedBlockData<ValuesTypeDescriptorTag> block(
+        values.data(),
+        shapes.data(),
+        values_byte_size,
+        shapes.size(),
+        nullptr);
     arcticdb::proto::encoding::EncodedField field;
-    arcticdb::proto::encoding::VariantCodec opt;
-
+    Buffer out(Encoder::max_compressed_size(passthorugh_encoding_options, block));
     std::ptrdiff_t pos = 0;
+    Encoder::encode(passthorugh_encoding_options, block, field, out, pos);
 
-    ColumnEncoder enc;
-    const auto [uncomp, req] = enc.max_compressed_size(opt, f);
-    Buffer out(req);
-    f.reset();
-    enc.encode(opt, f, field, out, pos);
+    const auto& nd = field.ndarray();
+    ASSERT_EQ(nd.items_count(), shapes.size());
 
-    auto &nd = field.ndarray();
-    ASSERT_EQ(nd.items_count(), 1);
-    ASSERT_FALSE(nd.shapes_size() > 0);
-    auto &vals = nd.values();
+    const auto& shapes = nd.shapes();
+    ASSERT_EQ(shapes[0].in_bytes(), shapes_byte_size);
+    ASSERT_EQ(shapes[0].out_bytes(), shapes_byte_size);
+    ASSERT_NE(0, shapes[0].hash());
 
-    auto expected_bytes = v.size() * sizeof(v[0]);
-    ASSERT_EQ(vals[0].in_bytes(), expected_bytes);
-    ASSERT_EQ(vals[0].out_bytes(), expected_bytes);
+    const auto& vals = nd.values();
+    ASSERT_EQ(vals[0].in_bytes(), values_expected_bytes);
+    ASSERT_EQ(vals[0].out_bytes(), values_expected_bytes);
     ASSERT_NE(0, vals[0].hash());
-    ASSERT_EQ(pos, expected_bytes);
+    ASSERT_EQ(pos, values_expected_bytes + shapes_byte_size);
 }
 
-TEST(FieldEncoderTest, PassthroughDim1) {
-    using TD = TypeDescriptorTag<
-        DataTypeTag<DataType::FLOAT64>,
-        DimensionTag<Dimension::Dim1>
-    >;
+TEST_F(FieldEncoderTestDim1, PassthroughV1NativeField) {
+    using Encoder = TypedBlockEncoderImpl<TypedBlockData, ValuesTypeDescriptorTag, EncodingVersion::V1>;
+    const TypedBlockData<ValuesTypeDescriptorTag> block(
+        values.data(),
+        shapes.data(),
+        values_byte_size,
+        shapes.size(),
+        nullptr);
+    // one block for shapes and one for values
+    constexpr size_t encoded_field_size = EncodedField::Size + 2 * sizeof(EncodedBlock);
+    std::array<uint8_t, encoded_field_size> encoded_field_memory;
+    EncodedField* field = new(encoded_field_memory.data()) EncodedField;
+
+    Buffer out(Encoder::max_compressed_size(passthorugh_encoding_options, block));
+    std::ptrdiff_t pos = 0;
+    Encoder::encode(passthorugh_encoding_options, block, *field, out, pos);
 
-    using Encoder = BlockEncoder<TD>;
-    using Field = Encoder::FieldType;
+    const auto& nd = field->ndarray();
+    ASSERT_EQ(nd.items_count(), shapes.size());
 
-    std::vector<double> v{0.1, 0.2, 0.3, 0.4};
-    std::vector<shape_t> s{2, 2};
-    Field f(v.data(), s.data(), v.size() * sizeof(double), s.size(), nullptr);
+    const auto& shapes = nd.shapes();
+    ASSERT_EQ(shapes[0].in_bytes(), shapes_byte_size);
+    ASSERT_EQ(shapes[0].out_bytes(), shapes_byte_size);
+    ASSERT_NE(0, shapes[0].hash());
 
-    arcticdb::proto::encoding::EncodedField field;
-    arcticdb::proto::encoding::VariantCodec opt;
+    const auto& vals = nd.values();
+    ASSERT_EQ(vals[0].in_bytes(), values_expected_bytes);
+    ASSERT_EQ(vals[0].out_bytes(), values_expected_bytes);
+    ASSERT_NE(0, vals[0].hash());
+    ASSERT_EQ(pos, values_expected_bytes + shapes_byte_size);
+}
 
-    Buffer out(Encoder::max_compressed_size(opt, f));
+TEST_F(FieldEncoderTestDim1, PassthroughV2ProtoField) {
+    using Encoder = TypedBlockEncoderImpl<TypedBlockData, ValuesTypeDescriptorTag, EncodingVersion::V2>;
+    using ShapesEncoder = TypedBlockEncoderImpl<TypedBlockData, arcticdb::ShapesBlockTDT, EncodingVersion::V2>;
+    const TypedBlockData<ValuesTypeDescriptorTag> values_block(
+        values.data(),
+        shapes.data(),
+        values_byte_size,
+        shapes.size(),
+        nullptr);
+    const TypedBlockData<arcticdb::ShapesBlockTDT> shapes_block(
+        shapes.data(),
+        nullptr,
+        shapes_byte_size,
+        0,
+        nullptr);
+    const size_t values_max_compressed_size = Encoder::max_compressed_size(passthorugh_encoding_options,
+        values_block);
+    const size_t shapes_max_compressed_size = ShapesEncoder::max_compressed_size(passthorugh_encoding_options,
+        shapes_block);
+    const size_t total_max_compressed_size = values_max_compressed_size + shapes_max_compressed_size;
+    arcticdb::proto::encoding::EncodedField field;
+    Buffer out(total_max_compressed_size);
     std::ptrdiff_t pos = 0;
+    ShapesEncoder::encode_shapes(passthorugh_encoding_options, shapes_block, field, out, pos);
+    Encoder::encode_values(passthorugh_encoding_options, values_block, field, out, pos);
 
-    Encoder::encode(opt, f, field, out, pos);
+    const auto& nd = field.ndarray();
+    ASSERT_EQ(nd.items_count(), shapes.size());
 
-    auto &nd = field.ndarray();
-    ASSERT_EQ(nd.items_count(), 2);
-    auto &shapes = nd.shapes();
-    auto shapes_bytes = 2 * sizeof(shape_t);
+    const auto& shapes = nd.shapes();
+    ASSERT_EQ(shapes[0].in_bytes(), shapes_byte_size);
+    ASSERT_EQ(shapes[0].out_bytes(), shapes_byte_size);
+    ASSERT_NE(0, shapes[0].hash());
+
+    const auto& vals = nd.values();
+    ASSERT_EQ(vals[0].in_bytes(), values_expected_bytes);
+    ASSERT_EQ(vals[0].out_bytes(), values_expected_bytes);
+    ASSERT_NE(0, vals[0].hash());
+    ASSERT_EQ(pos, values_expected_bytes + shapes_byte_size);
+}
+
+TEST_F(FieldEncoderTestDim1, PassthroughV2NativeField) {
+    using Encoder = TypedBlockEncoderImpl<TypedBlockData, ValuesTypeDescriptorTag, EncodingVersion::V2>;
+    using ShapesEncoder = TypedBlockEncoderImpl<TypedBlockData, arcticdb::ShapesBlockTDT, EncodingVersion::V2>;
+    const TypedBlockData<ValuesTypeDescriptorTag> values_block(
+        values.data(),
+        shapes.data(),
+        values_byte_size,
+        shapes.size(),
+        nullptr);
+    const TypedBlockData<arcticdb::ShapesBlockTDT> shapes_block(
+        shapes.data(),
+        nullptr,
+        shapes_byte_size,
+        0,
+        nullptr);
+    const size_t values_max_compressed_size = Encoder::max_compressed_size(passthorugh_encoding_options,
+        values_block);
+    const size_t shapes_max_compressed_size = ShapesEncoder::max_compressed_size(passthorugh_encoding_options,
+        shapes_block);
+    const size_t total_max_compressed_size = values_max_compressed_size + shapes_max_compressed_size;
+    // one block for shapes and one for values
+    constexpr size_t encoded_field_size = EncodedField::Size + 2 * sizeof(EncodedBlock);
+    std::array<uint8_t, encoded_field_size> encoded_field_memory;
+    EncodedField* field = new(encoded_field_memory.data()) EncodedField;
+    Buffer out(total_max_compressed_size);
+    std::ptrdiff_t pos = 0;
+    ShapesEncoder::encode_shapes(passthorugh_encoding_options, shapes_block, *field, out, pos);
+    Encoder::encode_values(passthorugh_encoding_options, values_block, *field, out, pos);
+
+    const auto& nd = field->ndarray();
+    ASSERT_EQ(nd.items_count(), shapes.size());
 
-    ASSERT_EQ(shapes[0].in_bytes(), shapes_bytes);
-    ASSERT_EQ(shapes[0].out_bytes(), shapes_bytes);
+    const auto& shapes = nd.shapes();
+    ASSERT_EQ(shapes[0].in_bytes(), shapes_byte_size);
+    ASSERT_EQ(shapes[0].out_bytes(), shapes_byte_size);
     ASSERT_NE(0, shapes[0].hash());
 
-    auto &vals = nd.values();
-    auto expected_bytes = v.size() * sizeof(TD::DataTypeTag::raw_type);
-    ASSERT_EQ(vals[0].in_bytes(), expected_bytes);
-    ASSERT_EQ(vals[0].out_bytes(), expected_bytes);
+    const auto& vals = nd.values();
+    ASSERT_EQ(vals[0].in_bytes(), values_expected_bytes);
+    ASSERT_EQ(vals[0].out_bytes(), values_expected_bytes);
     ASSERT_NE(0, vals[0].hash());
-    ASSERT_EQ(pos, expected_bytes + shapes_bytes);
+    ASSERT_EQ(pos, values_expected_bytes + shapes_byte_size);
 }
 
-TEST(SegmentEncoderTest, EncodeSingleStringV1) {
-    const auto tsd = create_tsd<DataTypeTag<DataType::ASCII_DYNAMIC64>, Dimension::Dim0>("thing", 1);
-    SegmentInMemory s(StreamDescriptor{tsd});
-    s.set_scalar(0, timestamp(123));
-    s.set_string(1, "happy");
-    s.end_row();
-    arcticdb::proto::encoding::VariantCodec opt;
-    auto lz4ptr = opt.mutable_lz4();
-    lz4ptr->set_acceleration(1);
-    auto copy = s.clone();
-    Segment seg = encode_v1(s.clone(), opt);
+class TestMultiblockData_Dim1 : public testing::Test {
+protected:
+    void SetUp() override {
+        data_buffer.add_block(first_block_data_byte_size, 0);
+        data_buffer.blocks()[0]->resize(first_block_data_byte_size);
+        data_buffer.add_block(second_block_data_byte_size, first_block_data_byte_size);
+        data_buffer.blocks()[1]->resize(second_block_data_byte_size);
+        shapes_buffer.ensure(shapes_data_byte_size);
+        data_buffer.blocks()[0]->copy_from(reinterpret_cast<const uint8_t*>(first_block_data.data()),
+            first_block_data_byte_size,
+            0);
+        data_buffer.blocks()[1]->copy_from(reinterpret_cast<const uint8_t*>(second_block_data.data()),
+            second_block_data_byte_size,
+            0);
+        memcpy(shapes_buffer.data(), shapes_data.data(), shapes_data_byte_size);
+    }
+    using ValuesTypeDescriptorTag = TypeDescriptorTag<DataTypeTag<DataType::INT64>, DimensionTag<Dimension::Dim1>>;
+    static constexpr TypeDescriptor type_descriptor = static_cast<TypeDescriptor>(ValuesTypeDescriptorTag());
+    static constexpr std::array<int64_t, 8> first_block_data = {1, 2, 3, 4, 5, 6, 7, 8};
+    static constexpr size_t first_block_data_byte_size =
+        sizeof(decltype(first_block_data)::value_type) * first_block_data.size();
+    static constexpr std::array<int64_t, 2> second_block_data = {9, 10};
+    static constexpr size_t second_block_data_byte_size =
+        sizeof(decltype(second_block_data)::value_type) * second_block_data.size();
+    static constexpr std::array<shape_t, 2> shapes_data = {first_block_data.size(), second_block_data.size()};
+    static constexpr size_t shapes_data_byte_size = sizeof(decltype(shapes_data)::value_type) * shapes_data.size();
+    arcticdb::proto::encoding::VariantCodec passthorugh_encoding_options;
+    ChunkedBuffer data_buffer;
+    Buffer shapes_buffer;
+};
 
-    SegmentInMemory res = decode_segment(std::move(seg));
-    ASSERT_EQ(copy.string_at(0, 1), res.string_at(0, 1));
-    ASSERT_EQ(std::string("happy"), res.string_at(0, 1));
+TEST_F(TestMultiblockData_Dim1, EncodingVersion_1) {
+    arcticdb::proto::encoding::EncodedField encoded_field;
+    ColumnData column_data(&data_buffer, &shapes_buffer, type_descriptor, nullptr);
+    const auto [_, max_compressed_size] = ColumnEncoderV1::max_compressed_size(passthorugh_encoding_options, column_data);
+    Buffer out(max_compressed_size);
+    ptrdiff_t out_pos = 0;
+    column_data.reset();
+    ColumnEncoderV1::encode(passthorugh_encoding_options, column_data, &encoded_field, out, out_pos);
+    const auto ndarray = encoded_field.ndarray();
+    ASSERT_EQ(ndarray.shapes_size(), 2);
+    ASSERT_EQ(ndarray.values_size(), 2);
+    ASSERT_EQ(ndarray.items_count(), shapes_data.size());
+}
+
+TEST_F(TestMultiblockData_Dim1, EncodingVersion_2) {
+    constexpr size_t encoded_field_size = EncodedField::Size + 3 * sizeof(EncodedBlock);
+    std::array<uint8_t, encoded_field_size> encoded_field_owner;
+    EncodedField* encoded_field = new(encoded_field_owner.data()) EncodedField;
+    ColumnData column_data(&data_buffer, &shapes_buffer, type_descriptor, nullptr);
+    const auto [_, max_compressed_size] = ColumnEncoderV2::max_compressed_size(passthorugh_encoding_options, column_data);
+    Buffer out(max_compressed_size);
+    ptrdiff_t out_pos = 0;
+    column_data.reset();
+    ColumnEncoderV2::encode(passthorugh_encoding_options, column_data, encoded_field, out, out_pos);
+    const auto ndarray = encoded_field->ndarray();
+    ASSERT_EQ(ndarray.shapes_size(), 1);
+    ASSERT_EQ(ndarray.values_size(), 2);
+    ASSERT_EQ(ndarray.items_count(), shapes_data.size());
 }
 
-TEST(SegmentEncoderTest, EncodeSingleStringV2) {
+template<typename EncodedFieldType>
+class SegmentStringEncodingTest : public testing::Test{};
+
+TYPED_TEST_SUITE(SegmentStringEncodingTest, EncoginVersions);
+
+TYPED_TEST(SegmentStringEncodingTest, EncodeSingleString) {
     const auto tsd = create_tsd<DataTypeTag<DataType::ASCII_DYNAMIC64>, Dimension::Dim0>("thing", 1);
     SegmentInMemory s(StreamDescriptor{tsd});
     s.set_scalar(0, timestamp(123));
@@ -149,14 +400,15 @@ TEST(SegmentEncoderTest, EncodeSingleStringV2) {
     auto lz4ptr = opt.mutable_lz4();
     lz4ptr->set_acceleration(1);
     auto copy = s.clone();
-    Segment seg = encode_v2(SegmentInMemory{s}, opt);
+    constexpr EncodingVersion encoding_version = TypeParam::value;
+    Segment seg = encode_dispatch(s.clone(), opt, encoding_version);
 
     SegmentInMemory res = decode_segment(std::move(seg));
     ASSERT_EQ(copy.string_at(0, 1), res.string_at(0, 1));
     ASSERT_EQ(std::string("happy"), res.string_at(0, 1));
 }
 
-TEST(SegmentencoderTest, EncodeStringsBasic) {
+TYPED_TEST(SegmentStringEncodingTest, EncodeStringsBasic) {
     const auto tsd = create_tsd<DataTypeTag<DataType::ASCII_DYNAMIC64>, Dimension::Dim0>();
     SegmentInMemory s(StreamDescriptor{tsd});
     s.set_scalar(0, timestamp(123));
@@ -175,7 +427,8 @@ TEST(SegmentencoderTest, EncodeStringsBasic) {
     auto lz4ptr = opt.mutable_lz4();
     lz4ptr->set_acceleration(1);
     auto copy = s.clone();
-    Segment seg = encode_v1(SegmentInMemory{s}, opt);
+    constexpr EncodingVersion encoding_version = TypeParam::value;
+    Segment seg = encode_dispatch(SegmentInMemory{s}, opt, encoding_version);
 
     SegmentInMemory res = decode_segment(std::move(seg));
     ASSERT_EQ(copy.string_at(0, 1), res.string_at(0, 1));
@@ -242,4 +495,4 @@ TEST(Segment, KeepAlive) {
         std::any_cast<TransactionalThing>(seg4.keepalive()).magic_.check();
     }
     ASSERT_EQ(TransactionalThing::destroyed, true);
-}
\ No newline at end of file
+}
diff --git a/cpp/arcticdb/entity/test/test_tensor.cpp b/cpp/arcticdb/entity/test/test_tensor.cpp
index 12a683fa07..77ab03daf3 100644
--- a/cpp/arcticdb/entity/test/test_tensor.cpp
+++ b/cpp/arcticdb/entity/test/test_tensor.cpp
@@ -28,7 +28,7 @@ auto get_f_tensor(size_t num_rows) {
     const auto ndim = 2;
     const DataType dt = DataType::UINT32;
 
-    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data())};
+    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data()), ndim};
     return std::make_pair(data, tensor);
 }
 
@@ -46,7 +46,7 @@ auto get_c_tensor(size_t num_rows) {
     const auto ndim = 2;
     const DataType dt = DataType::UINT32;
 
-    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data())};
+    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data()), ndim};
     return std::make_pair(data, tensor);
 }
 
@@ -159,7 +159,7 @@ auto get_sparse_array(size_t num_rows) {
     const auto ndim = 1;
     const DataType dt = DataType::UINT32;
 
-    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data())};
+    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data()), ndim};
     return std::make_pair(data, tensor);
 }
 
@@ -225,7 +225,7 @@ auto get_sparse_array_funky_strides() {
     const auto ndim = 1;
     const DataType dt = DataType::UINT32;
 
-    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data())};
+    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data()), ndim};
     return std::make_pair(data, tensor);
 }
 
@@ -290,7 +290,7 @@ auto get_sparse_array_uneven(size_t num_rows) {
     const auto ndim = 1;
     const DataType dt = DataType::UINT32;
 
-    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data())};
+    NativeTensor tensor{nbytes, ndim, strides.data(), shapes.data(), dt, get_type_size(dt), static_cast<const void*>(data->data()), ndim};
     return std::make_pair(data, tensor);
 }
 
@@ -320,4 +320,4 @@ TEST(SparseArray, SubDivideUneven) {
     for(uint32_t x = 0; x < num_rows; ++x) {
         ASSERT_EQ(output[x], x);
     }
-}
\ No newline at end of file
+}
diff --git a/cpp/arcticdb/storage/test/test_embedded.cpp b/cpp/arcticdb/storage/test/test_embedded.cpp
index 014718ad0d..1f2c8adaf7 100644
--- a/cpp/arcticdb/storage/test/test_embedded.cpp
+++ b/cpp/arcticdb/storage/test/test_embedded.cpp
@@ -175,7 +175,7 @@ TEST_P(SimpleTestSuite, Strings) {
     arcticdb::proto::encoding::VariantCodec opt;
     auto lz4ptr = opt.mutable_lz4();
     lz4ptr->set_acceleration(1);
-    Segment seg = encode_v1(s.clone(), opt);
+    Segment seg = encode_dispatch(s.clone(), opt, EncodingVersion::V1);
 
     auto environment_name = as::EnvironmentName{"res"};
     auto storage_name = as::StorageName{"lmdb_01"};
@@ -216,4 +216,3 @@ auto backend_generators = get_backend_generators();
 
 INSTANTIATE_TEST_SUITE_P(TestEmbedded, SimpleTestSuite, testing::ValuesIn(backend_generators),
     [](const testing::TestParamInfo<SimpleTestSuite::ParamType>& info) { return info.param.get_name(); });
-
diff --git a/cpp/arcticdb/stream/test/stream_test_common.hpp b/cpp/arcticdb/stream/test/stream_test_common.hpp
index 5af17b089d..80a7a3af50 100644
--- a/cpp/arcticdb/stream/test/stream_test_common.hpp
+++ b/cpp/arcticdb/stream/test/stream_test_common.hpp
@@ -146,13 +146,13 @@ struct DefaultStringGenerator {
 template<class ContainerType, typename DTT,
     std::enable_if_t<std::is_floating_point_v<typename DTT::raw_type> || std::is_integral_v<typename DTT::raw_type>,
                      int> = 0>
-NativeTensor test_column(ContainerType &container, DTT, size_t num_rows, size_t start_val, bool is_index) {
+NativeTensor test_column(ContainerType &container, DTT, shape_t num_rows, size_t start_val, bool is_index) {
     using RawType = typename DTT::raw_type;
     constexpr auto dt = DTT::data_type;
 
     shape_t shapes = num_rows;
     stride_t strides;
-    ssize_t elsize;
+    stride_t elsize;
 
     strides = sizeof(RawType);
     elsize = static_cast<ssize_t>(get_type_size(dt));
@@ -162,11 +162,11 @@ NativeTensor test_column(ContainerType &container, DTT, size_t num_rows, size_t
         fill_test_value_vector(container, RawType{}, num_rows, start_val);
 
     ssize_t bytes = shapes * strides;
-    return NativeTensor{bytes, 1, &strides, &shapes, dt, elsize, container.ptr()};
+    return NativeTensor{bytes, 1, &strides, &shapes, dt, elsize, container.ptr(), 1};
 }
 
 template<class ContainerType, typename DTT, class StringGenerator = DefaultStringGenerator>
-NativeTensor test_string_column(ContainerType &vec, DTT, size_t num_rows) {
+NativeTensor test_string_column(ContainerType &vec, DTT, shape_t num_rows) {
     constexpr auto dt = DTT::data_type;
     shape_t shapes = num_rows;
     stride_t strides;
@@ -178,7 +178,7 @@ NativeTensor test_string_column(ContainerType &vec, DTT, size_t num_rows) {
     string_gen.fill_string_vector(vec, num_rows);
 
     ssize_t bytes = shapes * strides;
-    return NativeTensor{bytes, 1, &strides, &shapes, dt, elsize, vec.data()};
+    return NativeTensor{bytes, 1, &strides, &shapes, dt, elsize, vec.data(), 1};
 }
 
 inline std::vector<entity::FieldRef> get_test_timeseries_fields() {
@@ -370,4 +370,4 @@ struct TestStore : ::testing::Test {
     std::shared_ptr<arcticdb::version_store::PythonVersionStore> test_store_;
 };
 
-} //namespace arcticdb
\ No newline at end of file
+} //namespace arcticdb
diff --git a/cpp/arcticdb/util/test/generators.hpp b/cpp/arcticdb/util/test/generators.hpp
index 4eb922b356..0bebf9bf9b 100644
--- a/cpp/arcticdb/util/test/generators.hpp
+++ b/cpp/arcticdb/util/test/generators.hpp
@@ -331,7 +331,8 @@ inline NativeTensor tensor_from_column(const Column &column) {
             shape_ptr,
             data_type,
             get_type_size(data_type),
-            column.ptr()
+            column.ptr(),
+            to_tensor_dim(dim)
         };
         return tensor;
     });
diff --git a/cpp/arcticdb/util/test/test_utils.hpp b/cpp/arcticdb/util/test/test_utils.hpp
index c876a9b498..57f9493457 100644
--- a/cpp/arcticdb/util/test/test_utils.hpp
+++ b/cpp/arcticdb/util/test/test_utils.hpp
@@ -38,8 +38,8 @@ struct TestValue {
     using raw_type = typename DataTypeTag::raw_type;
 
     std::vector<raw_type> data_;
-    std::vector<ssize_t> shapes_;
-    mutable std::vector<ssize_t> strides_;
+    std::vector<shape_t> shapes_;
+    mutable std::vector<stride_t> strides_;
     raw_type start_val_;
 
     TestValue(raw_type start_val = raw_type(), size_t num_vals = 20) :
@@ -49,7 +49,7 @@ struct TestValue {
             return;
         }
 
-        constexpr ssize_t itemsize = ssize_t(sizeof(raw_type));
+        constexpr int64_t itemsize = sizeof(raw_type);
 
         if (dimensions == Dimension::Dim1) {
             shapes_.push_back(num_vals);
@@ -92,7 +92,7 @@ struct TestValue {
     TensorType<raw_type> get_tensor() const {
         util::check_arg(dimensions != Dimension::Dim0, "get tensor called on scalar test value");
         reconstruct_strides();
-        return TensorType<raw_type>{shapes_.data(), ssize_t(dimensions), DataTypeTag::data_type, get_type_size(DataTypeTag::data_type), data_.data()};
+        return TensorType<raw_type>{shapes_.data(), ssize_t(dimensions), DataTypeTag::data_type, get_type_size(DataTypeTag::data_type), data_.data(), ssize_t(dimensions)};
     }
 
     bool check_tensor(TensorType<raw_type> &t) const {
@@ -108,7 +108,7 @@ struct TestValue {
         return check_impl(dimensions, 0, shapes, strides, data);
     }
 
-    bool check_impl(Dimension dim, int pos, const ssize_t *shapes, const ssize_t *strides, const raw_type *data) const {
+    bool check_impl(Dimension dim, int pos, const shape_t *shapes, const stride_t *strides, const raw_type *data) const {
         auto shape = shapes_[size_t(dim) - 1];
         auto stride = strides_[size_t(dim) - 1] / sizeof(raw_type);
         for (int i = 0; i < +shape; ++i) {
@@ -162,4 +162,3 @@ struct TestRow {
     mutable util::BitSet bitset_;
     std::vector<TestValue<TDT>> values_;
 };
-
diff --git a/python/tests/unit/arcticdb/toolbox/test_codec.py b/python/tests/unit/arcticdb/toolbox/test_codec.py
deleted file mode 100644
index be5180a110..0000000000
--- a/python/tests/unit/arcticdb/toolbox/test_codec.py
+++ /dev/null
@@ -1,167 +0,0 @@
-"""
-Copyright 2023 Man Group Operations Limited
-Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
-As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
-"""
-from arcticdb_ext.types import TypeDescriptor, Dimension, DataType
-from arcticdb_ext.codec import DynamicFieldBuffer, encode_field, decode_field
-from arcticc.pb2.encoding_pb2 import VariantCodec
-import numpy as np
-
-from numpy.testing import assert_equal
-import pytest
-
-from arcticdb.supported_types import sint_types, uint_types, float_types, numeric_types, int_types, Shape, get_data_type
-
-
-def shapes(*array):
-    return np.array(array, dtype=Shape)
-
-
-def test_passthrough_enc_dim0():
-    td = TypeDescriptor(DataType.INT32, Dimension.Dim0)
-    s = shapes(10)
-    a = np.arange(10, dtype=np.int32)
-
-    f = DynamicFieldBuffer(td, a, s)
-    codec = VariantCodec()
-    res = encode_field(codec, f)
-
-    # no shape when Dim0
-    a_enc = np.array(res.buffer).view(np.uint32)
-    assert_equal(a, a_enc)
-
-    res_dec = decode_field(td, res.encoded_field, np.array(res.buffer))
-    assert_equal(a, np.array(res_dec.values_buffer).view(np.uint32))
-
-
-def test_passthrough_enc_dim1():
-    td = TypeDescriptor(DataType.INT32, Dimension.Dim1)
-    s = shapes(5, 5)
-    a = np.arange(10, dtype=np.int32)
-
-    f = DynamicFieldBuffer(td, a, s)
-    codec = VariantCodec()
-    res = encode_field(codec, f)
-
-    p = Shape().itemsize * len(s)
-    s_enc = np.array(res.buffer)[:p].view(Shape)
-    assert_equal(s, s_enc)
-
-    a_enc = np.array(res.buffer)[p:].view(np.uint32)
-    assert_equal(a, a_enc)
-
-    res_dec = decode_field(td, res.encoded_field, np.array(res.buffer))
-    assert_equal(s, np.array(res_dec.shape_buffer).view(np.uint64))
-    assert_equal(a, np.array(res_dec.values_buffer).view(np.uint32))
-
-
-def test_passthrough_enc_dim2():
-    td = TypeDescriptor(DataType.INT32, Dimension.Dim2)
-    s = np.array([5, 5] * 4, dtype=Shape)
-    a = np.arange(100, dtype=np.int32)
-
-    f = DynamicFieldBuffer(td, a, s)
-    codec = VariantCodec()
-    res = encode_field(codec, f)
-
-    p = Shape().itemsize * len(s)
-    s_enc = np.array(res.buffer)[:p].view(Shape)
-    assert_equal(s, s_enc)
-
-    a_enc = np.array(res.buffer)[p:].view(np.uint32)
-    assert_equal(a, a_enc)
-
-    res_dec = decode_field(td, res.encoded_field, np.array(res.buffer))
-    assert_equal(s, np.array(res_dec.shape_buffer).view(np.uint64))
-    assert_equal(a, np.array(res_dec.values_buffer).view(np.uint32))
-
-
-def gen_codecs():
-    codecs = list()
-    codecs.append(VariantCodec())
-
-    vcm = VariantCodec()
-    vcm.zstd.level = 0
-    codecs.append(vcm)
-
-    vcm = VariantCodec()
-    vcm.lz4.acceleration = 0
-    codecs.append(vcm)
-
-    # vcm = VariantCodec()
-    # vcm.passthrough.mark = True
-    # codecs.append(vcm)
-
-    return codecs
-
-
-codecs = gen_codecs()
-
-
-def gen_param_dim0(size):
-    ans = []
-    for c in codecs:
-        for t in numeric_types:
-            data_type = get_data_type(t)
-            s = shapes(size)
-            a = np.arange(size, dtype=t)
-            ans.append((c, data_type, a, s))
-    return ans
-
-
-@pytest.mark.parametrize(("codec", "data_type", "array", "shape"), gen_param_dim0(10))
-def test_round_trip_dim0(codec, data_type, array, shape):
-    # type: (VariantCodec, np.array[Any], np.array[np.uint64])->None
-    td = TypeDescriptor(data_type, Dimension.Dim0)
-    f = DynamicFieldBuffer(td, array, shape)
-    res = encode_field(codec, f)
-
-    res_dec = decode_field(td, res.encoded_field, np.array(res.buffer))
-    assert_equal(array, np.array(res_dec.values_buffer).view(array.dtype))
-
-
-def gen_param_dim1(size):
-    ans = []
-    for c in codecs:
-        for t in numeric_types:
-            data_type = get_data_type(t)
-            s = shapes(size // 2, size // 2)
-            a = np.arange(size, dtype=t)
-            ans.append((c, data_type, a, s))
-    return ans
-
-
-@pytest.mark.parametrize(("codec", "data_type", "array", "shape"), gen_param_dim1(10))
-def test_round_trip_dim1(codec, data_type, array, shape):
-    # type: (VariantCodec, np.array[Any], np.array[np.uint64])->None
-    td = TypeDescriptor(data_type, Dimension.Dim1)
-    f = DynamicFieldBuffer(td, array, shape)
-    res = encode_field(codec, f)
-
-    res_dec = decode_field(td, res.encoded_field, np.array(res.buffer))
-    assert_equal(array, np.array(res_dec.values_buffer).view(array.dtype))
-    assert_equal(shape, np.array(res_dec.shape_buffer).view(np.uint64))
-
-
-def gen_param_dim2():
-    ans = []
-    for c in codecs:
-        for t in numeric_types:
-            data_type = get_data_type(t)
-            s = np.array([5, 5] * 4, dtype=Shape)
-            a = np.arange(100, dtype=t)
-            ans.append((c, data_type, a, s))
-    return ans
-
-
-@pytest.mark.parametrize(("codec", "data_type", "array", "shape"), gen_param_dim2())
-def test_round_trip_dim2(codec, data_type, array, shape):
-    # type: (VariantCodec, np.array[Any], np.array[np.uint64])->None
-    td = TypeDescriptor(data_type, Dimension.Dim2)
-    f = DynamicFieldBuffer(td, array, shape)
-    res = encode_field(codec, f)
-
-    res_dec = decode_field(td, res.encoded_field, np.array(res.buffer))
-    assert_equal(array, np.array(res_dec.values_buffer).view(array.dtype))
-    assert_equal(shape, np.array(res_dec.shape_buffer).view(np.uint64))
diff --git a/python/tests/unit/arcticdb/version_store/test_array_column_type.py b/python/tests/unit/arcticdb/version_store/test_array_column_type.py
new file mode 100644
index 0000000000..7b7ec4da65
--- /dev/null
+++ b/python/tests/unit/arcticdb/version_store/test_array_column_type.py
@@ -0,0 +1,149 @@
+"""
+Copyright 2023 Man Group Operations Limited
+
+Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+
+As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+"""
+import pandas as pd
+from pandas.testing import assert_frame_equal
+import numpy as np
+import pytest
+
+
+@pytest.fixture(params=("int32", "float32", "int64", "float64"))
+def array_type(request):
+    yield request.param
+
+
+def assert_db_in_out_match(version_store, df_in, symbol):
+    version_store.write(symbol, df_in)
+    df_out = version_store.read(symbol)
+    assert_frame_equal(df_in, df_out.data)
+
+
+class TestEmptyArrays:
+    def test_single_empty_array(self, lmdb_version_store):
+        df_in = pd.DataFrame({"col1": [np.array([])]})
+        assert_db_in_out_match(lmdb_version_store, df_in, "test_single_empty_array")
+
+    def test_multiple_empty_arrays(self, lmdb_version_store):
+        df_in = pd.DataFrame({"col1": [np.array([]), np.array([]), np.array([])]})
+        assert_db_in_out_match(lmdb_version_store, df_in, "test_multiple_empty_arrays")
+
+    def test_empty_array_can_coexist_with_nonempty_arrays(self, lmdb_version_store, array_type):
+        df_in = pd.DataFrame(
+            {
+                "col1": [
+                    np.array([]).astype(array_type),
+                    np.array([1, 2, 3, 4, 5]).astype(array_type),
+                    np.array([]).astype(array_type)
+                ]
+            }
+        )
+        assert_db_in_out_match(lmdb_version_store, df_in, "test_empty_array_can_coexist_with_nonempty_arrays")
+
+    def test_append_to_colum_with_empty_array(self, lmdb_version_store, array_type):
+        df_empty = pd.DataFrame({"col1": [np.array([]), np.array([])]})
+        lmdb_version_store.write("test_append_to_colum_with_empty_array", df_empty)
+        df_to_append = pd.DataFrame({"col1": [np.array([1, 2, 3]).astype(array_type)]})
+        lmdb_version_store.append("test_append_to_colum_with_empty_array", df_to_append)
+        df_out = lmdb_version_store.read("test_append_to_colum_with_empty_array")
+        df_target = pd.concat([df_empty, df_to_append], ignore_index=True)
+        assert_frame_equal(df_target, df_out.data)
+
+    def test_append_empty_arrays_to_column(self, lmdb_version_store, array_type):
+        df = pd.DataFrame({"col1": [np.array([1, 2, 3]).astype(array_type)]})
+        lmdb_version_store.write("test_append_to_colum_with_empty_array", df)
+        df_to_append = pd.DataFrame({"col1": [np.array([])]})
+        lmdb_version_store.append("test_append_to_colum_with_empty_array", df_to_append)
+        df_out = lmdb_version_store.read("test_append_to_colum_with_empty_array")
+        df_target = pd.concat([df, df_to_append], ignore_index=True)
+        assert_frame_equal(df_target, df_out.data)
+
+    def test_can_mix_empty_array_and_none_in_segment(self, lmdb_version_store):
+        df = pd.DataFrame({"col1": [np.array([]), None, None, np.array([])]})
+        lmdb_version_store.write("test_can_mix_empty_array_and_none_in_segment", df)
+        assert_db_in_out_match(lmdb_version_store, df, "test_can_mix_empty_array_and_none_in_segment")
+
+    def test_can_append_emptyval_to_empty_array(self, lmdb_version_store):
+        df = pd.DataFrame({"col1": [np.array([]), np.array([])]})
+        lmdb_version_store.write("test_can_append_empty_to_empty_array", df)
+        df_to_append = pd.DataFrame({"col1": [None, None]})
+        lmdb_version_store.append("test_can_append_empty_to_empty_array", df_to_append)
+        df_out = lmdb_version_store.read("test_can_append_empty_to_empty_array")
+        df_target = pd.concat([df, df_to_append], ignore_index=True)
+        assert_frame_equal(df_target, df_out.data)
+
+    def test_can_append_empty_array_to_emptyval(self, lmdb_version_store):
+        df = pd.DataFrame({"col1": [None, None]})
+        lmdb_version_store.write("test_can_append_empty_to_empty_array", df)
+        df_to_append = pd.DataFrame({"col1": [np.array([]), np.array([])]})
+        lmdb_version_store.append("test_can_append_empty_to_empty_array", df_to_append)
+        df_out = lmdb_version_store.read("test_can_append_empty_to_empty_array")
+        df_target = pd.concat([df, df_to_append], ignore_index=True)
+        assert_frame_equal(df_target, df_out.data)
+
+
+class TestNonEmptyArrays:
+    def test_single_array(self, lmdb_version_store, array_type):
+        df_in = pd.DataFrame({"col1": [np.array([1, 2, 3]).astype(array_type)]})
+        assert_db_in_out_match(lmdb_version_store, df_in, "test_single_array")
+
+    def test_differently_shaped_rows(self, lmdb_version_store, array_type):
+        df_in = pd.DataFrame(
+            {
+                "col1": [
+                    np.array([1, 2, 3]).astype(array_type),
+                    np.array([1]).astype(array_type),
+                    np.array([1, 2, 3, 4]).astype(array_type),
+                    np.array([2, 4]).astype(array_type),
+                ]
+            }
+        )
+        assert_db_in_out_match(lmdb_version_store, df_in, "test_differently_shaped_rows")
+
+    def test_rows_can_be_none(self, lmdb_version_store, array_type):
+        df_in = pd.DataFrame({"col1": [None, np.array([1, 2]).astype(array_type), None]})
+        assert_db_in_out_match(lmdb_version_store, df_in, "test_rows_can_be_none")
+
+    def test_can_contain_nan(self, lmdb_version_store):
+        # Note integers don't have a NaN value
+        df_in = pd.DataFrame({"col1": [np.array([np.NAN, float("NaN")])]})
+        assert_db_in_out_match(lmdb_version_store, df_in, "test_can_contain_nan")
+
+    def test_can_append(self, lmdb_version_store, array_type):
+        df = pd.DataFrame({"col1": [np.array([1, 2, 3]).astype(array_type)]})
+        lmdb_version_store.write("test_can_append", df)
+        df_to_append = pd.DataFrame(
+            {"col1": [np.array([10]).astype(array_type), np.array([20, 30, 40, 50]).astype(array_type)]}
+        )
+        lmdb_version_store.append("test_can_append", df_to_append)
+        df_out = lmdb_version_store.read("test_can_append")
+        df_target = pd.concat([df, df_to_append], ignore_index=True)
+        assert_frame_equal(df_target, df_out.data)
+
+
+class TestFailure:
+    def test_cannot_mix_scalars_and_arrays(self, lmdb_version_store):
+        with pytest.raises(Exception):
+            df = pd.DataFrame({"col1": [1, np.array([1, 2, 3])]})
+            lmdb_version_store.write("test_cannot_mix_scalars_and_arrays", df)
+
+        with pytest.raises(Exception):
+            df = pd.DataFrame({"col1": [np.array([1, 2, 3]), 1]})
+            lmdb_version_store.write("test_cannot_mix_scalars_and_arrays", df)
+
+    def test_cannot_append_scalar_to_array(self, lmdb_version_store):
+        df = pd.DataFrame({"col": [np.array([1, 2, 3])]})
+        lmdb_version_store.write("test_cannot_append_scalar_to_array", df)
+        df_to_append = pd.DataFrame({"col": [1]})
+        with pytest.raises(Exception):
+            lmdb_version_store.append("test_cannot_append_scalar_to_array", df_to_append)
+
+    def test_cannot_append_array_to_scalar(self, lmdb_version_store):
+        df = pd.DataFrame({"col": [1]})
+        lmdb_version_store.write("test_cannot_append_scalar_to_array", df)
+        df_to_append = pd.DataFrame({"col": [np.array([1, 2, 3])]})
+        with pytest.raises(Exception):
+            lmdb_version_store.append("test_cannot_append_scalar_to_array", df_to_append)
diff --git a/python/tests/unit/arcticdb/version_store/test_empty_column_type.py b/python/tests/unit/arcticdb/version_store/test_empty_column_type.py
new file mode 100644
index 0000000000..1dc64f9881
--- /dev/null
+++ b/python/tests/unit/arcticdb/version_store/test_empty_column_type.py
@@ -0,0 +1,44 @@
+"""
+Copyright 2023 Man Group Operations Limited
+
+Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+
+As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+"""
+import pandas as pd
+from pandas.testing import assert_frame_equal
+import numpy as np
+import pytest
+
+@pytest.fixture(params=(int, float, str))
+def element_type_constructor(request):
+    yield request.param
+
+
+@pytest.mark.xfail(reason="Not implemented")
+class TestEmptyColumnTypeAppend:
+    def test_can_append_non_empty_to_empty(self, lmdb_version_store, element_type_constructor):
+        df_empty = pd.DataFrame({"col1": [None, None]})
+        lmdb_version_store.write("test_can_append_to_empty_column", df_empty)
+        df_non_empty = pd.DataFrame({"col1": map(element_type_constructor, [1, 2, 3])})
+        lmdb_version_store.append("test_can_append_to_empty_column", df_non_empty)
+        expected_result = pd.concat([df_empty, df_non_empty], ignore_index=True)
+        assert_frame_equal(lmdb_version_store.read("test_can_append_to_empty_column").data, expected_result)
+
+    def test_can_append_empty_to_non_empty(self, lmdb_version_store, element_type_constructor):
+        df_empty = pd.DataFrame({"col1": [None, None]})
+        df_non_empty = pd.DataFrame({"col1": map(element_type_constructor, [1, 2, 3])})
+        lmdb_version_store.write("test_can_append_to_empty_column", df_non_empty)
+        lmdb_version_store.append("test_can_append_to_empty_column", df_empty)
+        expected_result = pd.concat([df_non_empty, df_empty], ignore_index=True)
+        assert_frame_equal(lmdb_version_store.read("test_can_append_to_empty_column").data, expected_result)
+
+    def test_can_append_empty_to_empty(self, lmdb_version_store, element_type_constructor):
+        df_empty_1 = pd.DataFrame({"col1": [None, None]})
+        df_empty_2 = pd.DataFrame({"col1": []})
+        df_empty_3 = pd.DataFrame({"col1": [None]})
+        lmdb_version_store.write("test_can_append_to_empty_column", df_empty_1)
+        lmdb_version_store.append("test_can_append_to_empty_column", df_empty_2)
+        lmdb_version_store.append("test_can_append_to_empty_column", df_empty_3)
+        expected_result = pd.concat([df_empty_1, df_empty_2, df_empty_3], ignore_index=True)
+        assert_frame_equal(lmdb_version_store.read("test_can_append_to_empty_column").data, expected_result)
diff --git a/python/tests/unit/arcticdb/version_store/test_nullable_boolean_column_type.py b/python/tests/unit/arcticdb/version_store/test_nullable_boolean_column_type.py
new file mode 100644
index 0000000000..838ae4d2fa
--- /dev/null
+++ b/python/tests/unit/arcticdb/version_store/test_nullable_boolean_column_type.py
@@ -0,0 +1,38 @@
+"""
+Copyright 2023 Man Group Operations Limited
+
+Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+
+As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+"""
+from arcticdb.arctic import Arctic
+import pandas as pd
+from pandas.testing import assert_frame_equal
+import numpy as np
+import pytest
+
+
+@pytest.fixture(params=(list, lambda arr: np.array(arr, dtype="bool"), lambda arr: pd.array(arr, dtype="bool")))
+def array_constructor(request):
+    yield request.param
+
+
+def assert_db_in_out_match(version_store, df_in):
+    version_store.write("test", df_in)
+    df_out = version_store.read("test")
+    assert_frame_equal(df_in, df_out.data)
+
+
+def test_all_none(lmdb_version_store, array_constructor):
+    df_in = pd.DataFrame({"col1": array_constructor([None, None, None])})
+    assert_db_in_out_match(lmdb_version_store, df_in)
+
+
+def test_values_and_none(lmdb_version_store, array_constructor):
+    df_in = pd.DataFrame({"col1": array_constructor([True, None, False, None])})
+    assert_db_in_out_match(lmdb_version_store, df_in)
+
+
+def test_values_only(lmdb_version_store, array_constructor):
+    df_in = pd.DataFrame({"col1": array_constructor([True, False])})
+    assert_db_in_out_match(lmdb_version_store, df_in)