Repository: antgroup/vsag (language: python)

Issue:
Add a new Layer "DataCell" to manage data access 
I think high level index should ignore the specific implement of quantization and io type, So the quantization and io components can be managed by new layer DataCell which provide different data access interface to the index layer. just like the following show

|Layer| Description|Implement|
|-------|--------|---------|
|Index Layer | ANN algorithm & architecture.          |                       Graph (HNSW,NNDescent) + Reorder,  IVF(PQ,SQ,fastscan), HNSW_IVF, MultiIndex ......|
|DataCell Layer| data access, use quantization, io and others    | FlattenDC, GraphNeighborhoodDC, MixtureDC .......|
|Operator Layer| quantizations, IO and other basic components                                                 | PQ,SQ,MemoryIO,SSDIO,......|

this abstract layered approach may facilitate scalability and code reuse
currently we only have index layer, so the issue will provide Datacell Layer and Operator Layer in this feature.
the related pull requests:

- #25 
- #34 
- #37 
- #41 
- #35 
- #59 
- #60 
- #61 
- #62 
- #79 
- #80 
- #96 
- #111 
- #114 
- #125 
todo tasks:
- support mixture DataCell
- adapt Graph DataCell for HNSW
- implement HNSW + SQ4 + SQ8 Index 




Discussion / hints:

Fix patch:
diff --git a/src/io/memory_block_io.h b/src/io/memory_block_io.h
index 168b7a62..1056b8fd 100644
--- a/src/io/memory_block_io.h
+++ b/src/io/memory_block_io.h
@@ -24,7 +24,6 @@
 #include <cstring>
 #include <nlohmann/json.hpp>
 #include <stdexcept>
-#include <vector>
 
 #include "basic_io.h"
 #include "common.h"
@@ -37,16 +36,15 @@ namespace vsag {
 class MemoryBlockIO : public BasicIO<MemoryBlockIO> {
 public:
     explicit MemoryBlockIO(Allocator* allocator, uint64_t block_size = DEFAULT_BLOCK_SIZE)
-        : block_size_(block_size), allocator_(allocator) {
+        : block_size_(block_size), allocator_(allocator), blocks_(0, allocator) {
     }
 
     MemoryBlockIO(const nlohmann::json& io_obj, const IndexCommonParam& common_param)
-        : allocator_(common_param.allocator_) {
-        CHECK_ARGUMENT(
-            io_obj.contains(BLOCK_IO_BLOCK_SIZE_KEY),
-            fmt::format("memory block io parameters must contains {}", BLOCK_IO_BLOCK_SIZE_KEY));
-
-        this->block_size_ = io_obj[BLOCK_IO_BLOCK_SIZE_KEY];  // TODO(LHT): trans str to uint64_t
+        : allocator_(common_param.allocator_), blocks_(0, common_param.allocator_) {
+        if (io_obj.contains(BLOCK_IO_BLOCK_SIZE_KEY)) {
+            this->block_size_ =
+                io_obj[BLOCK_IO_BLOCK_SIZE_KEY];  // TODO(LHT): trans str to uint64_t
+        }
     }
 
     ~MemoryBlockIO() override {
@@ -106,7 +104,7 @@ class MemoryBlockIO : public BasicIO<MemoryBlockIO> {
 private:
     uint64_t block_size_{DEFAULT_BLOCK_SIZE};
 
-    std::vector<uint8_t*> blocks_{};
+    Vector<uint8_t*> blocks_;
 
     Allocator* const allocator_{nullptr};
 
@@ -140,9 +138,9 @@ MemoryBlockIO::ReadImpl(uint64_t size, uint64_t offset, uint8_t* data) const {
         auto start_off = offset % block_size_;
         auto max_size = block_size_ - start_off;
         while (cur_size < size) {
-            const uint8_t* cur_write = blocks_[start_no] + start_off;
+            const uint8_t* cur_read = blocks_[start_no] + start_off;
             auto cur_length = std::min(size - cur_size, max_size);
-            memcpy(data + cur_size, cur_write, cur_length);
+            memcpy(data + cur_size, cur_read, cur_length);
             cur_size += cur_length;
             max_size = block_size_;
             ++start_no;
@@ -160,7 +158,7 @@ MemoryBlockIO::ReadImpl(uint64_t size, uint64_t offset, bool& need_release) cons
             return this->get_data_ptr(offset);
         } else {
             need_release = true;
-            uint8_t* ptr = reinterpret_cast<uint8_t*>(allocator_->Allocate(size));
+            auto* ptr = reinterpret_cast<uint8_t*>(allocator_->Allocate(size));
             this->ReadImpl(size, offset, ptr);
             return ptr;
         }
diff --git a/src/io/memory_io.h b/src/io/memory_io.h
index 545afafc..534644ee 100644
--- a/src/io/memory_io.h
+++ b/src/io/memory_io.h
@@ -71,13 +71,13 @@ class MemoryIO : public BasicIO<MemoryIO> {
 
 private:
     [[nodiscard]] inline bool
-    checkValidOffset(uint64_t size) const {
+    check_valid_offset(uint64_t size) const {
         return size <= current_size_;
     }
 
     void
-    checkAndRealloc(uint64_t size) {
-        if (checkValidOffset(size)) {
+    check_and_realloc(uint64_t size) {
+        if (check_valid_offset(size)) {
             return;
         }
         start_ = reinterpret_cast<uint8_t*>(allocator_->Reallocate(start_, size));
@@ -93,13 +93,13 @@ class MemoryIO : public BasicIO<MemoryIO> {
 
 void
 MemoryIO::WriteImpl(const uint8_t* data, uint64_t size, uint64_t offset) {
-    checkAndRealloc(size + offset);
+    check_and_realloc(size + offset);
     memcpy(start_ + offset, data, size);
 }
 
 bool
 MemoryIO::ReadImpl(uint64_t size, uint64_t offset, uint8_t* data) const {
-    bool ret = checkValidOffset(size + offset);
+    bool ret = check_valid_offset(size + offset);
     if (ret) {
         memcpy(data, start_ + offset, size);
     }
@@ -109,7 +109,7 @@ MemoryIO::ReadImpl(uint64_t size, uint64_t offset, uint8_t* data) const {
 const uint8_t*
 MemoryIO::ReadImpl(uint64_t size, uint64_t offset, bool& need_release) const {
     need_release = false;
-    if (checkValidOffset(size + offset)) {
+    if (check_valid_offset(size + offset)) {
         return start_ + offset;
     }
     return nullptr;


Test patch:
diff --git a/src/io/memory_block_io_test.cpp b/src/io/memory_block_io_test.cpp
index 1188d0f3..a1cb762a 100644
--- a/src/io/memory_block_io_test.cpp
+++ b/src/io/memory_block_io_test.cpp
@@ -22,7 +22,7 @@
 #include "default_allocator.h"
 using namespace vsag;
 
-auto block_memory_io_block_sizes = {64, 1023, 4096, 123123, 1024 * 1024 * 1024};
+auto block_memory_io_block_sizes = {64, 1023, 4096, 123123, 1024 * 1024 * 256};
 
 TEST_CASE("read&write [ut][memory_block_io]") {
     auto allocator = std::make_unique<DefaultAllocator>();