Repository: mabel-dev/opteryx

Issue:
🪲 memcached test failure
### Thank you for taking the time to report a problem with Opteryx.
_To help us to respond to your request we ask that you try to provide the below detail about the bug._

**Describe the bug** _A clear and specific description of what the bug is. What the error, incorrect or unexpected behaviour was._


**Expected behaviour** _A clear and concise description of what you expected to happen._


**Sample Code/Statement** _If you can, please submit the SQL statement or Python code snippet, or a representative example using the sample datasets._

~~~sql

~~~

**Additional context** _Add any other context about the problem here, for example what you have done to try to diagnose or workaround the problem._



Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 438
+__build__ = 440
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/shared/buffer_pool.py b/opteryx/shared/buffer_pool.py
--- a/opteryx/shared/buffer_pool.py
+++ b/opteryx/shared/buffer_pool.py
@@ -48,7 +48,7 @@ def __init__(self):
             name="BufferPool", size=cache_manager.max_local_buffer_capacity
         )
 
-    def get(self, key: bytes) -> bytes:
+    def get(self, key: bytes) -> Optional[bytes]:
         """
         Retrieve an item from the pool, return None if the item isn't found.
         If cache is provided and item is not in pool, attempt to get it from cache.
@@ -56,6 +56,7 @@ def get(self, key: bytes) -> bytes:
         mp_key = self._lru.get(key)
         if mp_key is not None:
             return self._memory_pool.read(mp_key)
+        return None
 
     def set(self, key: bytes, value) -> Optional[str]:
         """
diff --git a/opteryx/shared/memory_pool.py b/opteryx/shared/memory_pool.py
--- a/opteryx/shared/memory_pool.py
+++ b/opteryx/shared/memory_pool.py
@@ -121,6 +121,14 @@ def commit(self, data: bytes) -> int:
         not enough, we consolidate all of the free blocks together.
         """
         len_data = len(data)
+        # special case for 0 byte segments
+        if len_data == 0:
+            new_segment = MemorySegment(0, 0)
+            ref_id = random_int()
+            self.used_segments[ref_id] = new_segment
+            self.commits += 1
+            return ref_id
+
         # always acquire a lock to write
         with self.lock:
             segment_index = self._find_free_segment(len_data)


Test patch:
diff --git a/tests/misc/test_memory_pool.py b/tests/misc/test_memory_pool.py
--- a/tests/misc/test_memory_pool.py
+++ b/tests/misc/test_memory_pool.py
@@ -1,10 +1,14 @@
 import os
 import sys
 import pytest
+import random
+import time
+import threading
 
 sys.path.insert(1, os.path.join(sys.path[0], "../.."))
 
 from opteryx.shared import MemoryPool
+from orso.tools import random_string
 
 
 def test_commit_and_read():
@@ -108,7 +112,187 @@ def test_pool_only_l1_compaction():
     assert mp.l2_compaction < mp.l1_compaction
 
 
+def test_repeated_commits_and_releases():
+    mp = MemoryPool(size=4000)
+    refs = []
+    for _ in range(1000):
+        ref = mp.commit(b"Data")
+        assert ref is not None
+        refs.append(ref)
+    for ref in refs:
+        mp.release(ref)
+    # Optional: Check internal state to ensure all resources are available again
+    mp._level1_compaction()
+    assert (
+        mp.free_segments[0].length == mp.size
+    ), "Memory leak detected after repeated commits and releases."
+
+
+def test_stress_with_random_sized_data():
+    mp = MemoryPool(size=500 * 200)
+    refs = []
+    try:
+        for _ in range(500):
+            size = random.randint(20, 200)  # Random data size between 20 and 200 bytes
+            data = bytes(size)
+            ref = mp.commit(data)
+            if ref is not None:
+                refs.append((ref, size))
+    finally:
+        for ref, _ in refs:
+            mp.release(ref)
+    # Ensure that the pool is not fragmented or leaking
+    mp_available_space = sum(segment.length for segment in mp.free_segments)
+    assert mp_available_space >= mp.size - sum(
+        size for _, size in refs if size < mp.size
+    ), "Memory fragmentation or leak detected."
+
+
+def test_compaction_effectiveness():
+    mp = MemoryPool(size=15)
+    # Fill the pool with alternating commitments and releases to create fragmentation
+    ref1 = mp.commit(b"AAA")
+    ref2 = mp.commit(b"BBBBB")
+    ref3 = mp.commit(b"CCCCC")
+    mp.release(ref2)
+    mp.release(ref1)
+    # This should trigger a compaction
+    ref4 = mp.commit(b"DDDDDDDD")
+    assert ref4 is not None, "Compaction failed to consolidate free memory effectively."
+    mp.release(ref3)
+    mp.release(ref4)
+    assert mp.l1_compaction > 0, "Expected L1 compaction did not occur."
+
+
+def test_repeated_zero_length_commits():
+    mp = MemoryPool(size=100)  # Assuming a moderate size for the memory pool.
+    refs = []
+    try:
+        # Commit a zero-length array many times.
+        for _ in range(1000):
+            ref = mp.commit(b"")  # Committing an empty byte string.
+            assert ref is not None, "Commit of zero-length data should not fail."
+            refs.append(ref)
+
+        # Check if each reference is unique which implies correct handling of zero-length data.
+        assert len(set(refs)) == len(refs), "References should be unique even for zero-length data."
+
+    finally:
+        # Cleanup: release all references.
+        for ref in refs:
+            mp.release(ref)
+
+    # Optional: Verify if the memory pool is back to its initial state.
+    mp_available_space = sum(segment.length for segment in mp.free_segments)
+    assert mp_available_space == mp.size, "Memory pool did not recover fully after releases."
+
+
+def test_rapid_commit_release():
+    mp = MemoryPool(size=1000)
+    for _ in range(1000):
+        ref = mp.commit(b"Some data")
+        assert ref is not None, "Failed to commit data."
+        mp.release(ref)
+    # Verify memory integrity and state post rapid operations
+    mp_available_space = sum(segment.length for segment in mp.free_segments)
+    assert mp_available_space == mp.size, "Memory pool did not return to full availability."
+
+
+def test_commit_max_capacity():
+    data = b"a" * 1000  # Assuming the pool size is 1000 bytes.
+    mp = MemoryPool(size=1000)
+    ref = mp.commit(data)
+    assert ref is not None, "Failed to commit data that exactly matches the pool capacity."
+    mp.release(ref)
+    mp_available_space = sum(segment.length for segment in mp.free_segments)
+    assert mp_available_space == mp.size, "Memory pool did not correctly free up space."
+
+
+def test_sequential_commits_without_space():
+    mp = MemoryPool(size=500)
+    mp.commit(b"a" * 250)
+    mp.commit(b"b" * 250)
+    ref = mp.commit(b"Third commit should fail")
+    assert ref is None, "Memory pool erroneously allowed overcommitment."
+
+
+def test_stress_with_variable_data_sizes():
+    mp = MemoryPool(size=10000)
+    refs = []
+    try:
+        for _ in range(100):
+            size = random.randint(1, 200)  # Data size between 1 and 200 bytes.
+            data = bytes(random.choices(range(256), k=size))
+            ref = mp.commit(data)
+            if ref is not None:
+                refs.append(ref)
+    finally:
+        for ref in refs:
+            mp.release(ref)
+    # Ensure all space is reclaimed
+    mp_available_space = sum(segment.length for segment in mp.free_segments)
+    assert mp_available_space == mp.size, "Memory leakage detected with variable data sizes."
+
+
+def test_zero_byte_commit_on_full_pool():
+    mp = MemoryPool(size=500)
+    # Fill up the pool
+    mp.commit(b"x" * 500)
+    # Try committing zero bytes
+    ref = mp.commit(b"")
+    assert ref is not None, "Memory pool failed to handle zero-byte commit on a full pool."
+
+
+def test_random_release_order():
+    mp = MemoryPool(size=1000)
+    refs = [mp.commit(b"Data" + bytes([i])) for i in range(10)]
+    random.shuffle(refs)  # Randomize the order of releases
+    for ref in refs:
+        mp.release(ref)
+    mp_available_space = sum(segment.length for segment in mp.free_segments)
+    assert (
+        mp_available_space == mp.size
+    ), "Memory pool failed to reclaim space correctly after random releases."
+
+
+def test_concurrent_access():
+    pool_size = 100
+    memory_pool = MemoryPool(size=pool_size)
+    threads = []
+    errors = []
+
+    def thread_task():
+        try:
+            for _ in range(500):  # Number of operations per thread
+                time.sleep(0.001)  # It's fast, add in a delay to the threads overlap
+                data = random_string().encode()  # unique per write
+                ref = memory_pool.commit(data)
+                if ref is not None:
+                    read_data = memory_pool.read(ref)
+                    assert read_data == data, "Data integrity check failed"
+                    memory_pool.release(ref)
+                else:
+                    # Handle commit failure (e.g., no space)
+                    pass
+        except AssertionError as e:
+            errors.append(str(e))
+
+    # Create multiple threads to simulate concurrent access
+    for _ in range(32):
+        thread = threading.Thread(target=thread_task)
+        threads.append(thread)
+        thread.start()
+
+    # Wait for all threads to complete
+    for thread in threads:
+        thread.join()
+
+    # Check for errors collected during the test
+    assert not errors, f"Errors encountered during concurrent access: {errors}"
+
+
 if __name__ == "__main__":  # pragma: no cover
     from tests.tools import run_tests
 
+    test_concurrent_access()
     run_tests()
diff --git a/tests/storage/test_cache_memcached.py b/tests/storage/test_cache_memcached.py
--- a/tests/storage/test_cache_memcached.py
+++ b/tests/storage/test_cache_memcached.py
@@ -41,7 +41,7 @@ def test_memcached_cache():
         cache.errors == 0
     ), f"hits: {cache.hits}, misses: {cache.misses}, skips: {cache.skips}, errors: {cache.errors}"
 
-    assert stats["cache_hits"] >= stats["blobs_read"], stats
+    assert stats["remote_cache_hits"] >= stats["blobs_read"], stats
     # assert stats.get("cache_misses", 0) == 0, stats