Repository: qdrant/qdrant (language: rust)

Issue:
Panic on search after GPU indexing
<!--- Provide a general summary of the issue in the Title above -->

## Current Behavior

Search fails after index was build with GPU


## Steps to Reproduce

Build index on the machine with GPU

```
docker run --rm -it -e --network=host qdrant/bfb:dev ./bfb -n 14000000 -d 768 -p 10 -t 2 --quantization-in-ram true --quantization binary --on-disk-payload --on-disk-vectors true --max-segment-size 2000000
```

After indexing, start searches:

```
docker run --rm -it --network=host qdrant/bfb:dev ./bfb -d 768 -n 10000 --quantization binary --skip-create --skip-upload --skip-wait-index --quantization-rescore false --search
```


Search randomly crashes with

```
2025-01-20T20:41:18.667923Z ERROR qdrant::startup: Panic occurred in file lib/segment/src/index/hnsw_index/graph_links.rs at line 233: called `Option::unwrap()` on a `None` value

2025-01-20T20:41:18.805262Z DEBUG collection::shards::replica_set::execute_read_operation: Read operation failed: Service internal error: task 555390 panicked with message "called `Option::unwrap()` on a `None` value"
2025-01-20T20:41:18.806298Z ERROR qdrant::tonic::logging: gRPC /qdrant.Points/Search unexpectedly failed with Internal error "Service internal error: 1 of 1 read operations failed:\n  Service internal error: task 555390 panicked with message \"called `Option::unwrap()` on a `None` value\"" 0.142194
2025-01-20T20:41:18.809868Z ERROR qdrant::startup: Panic backtrace:
   0: qdrant::startup::setup_panic_hook::{{closure}}
   1: std::panicking::rust_panic_with_hook
   2: std::panicking::begin_panic_handler::{{closure}}
   3: std::sys::backtrace::__rust_end_short_backtrace
   4: rust_begin_unwind
   5: core::panicking::panic_fmt
   6: core::panicking::panic
   7: core::option::unwrap_failed
   8: segment::index::hnsw_index::hnsw::HNSWIndex::search_with_graph
   9: <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::next
  10: segment::index::hnsw_index::hnsw::HNSWIndex::search_vectors_with_graph
  11: <segment::index::vector_index_base::VectorIndexEnum as segment::index::vector_index_base::VectorIndex>::search
  12: segment::segment::entry::<impl segment::entry::entry_point::SegmentEntry for segment::segment::Segment>::search_batch
  13: collection::collection_manager::segments_searcher::execute_batch_search
  14: collection::collection_manager::segments_searcher::search_in_segment
  15: tokio::runtime::task::raw::poll
  16: std::sys::backtrace::__rust_begin_short_backtrace
  17: core::ops::function::FnOnce::call_once{{vtable.shim}}
  18: std::sys::pal::unix::thread::Thread::new::thread_start
  19: <unknown>
  20: clone
```


If this is related to GPU or HNSW index compression is unknown yet


Discussion / hints:
After the index is built, some points on higher levels have a link to point with index `0`. e.g.

```
edges[27943][2] == [8999, 28351, 5774, 26931, 42304, 31243, 3647, 41559, 8026, 0, 4992, 27757, 37936, 17025, 40831, 11382]
             ^ level=2                                                         ^ ???
```

But the point with index `0` is not present on level 2, i.e. `edges[0].len() == 1`.

In [this test commit](https://github.com/qdrant/qdrant/commit/ff67dedb669026a5d21813f01715c6180b718211) I've added an assert to catch this situation earlier inside `GraphLinksConverter::new`.

I suspect it's related GPU as I couldn't reproduce it with CPU. cc @IvanPleshkov
Fixed by <https://github.com/qdrant/qdrant/pull/5848>

Fix patch:
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/links.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/links.comp
index 25cb4cdc80e..888d60ee9df 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/links.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/links.comp
@@ -14,7 +14,7 @@ readonly uniform LinksParams {
 
 layout(set = LINKS_LAYOUT_SET, binding = 1)
 buffer Links {
-    POINT_ID data[];
+    coherent POINT_ID data[];
 } links;
 
 #define LEVEL_M links_params.m
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/run_greedy_search.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/run_greedy_search.comp
index d675d3fd0b0..e85d1f14003 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/run_greedy_search.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/run_greedy_search.comp
@@ -22,11 +22,11 @@ struct SearchRequest {
 };
 
 layout(set = 0, binding = 0) buffer SearchRequests {
-    SearchRequest data[];
+    readonly SearchRequest data[];
 } search_requests;
 
 layout(set = 0, binding = 1) buffer SearchResults {
-    uint data[];
+    writeonly uint data[];
 } search_results;
 
 void main() {
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/run_insert_vector.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/run_insert_vector.comp
index 8a9f9bfa124..28aa3846883 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/run_insert_vector.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/run_insert_vector.comp
@@ -22,15 +22,15 @@ struct Request {
 };
 
 layout(set = 0, binding = 0) buffer Requests {
-    Request data[];
+    readonly Request data[];
 } requests;
 
 layout(set = 0, binding = 1) buffer NewEntries {
-    uint data[];
+    writeonly uint data[];
 } new_entries;
 
 layout(set = 0, binding = 2) buffer Atomics {
-    uint data[];
+    coherent uint data[];
 } atomics;
 
 void main() {
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/visited_flags.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/visited_flags.comp
index 19ea3af7cb2..d001a00c004 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/visited_flags.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/visited_flags.comp
@@ -15,13 +15,13 @@ readonly uniform VisitedFlagsParams {
 
 layout(set = VISITED_FLAGS_LAYOUT_SET, binding = 1)
 buffer VisitedFlagsBuffer {
-    uint8_t data[];
+    coherent uint8_t data[];
 } visited_flags;
 
 #ifdef VISITED_FLAGS_REMAP
 layout(set = VISITED_FLAGS_LAYOUT_SET, binding = 2)
 buffer VisitedFlagsRemapBuffer {
-    POINT_ID data[];
+    readonly POINT_ID data[];
 } visited_flags_remap;
 #endif
 


Test patch:
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_candidates_heap.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_candidates_heap.comp
index f0b06ee513d..128fdef29a0 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_candidates_heap.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_candidates_heap.comp
@@ -17,11 +17,11 @@ readonly uniform TestParams {
 } test_params;
 
 layout(set = 0, binding = 1) buffer ScoredPointsInput {
-    ScoredPoint data[];
+    readonly ScoredPoint data[];
 } input_points;
 
 layout(set = 0, binding = 2) buffer ScoresOuput {
-    ScoredPoint data[];
+    writeonly ScoredPoint data[];
 } scores_output;
 
 uint candidates_count = 0;
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_heuristic.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_heuristic.comp
index e7fd01ae6e0..20f9c3c7e17 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_heuristic.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_heuristic.comp
@@ -22,11 +22,11 @@ struct SearchRequest {
 };
 
 layout(set = 0, binding = 0) buffer SearchRequests {
-    SearchRequest data[];
+    readonly SearchRequest data[];
 } search_requests;
 
 layout(set = 0, binding = 1) buffer SearchResults {
-    ScoredPoint data[];
+    writeonly ScoredPoint data[];
 } search_results;
 
 void main() {
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_hnsw_search.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_hnsw_search.comp
index 0ca3baa86b8..5355b0aba49 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_hnsw_search.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_hnsw_search.comp
@@ -22,11 +22,11 @@ struct SearchRequest {
 };
 
 layout(set = 0, binding = 0) buffer SearchRequests {
-    SearchRequest data[];
+    readonly SearchRequest data[];
 } search_requests;
 
 layout(set = 0, binding = 1) buffer SearchResults {
-    ScoredPoint data[];
+    writeonly ScoredPoint data[];
 } search_results;
 
 void main() {
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_nearest_heap.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_nearest_heap.comp
index 23a02e9d4ef..b3088e0f011 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_nearest_heap.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_nearest_heap.comp
@@ -17,15 +17,15 @@ readonly uniform TestParams {
 } test_params;
 
 layout(set = 0, binding = 1) buffer ScoredPointsInput {
-    ScoredPoint data[];
+    readonly ScoredPoint data[];
 } input_points;
 
 layout(set = 0, binding = 2) buffer ScoresOuput {
-    float data[];
+    writeonly float data[];
 } scores_output;
 
 layout(set = 0, binding = 3) buffer SortedOuput {
-    uint data[];
+    writeonly uint data[];
 } sorted_output;
 
 uint nearest_count = 0;
diff --git a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_vector_storage.comp b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_vector_storage.comp
index e713e2f5fd7..56451f61067 100644
--- a/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_vector_storage.comp
+++ b/lib/segment/src/index/hnsw_index/gpu/shaders/tests/test_vector_storage.comp
@@ -9,7 +9,7 @@ layout(local_size_x = SUBGROUP_SIZE, local_size_y = 1, local_size_z = 1) in;
 #include "vector_storage.comp"
 
 layout(set = 0, binding = 0) buffer ScoresData {
-    float data[];
+    writeonly float data[];
 } scores;
 
 void main() {