Repository: meilisearch/meilisearch
File: crates/milli/src/update/new/words_prefix_docids.rs (Rust)

Code before:
                        }
                    }
                }

                Ok((indexes, file))
            })
            .collect::<Result<Vec<_>>>()?;

        // We iterate over all the collected and serialized bitmaps through
        // the files and entries to eventually put them in the final database.
        let mut key_buffer = Vec::new();
        let mut buffer = Vec::new();
        for (index, file) in outputs {
            let mut file = file.into_inner().map_err(|e| e.into_error())?;
            file.rewind()?;
            let mut file = BufReader::new(file);
            for PrefixIntegerEntry { prefix, pos, serialized_length } in index {
                key_buffer.clear();
                key_buffer.extend_from_slice(prefix.as_bytes());
                key_buffer.push(0);
                key_buffer.extend_from_slice(&pos.to_be_bytes());
                match serialized_length {
                    Some(serialized_length) => {
                        buffer.resize(serialized_length, 0);
                        file.read_exact(&mut buffer)?;
                        self.prefix_database.remap_data_type::<Bytes>().put(
                            wtxn,
                            &key_buffer,
                            &buffer,
                        )?;
                    }
                    None => {
                        self.prefix_database.delete(wtxn, &key_buffer)?;
                    }
                }
            }
        }

        Ok(())
    }

    #[tracing::instrument(level = "trace", skip_all, target = "indexing::prefix")]
    fn recompute_modified_prefixes(
        &self,
        wtxn: &mut RwTxn,
        prefixes: &BTreeSet<Prefix>,
    ) -> Result<()> {
        // We fetch the docids associated to the newly added word prefix fst only.
        // And collect the CboRoaringBitmaps pointers in an HashMap.
        let frozen = FrozenPrefixIntegerBitmaps::from_prefixes(self.database, wtxn, prefixes)?;


Review comment (suggestion):
The source database is serialized as `word, 0, position -> docids`, which means it will be ordered as below:
```
he, 0, 0 -> [docids]
he, 0, 1 -> [docids]
he, 0, 2 -> [docids]
hell, 0, 0 -> [docids]
hell, 0, 1 -> [docids]
hell, 0, 2 -> [docids]
hello, 0, 0 -> [docids]
hello, 0, 1 -> [docids]
hello, 0, 2 -> [docids]
heritage, 0, 0 -> [docids]
heritage, 0, 1 -> [docids]
heritage, 0, 2 -> [docids]
```

Meaning that for the prefix `h`, it would create more than one `PrefixIntegerEntry` per position:

```rust
PrefixIntegerEntry { prefix: "h", pos: 0, .. } // he, 0, 0
PrefixIntegerEntry { prefix: "h", pos: 1, .. } // he, 0, 1 : because we changed position
PrefixIntegerEntry { prefix: "h", pos: 2, .. } // he, 0, 2 : because we changed position
PrefixIntegerEntry { prefix: "h", pos: 0, .. } // hell, 0, 0 : we come back to position 0 and we push the same PrefixIntegerEntry as the first one but with different docids.
...
```

then when writing into the database, we iterate over the indexes `for PrefixIntegerEntry { prefix, pos, serialized_length } in index` and override the value.

I think the persisted value for each prefixes will only contain the  `docids` of the last word starting with the prefix for each position.
In our example, we would only have the `heritage` docids for the `h` prefix.

Am I right, or am I missing something? 🤔 



Code after:
            let mut file = file.into_inner().map_err(|e| e.into_error())?;
            file.rewind()?;
            let mut file = BufReader::new(file);
            for PrefixIntegerEntry { prefix, pos, serialized_length } in index {
                key_buffer.clear();
                key_buffer.extend_from_slice(prefix.as_bytes());
                key_buffer.push(0);
                key_buffer.extend_from_slice(&pos.to_be_bytes());
                match serialized_length {
                    Some(serialized_length) => {
                        buffer.resize(serialized_length, 0);
                        file.read_exact(&mut buffer)?;
                        self.prefix_database.remap_data_type::<Bytes>().put(
                            wtxn,
                            &key_buffer,
                            &buffer,
                        )?;
                    }
                    None => {
                        self.prefix_database.delete(wtxn, &key_buffer)?;
                    }
                }
            }
        }

        Ok(())
    }

    #[tracing::instrument(level = "trace", skip_all, target = "indexing::prefix")]
    fn recompute_modified_prefixes(
        &self,
        wtxn: &mut RwTxn,
        prefixes: &BTreeSet<Prefix>,
    ) -> Result<()> {
        // We fetch the docids associated to the newly added word prefix fst only.
        // And collect the CboRoaringBitmaps pointers in an HashMap.
        let frozen = FrozenPrefixIntegerBitmaps::from_prefixes(self.database, wtxn, prefixes)?;

        // We access this HashMap in parallel to compute the *union* of all
        // of them and *serialize* them into files. There is one file by CPU.
        let local_entries = ThreadLocal::with_capacity(rayon::current_num_threads());
        prefixes.into_par_iter().map(AsRef::as_ref).try_for_each(|prefix| -> Result<()> {
            let refcell = local_entries.get_or(|| {
                let file = BufWriter::new(spooled_tempfile(
                    self.max_memory_by_thread.unwrap_or(usize::MAX),
                ));
                RefCell::new((Vec::new(), file, Vec::new()))
            });

            let mut refmut = refcell.borrow_mut_or_yield();
            let (ref mut index, ref mut file, ref mut buffer) = *refmut;