Repository: meilisearch/meilisearch
File: milli/src/update/index_documents/extract/extract_docid_word_positions.rs (Rust)

Code before:
        // context: https://github.com/meilisearch/meilisearch/issues/3565
        if script_language_word_count.values().any(potential_language_detection_error) {
            // build an allow list with the most frequent detected languages in the document.
            let script_language: HashMap<_, _> =
                script_language_word_count.iter().filter_map(most_frequent_languages).collect();

            // if the allow list is empty, meaning that no Language is considered frequent,
            // then we don't rerun the extraction.
            if !script_language.is_empty() {
                // build a new temporar tokenizer including the allow list.
                let mut tokenizer_builder = TokenizerBuilder::new();
                if let Some(stop_words) = stop_words {
                    tokenizer_builder.stop_words(stop_words);
                }
                tokenizer_builder.allow_list(&script_language);
                let tokenizer = tokenizer_builder.build();

                script_language_word_count.clear();

                // rerun the extraction.
                extract_tokens_from_document(
                    &obkv,
                    searchable_fields,
                    &tokenizer,
                    max_positions_per_attributes,
                    &mut key_buffer,
                    &mut field_buffer,
                    &mut script_language_word_count,
                    &mut docid_word_positions_sorter,
                )?;
            }
        }

        for (script, languages_frequency) in script_language_word_count {
            for (language, _) in languages_frequency {
                let entry = script_language_docids
                    .entry((script, language))
                    .or_insert_with(RoaringBitmap::new);
                entry.push(document_id);
            }
        }
    }

    sorter_into_reader(docid_word_positions_sorter, indexer)
        .map(|reader| (documents_ids, reader, script_language_docids))
}

#[allow(clippy::too_many_arguments)]
fn extract_tokens_from_document<T: AsRef<[u8]>>(
    obkv: &KvReader<FieldId>,
    searchable_fields: &Option<HashSet<FieldId>>,

Review comment (refactor):
When we rerun the extraction, we do not clear the `key_buffer` again. Could that be an issue?

Code after:
            .values()
            .map(Vec::as_slice)
            .any(potential_language_detection_error)
        {
            // build an allow list with the most frequent detected languages in the document.
            let script_language: HashMap<_, _> =
                script_language_word_count.iter().filter_map(most_frequent_languages).collect();

            // if the allow list is empty, meaning that no Language is considered frequent,
            // then we don't rerun the extraction.
            if !script_language.is_empty() {
                // build a new temporary tokenizer including the allow list.
                let mut tokenizer_builder = TokenizerBuilder::new();
                if let Some(stop_words) = stop_words {
                    tokenizer_builder.stop_words(stop_words);
                }
                tokenizer_builder.allow_list(&script_language);
                let tokenizer = tokenizer_builder.build();

                script_language_word_count.clear();

                // rerun the extraction.
                extract_tokens_from_document(
                    &obkv,
                    searchable_fields,
                    &tokenizer,
                    max_positions_per_attributes,
                    &mut buffers,
                    &mut script_language_word_count,
                    &mut docid_word_positions_sorter,
                )?;
            }
        }

        for (script, languages_frequency) in script_language_word_count {
            for (language, _) in languages_frequency {
                let entry = script_language_docids
                    .entry((script, language))
                    .or_insert_with(RoaringBitmap::new);
                entry.push(document_id);
            }
        }
    }

    sorter_into_reader(docid_word_positions_sorter, indexer)
        .map(|reader| (documents_ids, reader, script_language_docids))
}

fn extract_tokens_from_document<T: AsRef<[u8]>>(
    obkv: &KvReader<FieldId>,
    searchable_fields: &Option<HashSet<FieldId>>,