Repository: meilisearch/meilisearch
File: milli/src/update/index_documents/extract/extract_docid_word_positions.rs (Rust)

Code before:

        let mut script_language_word_count = HashMap::new();

        extract_tokens_from_document(
            &obkv,
            searchable_fields,
            &tokenizer,
            max_positions_per_attributes,
            &mut key_buffer,
            &mut field_buffer,
            &mut script_language_word_count,
            &mut docid_word_positions_sorter,
        )?;

        // if we detect a potetial mistake in the language detection,
        // we rerun the extraction forcing the tokenizer to detect the most frequently detected Languages.
        // context: https://github.com/meilisearch/meilisearch/issues/3565
        if script_language_word_count.values().any(potential_language_detection_error) {
            // build an allow list with the most frequent detected languages in the document.
            let script_language: HashMap<_, _> =
                script_language_word_count.iter().filter_map(most_frequent_languages).collect();

            // if the allow list is empty, meaning that no Language is considered frequent,
            // then we don't rerun the extraction.
            if !script_language.is_empty() {
                // build a new temporar tokenizer including the allow list.
                let mut tokenizer_builder = TokenizerBuilder::new();
                if let Some(stop_words) = stop_words {
                    tokenizer_builder.stop_words(stop_words);
                }
                tokenizer_builder.allow_list(&script_language);
                let tokenizer = tokenizer_builder.build();

                script_language_word_count.clear();

                // rerun the extraction.
                extract_tokens_from_document(
                    &obkv,
                    searchable_fields,
                    &tokenizer,
                    max_positions_per_attributes,
                    &mut key_buffer,
                    &mut field_buffer,
                    &mut script_language_word_count,
                    &mut docid_word_positions_sorter,
                )?;
            }
        }

        for (script, languages_frequency) in script_language_word_count {
            for (language, _) in languages_frequency {

Review comment (suggestion):
```suggestion
                // build a new temporary tokenizer including the allow list.
```

Code after:
        let mut script_language_word_count = HashMap::new();

        extract_tokens_from_document(
            &obkv,
            searchable_fields,
            &tokenizer,
            max_positions_per_attributes,
            &mut buffers,
            &mut script_language_word_count,
            &mut docid_word_positions_sorter,
        )?;

        // if we detect a potetial mistake in the language detection,
        // we rerun the extraction forcing the tokenizer to detect the most frequently detected Languages.
        // context: https://github.com/meilisearch/meilisearch/issues/3565
        if script_language_word_count
            .values()
            .map(Vec::as_slice)
            .any(potential_language_detection_error)
        {
            // build an allow list with the most frequent detected languages in the document.
            let script_language: HashMap<_, _> =
                script_language_word_count.iter().filter_map(most_frequent_languages).collect();

            // if the allow list is empty, meaning that no Language is considered frequent,
            // then we don't rerun the extraction.
            if !script_language.is_empty() {
                // build a new temporary tokenizer including the allow list.
                let mut tokenizer_builder = TokenizerBuilder::new();
                if let Some(stop_words) = stop_words {
                    tokenizer_builder.stop_words(stop_words);
                }
                tokenizer_builder.allow_list(&script_language);
                let tokenizer = tokenizer_builder.build();

                script_language_word_count.clear();

                // rerun the extraction.
                extract_tokens_from_document(
                    &obkv,
                    searchable_fields,
                    &tokenizer,
                    max_positions_per_attributes,
                    &mut buffers,
                    &mut script_language_word_count,
                    &mut docid_word_positions_sorter,
                )?;
            }
        }

        for (script, languages_frequency) in script_language_word_count {