Repository: meilisearch/meilisearch
File: milli/src/update/index_documents/extract/extract_docid_word_positions.rs (Rust)

Code before:
                    searchable_fields,
                    &tokenizer,
                    max_positions_per_attributes,
                    &mut key_buffer,
                    &mut field_buffer,
                    &mut script_language_word_count,
                    &mut docid_word_positions_sorter,
                )?;
            }
        }

        for (script, languages_frequency) in script_language_word_count {
            for (language, _) in languages_frequency {
                let entry = script_language_docids
                    .entry((script, language))
                    .or_insert_with(RoaringBitmap::new);
                entry.push(document_id);
            }
        }
    }

    sorter_into_reader(docid_word_positions_sorter, indexer)
        .map(|reader| (documents_ids, reader, script_language_docids))
}

#[allow(clippy::too_many_arguments)]
fn extract_tokens_from_document<T: AsRef<[u8]>>(
    obkv: &KvReader<FieldId>,
    searchable_fields: &Option<HashSet<FieldId>>,
    tokenizer: &Tokenizer<T>,
    max_positions_per_attributes: u32,
    key_buffer: &mut Vec<u8>,
    field_buffer: &mut String,
    script_language_word_count: &mut HashMap<Script, Vec<(Language, usize)>>,
    docid_word_positions_sorter: &mut grenad::Sorter<MergeFn>,
) -> Result<()> {
    for (field_id, field_bytes) in obkv.iter() {
        if searchable_fields.as_ref().map_or(true, |sf| sf.contains(&field_id)) {
            let value = serde_json::from_slice(field_bytes).map_err(InternalError::SerdeJson)?;
            field_buffer.clear();
            if let Some(field) = json_to_string(&value, field_buffer) {
                let tokens = process_tokens(tokenizer.tokenize(field))
                    .take_while(|(p, _)| (*p as u32) < max_positions_per_attributes);

                for (index, token) in tokens {
                    // if a language has been detected for the token, we update the counter.
                    if let Some(language) = token.language {
                        let script = token.script;
                        let entry =
                            script_language_word_count.entry(script).or_insert_with(Vec::new);
                        match entry.iter_mut().find(|(l, _)| *l == language) {

Review comment (suggestion):
I wonder if some of the arguments to this function could be grouped semantically in a struct.

For example, most of the `&mut` arguments appear to be used as a "scratch pad" of sorts, maybe they could be grouped in a struct that also handles clearing them before reuse.

Code after:
                extract_tokens_from_document(
                    &obkv,
                    searchable_fields,
                    &tokenizer,
                    max_positions_per_attributes,
                    &mut buffers,
                    &mut script_language_word_count,
                    &mut docid_word_positions_sorter,
                )?;
            }
        }

        for (script, languages_frequency) in script_language_word_count {
            for (language, _) in languages_frequency {
                let entry = script_language_docids
                    .entry((script, language))
                    .or_insert_with(RoaringBitmap::new);
                entry.push(document_id);
            }
        }
    }

    sorter_into_reader(docid_word_positions_sorter, indexer)
        .map(|reader| (documents_ids, reader, script_language_docids))
}

fn extract_tokens_from_document<T: AsRef<[u8]>>(
    obkv: &KvReader<FieldId>,
    searchable_fields: &Option<HashSet<FieldId>>,
    tokenizer: &Tokenizer<T>,
    max_positions_per_attributes: u32,
    buffers: &mut Buffers,
    script_language_word_count: &mut HashMap<Script, Vec<(Language, usize)>>,
    docid_word_positions_sorter: &mut grenad::Sorter<MergeFn>,
) -> Result<()> {
    for (field_id, field_bytes) in obkv.iter() {
        if searchable_fields.as_ref().map_or(true, |sf| sf.contains(&field_id)) {
            let value = serde_json::from_slice(field_bytes).map_err(InternalError::SerdeJson)?;
            buffers.field_buffer.clear();
            if let Some(field) = json_to_string(&value, &mut buffers.field_buffer) {
                let tokens = process_tokens(tokenizer.tokenize(field))
                    .take_while(|(p, _)| (*p as u32) < max_positions_per_attributes);

                for (index, token) in tokens {
                    // if a language has been detected for the token, we update the counter.
                    if let Some(language) = token.language {
                        let script = token.script;
                        let entry =
                            script_language_word_count.entry(script).or_insert_with(Vec::new);
                        match entry.iter_mut().find(|(l, _)| *l == language) {
                            Some((_, n)) => *n += 1,