Repository: meilisearch/meilisearch
File: milli/src/update/index_documents/mod.rs (Rust)

Code before:
                    Some(field_ids)
                } else {
                    None
                }
            }
            None => None,
        };
        // get the fid of the `_vectors` field.
        let vectors_field_id = self.index.fields_ids_map(self.wtxn)?.id("_vectors");

        let stop_words = self.index.stop_words(self.wtxn)?;
        let exact_attributes = self.index.exact_attributes_ids(self.wtxn)?;

        let pool_params = GrenadParameters {
            chunk_compression_type: self.indexer_config.chunk_compression_type,
            chunk_compression_level: self.indexer_config.chunk_compression_level,
            max_memory: self.indexer_config.max_memory,
            max_nb_chunks: self.indexer_config.max_nb_chunks, // default value, may be chosen.
        };
        let documents_chunk_size =
            self.indexer_config.documents_chunk_size.unwrap_or(1024 * 1024 * 4); // 4MiB
        let max_positions_per_attributes = self.indexer_config.max_positions_per_attributes;

        // Run extraction pipeline in parallel.
        pool.install(|| {
            puffin::profile_scope!("split_grenad_by_chunks");
            // split obkv file into several chunks
            let original_chunk_iter =
                grenad_obkv_into_chunks(original_documents, pool_params, documents_chunk_size);

            // split obkv file into several chunks
            let flattened_chunk_iter =
                grenad_obkv_into_chunks(flattened_documents, pool_params, documents_chunk_size);

            let result = original_chunk_iter.and_then(|original_chunk| {
                let flattened_chunk = flattened_chunk_iter?;
                // extract all databases from the chunked obkv douments
                extract::data_from_obkv_documents(
                    original_chunk,
                    flattened_chunk,
                    pool_params,
                    lmdb_writer_sx.clone(),
                    searchable_fields,
                    faceted_fields,
                    primary_key_id,
                    geo_fields_ids,
                    vectors_field_id,
                    stop_words,
                    max_positions_per_attributes,
                    exact_attributes,
                )

Review comment (suggestion):
This profile scope doesn't seem to only measure the `split_grenad_by_chunks` process but`data_from_obkv_documents` too.

Code after:
                    Some(field_ids)
                } else {
                    None
                }
            }
            None => None,
        };
        // get the fid of the `_vectors` field.
        let vectors_field_id = self.index.fields_ids_map(self.wtxn)?.id("_vectors");

        let stop_words = self.index.stop_words(self.wtxn)?;
        let exact_attributes = self.index.exact_attributes_ids(self.wtxn)?;

        let pool_params = GrenadParameters {
            chunk_compression_type: self.indexer_config.chunk_compression_type,
            chunk_compression_level: self.indexer_config.chunk_compression_level,
            max_memory: self.indexer_config.max_memory,
            max_nb_chunks: self.indexer_config.max_nb_chunks, // default value, may be chosen.
        };
        let documents_chunk_size =
            self.indexer_config.documents_chunk_size.unwrap_or(1024 * 1024 * 4); // 4MiB
        let max_positions_per_attributes = self.indexer_config.max_positions_per_attributes;

        // Run extraction pipeline in parallel.
        pool.install(|| {
            puffin::profile_scope!("extract_and_send_grenad_chunks");
            // split obkv file into several chunks
            let original_chunk_iter =
                grenad_obkv_into_chunks(original_documents, pool_params, documents_chunk_size);

            // split obkv file into several chunks
            let flattened_chunk_iter =
                grenad_obkv_into_chunks(flattened_documents, pool_params, documents_chunk_size);

            let result = original_chunk_iter.and_then(|original_chunk| {
                let flattened_chunk = flattened_chunk_iter?;
                // extract all databases from the chunked obkv douments
                extract::data_from_obkv_documents(
                    original_chunk,
                    flattened_chunk,
                    pool_params,
                    lmdb_writer_sx.clone(),
                    searchable_fields,
                    faceted_fields,
                    primary_key_id,
                    geo_fields_ids,
                    vectors_field_id,
                    stop_words,
                    max_positions_per_attributes,
                    exact_attributes,
                )