Repository: meilisearch/meilisearch
File: crates/meilitool/src/main.rs (Rust)

Code before:
        .with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?;

    let rtxn = env.read_txn()?;
    let index_mapping: Database<Str, UuidCodec> =
        try_opening_database(&env, &rtxn, "index-mapping")?;

    for result in index_mapping.iter(&rtxn)? {
        let (uid, uuid) = result?;
        if uid == index_name {
            let index_path = db_path.join("indexes").join(uuid.to_string());
            let index =
                Index::new(EnvOpenOptions::new(), &index_path, false).with_context(|| {
                    format!("While trying to open the index at path {:?}", index_path.display())
                })?;

            let rtxn = index.read_txn()?;
            let fields_ids_map = index.fields_ids_map(&rtxn)?;
            let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
            let embedding_configs = index.embedding_configs(&rtxn)?;

            if let Some(offset) = offset {
                eprintln!("Skipping {offset} documents");
            }

            let mut stdout = BufWriter::new(std::io::stdout());
            for (i, ret) in index.all_documents(&rtxn)?.skip(offset.unwrap_or(0)).enumerate() {
                let (id, doc) = ret?;
                let mut document = obkv_to_json(&all_fields, &fields_ids_map, doc)?;

                if i % 10_000 == 0 {
                    eprintln!("Starting the {}th document", i + offset.unwrap_or(0));
                }

                if !ignore_vectors {
                    'inject_vectors: {
                        let embeddings = index.embeddings(&rtxn, id)?;

                        if embeddings.is_empty() {
                            break 'inject_vectors;
                        }

                        let vectors = document
                            .entry(RESERVED_VECTORS_FIELD_NAME)
                            .or_insert(Object(Default::default()));

                        let Object(vectors) = vectors else {
                            return Err(meilisearch_types::milli::Error::UserError(
                                meilisearch_types::milli::UserError::InvalidVectorsMapType {
                                    document_id: {
                                        if let Ok(Some(Ok(index))) = index
                                            .external_id_of(&rtxn, std::iter::once(id))

Review comment (performance):
I fear this way of skipping documents from `all_documents` must be much slower to the one I proposed:

```
           let all_documents = index.documents_ids(&rtxn)?;
            if let Some(offset) = offset {
                eprintln!("Skipping {offset} documents");
            }
            for (document_index, ret) in index.iter_documents(&rtxn, all_documents.into_iter().skip(offset.unwrap_or_default()))?.enumerate() {
```

this avoids spawning each document for real. Care must be taken to add the `document_index` to `offset` though for displaying the correct progress, though

Code after:
        .with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?;

    let rtxn = env.read_txn()?;
    let index_mapping: Database<Str, UuidCodec> =
        try_opening_database(&env, &rtxn, "index-mapping")?;

    for result in index_mapping.iter(&rtxn)? {
        let (uid, uuid) = result?;
        if uid == index_name {
            let index_path = db_path.join("indexes").join(uuid.to_string());
            let index =
                Index::new(EnvOpenOptions::new(), &index_path, false).with_context(|| {
                    format!("While trying to open the index at path {:?}", index_path.display())
                })?;

            let rtxn = index.read_txn()?;
            let fields_ids_map = index.fields_ids_map(&rtxn)?;
            let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
            let embedding_configs = index.embedding_configs(&rtxn)?;

            if let Some(offset) = offset {
                eprintln!("Skipping {offset} documents");
            }

            let mut stdout = BufWriter::new(std::io::stdout());
            let all_documents = index.documents_ids(&rtxn)?.into_iter().skip(offset.unwrap_or(0));
            for (i, ret) in index.iter_documents(&rtxn, all_documents)?.enumerate() {
                let (id, doc) = ret?;
                let mut document = obkv_to_json(&all_fields, &fields_ids_map, doc)?;

                if i % 10_000 == 0 {
                    eprintln!("Starting the {}th document", i + offset.unwrap_or(0));
                }

                if !ignore_vectors {
                    'inject_vectors: {
                        let embeddings = index.embeddings(&rtxn, id)?;

                        if embeddings.is_empty() {
                            break 'inject_vectors;
                        }

                        let vectors = document
                            .entry(RESERVED_VECTORS_FIELD_NAME)
                            .or_insert(Object(Default::default()));

                        let Object(vectors) = vectors else {
                            return Err(meilisearch_types::milli::Error::UserError(
                                meilisearch_types::milli::UserError::InvalidVectorsMapType {
                                    document_id: {
                                        if let Ok(Some(Ok(index))) = index