Repository: meilisearch/meilisearch
File: meilisearch-types/src/document_formats.rs (Rust)

Code before:
    Ok(count as usize)
}

/// Reads JSON from input and write an obkv batch to writer.
pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result<usize> {
    let mut builder = DocumentsBatchBuilder::new(writer);
    let reader = BufReader::new(input);

    #[derive(Deserialize, Debug)]
    #[serde(transparent)]
    struct ArrayOrSingleObject {
        #[serde(with = "either::serde_untagged")]
        inner: Either<Vec<Object>, Object>,
    }

    let content: ArrayOrSingleObject =
        serde_json::from_reader(reader).map_err(Error::Json).map_err(|e| (PayloadType::Json, e))?;

    for object in content.inner.map_right(|o| vec![o]).into_inner() {
        builder
            .append_json_object(&object)
            .map_err(Into::into)
            .map_err(DocumentFormatError::Internal)?;
    }

    let count = builder.documents_count();
    let _ = builder.into_inner().map_err(Into::into).map_err(DocumentFormatError::Internal)?;

    Ok(count as usize)
}


Review comment (suggestion):
We need to only deserialize the `Object` and not the `Vec<Object>` type.

Code after:
    file: &File,
    writer: impl Write + Seek,
    payload_type: PayloadType,
) -> Result<usize> {
    let mut builder = DocumentsBatchBuilder::new(writer);
    let mmap = unsafe { MmapOptions::new().map(file)? };
    let mut deserializer = serde_json::Deserializer::from_slice(&mmap);

    match array_each(&mut deserializer, |obj: Object| builder.append_json_object(&obj)) {
        // The json data has been successfully deserialised and does not need to be processed again.
        // the data has been successfully transferred to the "update_file" during the deserialisation process.
        // count ==0 means an empty array
        Ok(Ok(count)) => {
            if count == 0 {
                return Ok(count as usize);
            }
        }
        Ok(Err(e)) => return Err(DocumentFormatError::Internal(Box::new(e))),
        // Prefer deserialization as a json array. Failure to do deserialisation using the traditional method.
        Err(_e) => {
            #[derive(Deserialize, Debug)]
            #[serde(transparent)]
            struct ArrayOrSingleObject {
                #[serde(with = "either::serde_untagged")]
                inner: Either<Vec<Object>, Object>,
            }

            let content: ArrayOrSingleObject = serde_json::from_reader(file)
                .map_err(Error::Json)
                .map_err(|e| (payload_type, e))?;

            for object in content.inner.map_right(|o| vec![o]).into_inner() {
                builder
                    .append_json_object(&object)
                    .map_err(Into::into)
                    .map_err(DocumentFormatError::Internal)?;
            }
        }
    }

    let count = builder.documents_count();
    let _ = builder.into_inner().map_err(Into::into).map_err(DocumentFormatError::Internal)?;

    Ok(count as usize)
}

/**
 * The actual handling of the deserialization process in the serde avoids storing the deserialized object in memory.
 * Reference:
 * https://serde.rs/stream-array.html
 * https://github.com/serde-rs/json/issues/160