Repository: meilisearch/meilisearch
File: meilisearch-types/src/document_formats.rs (Rust)

Code before:

/// Reads JSON from temporary file  and write an obkv batch to writer.
pub fn read_json(file: &File, writer: impl Write + Seek) -> Result<usize> {
    read_json_inner(file, writer, PayloadType::Json)
}

/// Reads JSON from temporary file  and write an obkv batch to writer.
pub fn read_ndjson(file: &File, writer: impl Write + Seek) -> Result<usize> {
    read_json_inner(file, writer, PayloadType::Ndjson)
}

/// Reads JSON from temporary file  and write an obkv batch to writer.
fn read_json_inner(
    file: &File,
    writer: impl Write + Seek,
    payload_type: PayloadType,
) -> Result<usize> {
    let mut builder = DocumentsBatchBuilder::new(writer);
    let mmap = unsafe { MmapOptions::new().map(file).unwrap() };
    let mut deserializer = serde_json::Deserializer::from_slice(&mmap);

    match array_each(&mut deserializer, |obj: Object| builder.append_json_object(&obj)) {
        Ok(Ok(count)) => debug!("serde json array size: {}", count),
        Ok(Err(e)) => return Err(DocumentFormatError::Internal(Box::new(e))),
        Err(_e) => {
            debug!("deserialize single json");
            #[derive(Deserialize, Debug)]
            #[serde(transparent)]
            struct ArrayOrSingleObject {
                #[serde(with = "either::serde_untagged")]
                inner: Either<Vec<Object>, Object>,
            }

            let content: ArrayOrSingleObject = serde_json::from_reader(file)
                .map_err(Error::Json)
                .map_err(|e| (payload_type, e))?;

            for object in content.inner.map_right(|o| vec![o]).into_inner() {
                builder
                    .append_json_object(&object)
                    .map_err(Into::into)
                    .map_err(DocumentFormatError::Internal)?;
            }
        }
    }

    let count = builder.documents_count();
    let _ = builder.into_inner().map_err(Into::into).map_err(DocumentFormatError::Internal)?;

    Ok(count as usize)
}

Review comment (bug):
Would it be possible to replace this debug log by a little more explicit comment, here?

Code after:
}

/// Reads JSON from temporary file  and write an obkv batch to writer.
pub fn read_json(file: &File, writer: impl Write + Seek) -> Result<usize> {
    read_json_inner(file, writer, PayloadType::Json)
}

/// Reads JSON from temporary file  and write an obkv batch to writer.
pub fn read_ndjson(file: &File, writer: impl Write + Seek) -> Result<usize> {
    read_json_inner(file, writer, PayloadType::Ndjson)
}

/// Reads JSON from temporary file  and write an obkv batch to writer.
fn read_json_inner(
    file: &File,
    writer: impl Write + Seek,
    payload_type: PayloadType,
) -> Result<usize> {
    let mut builder = DocumentsBatchBuilder::new(writer);
    let mmap = unsafe { MmapOptions::new().map(file)? };
    let mut deserializer = serde_json::Deserializer::from_slice(&mmap);

    match array_each(&mut deserializer, |obj: Object| builder.append_json_object(&obj)) {
        // The json data has been successfully deserialised and does not need to be processed again.
        // the data has been successfully transferred to the "update_file" during the deserialisation process.
        // count ==0 means an empty array
        Ok(Ok(count)) => {
            if count == 0 {
                return Ok(count as usize);
            }
        }
        Ok(Err(e)) => return Err(DocumentFormatError::Internal(Box::new(e))),
        // Prefer deserialization as a json array. Failure to do deserialisation using the traditional method.
        Err(_e) => {
            #[derive(Deserialize, Debug)]
            #[serde(transparent)]
            struct ArrayOrSingleObject {
                #[serde(with = "either::serde_untagged")]
                inner: Either<Vec<Object>, Object>,
            }

            let content: ArrayOrSingleObject = serde_json::from_reader(file)
                .map_err(Error::Json)
                .map_err(|e| (payload_type, e))?;

            for object in content.inner.map_right(|o| vec![o]).into_inner() {
                builder
                    .append_json_object(&object)
                    .map_err(Into::into)
                    .map_err(DocumentFormatError::Internal)?;
            }