Repository: qdrant/qdrant
File: lib/api/src/rest/schema.rs (Rust)

Code before:
    /// Maximum token length to keep. If token is longer than this, it will be discarded.
    /// Default is `None`, which means no maximum length.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub max_token_len: Option<usize>,
}

impl Bm25Config {
    pub fn to_options(&self) -> HashMap<String, Value> {
        let Self {
            k,
            b,
            avg_len,
            tokenizer,
            text_preprocessing_config,
        } = self;

        let mut options = HashMap::new();
        options.insert("k".to_string(), Value::from(k.into_inner()));
        options.insert("b".to_string(), Value::from(b.into_inner()));
        options.insert("avg_len".to_string(), Value::from(avg_len.into_inner()));
        options.insert(
            "tokenizer".to_string(),
            serde_json::to_value(tokenizer)
                .expect("conversion of internal structure to JSON should never fail"),
        );
        options.insert(
            "text_preprocessing_config".to_string(),
            serde_json::to_value(text_preprocessing_config)
                .expect("conversion of internal structure to JSON should never fail"),
        );
        options
    }
}

/// Option variants for text documents.
/// Ether general-purpose options or BM25-specific options.
/// BM25-specific will only take effect if the `qdrant/bm25` is specified as a model.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
#[serde(untagged, rename_all = "snake_case")]
pub enum DocumentOptions {
    // This option should go first
    Common(HashMap<String, Value>),
    // This should never be deserialized into, but we keep it for schema generation
    Bm25(Bm25Config),
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]

Review comment (suggestion):
The `text_preprocessing_config` is flattened in the original config. I think we should flatten here as well, so the values get applied correctly.

Code after:
    /// Maximum token length to keep. If token is longer than this, it will be discarded.
    /// Default is `None`, which means no maximum length.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub max_token_len: Option<usize>,
}

impl Bm25Config {
    pub fn to_options(&self) -> HashMap<String, Value> {
        debug_assert!(
            false,
            "this code should never be called, it is only for schema generation",
        );

        let value = serde_json::to_value(self)
            .expect("conversion of internal structure to JSON should never fail");

        match value {
            Value::Null
            | Value::Bool(_)
            | Value::Number(_)
            | Value::String(_)
            | Value::Array(_) => HashMap::default(), // not expected
            Value::Object(map) => map.into_iter().collect(),
        }
    }
}

/// Option variants for text documents.
/// Ether general-purpose options or BM25-specific options.
/// BM25-specific will only take effect if the `qdrant/bm25` is specified as a model.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
#[serde(untagged, rename_all = "snake_case")]
pub enum DocumentOptions {
    // This option should go first
    Common(HashMap<String, Value>),
    // This should never be deserialized into, but we keep it for schema generation
    Bm25(Bm25Config),
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_document_options_should_deserialize_into_common() {
        let json = Bm25Config {
            tokenizer: TokenizerType::Word,
            ..Default::default()
        };
        let valid_bm25_config = serde_json::to_string(&json).unwrap();
        let options: DocumentOptions = serde_json::from_str(&valid_bm25_config).unwrap();