Repository: qdrant/qdrant
File: lib/api/src/rest/schema.rs (Rust)

Code before:
    pub language: Option<String>,
    /// Lowercase the text before tokenization.
    /// Default is `true`.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub lowercase: Option<bool>,
    /// Configuration of the stopwords filter. Supports list of pre-defined languages and custom stopwords.
    /// Default: initialized for specified `language` or English if not specified.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub stopwords: Option<StopwordsInterface>,
    /// Configuration of the stemmer. Processes tokens to their root form.
    /// Default: initialized Snowball stemmer for specified `language` or English if not specified.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub stemmer: Option<StemmingAlgorithm>,
    /// Minimum token length to keep. If token is shorter than this, it will be discarded.
    /// Default is `None`, which means no minimum length.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub min_token_len: Option<usize>,
    /// Maximum token length to keep. If token is longer than this, it will be discarded.
    /// Default is `None`, which means no maximum length.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub max_token_len: Option<usize>,
}

impl Bm25Config {
    pub fn to_options(&self) -> HashMap<String, Value> {
        let Self {
            k,
            b,
            avg_len,
            tokenizer,
            text_preprocessing_config,
        } = self;

        let mut options = HashMap::new();
        options.insert("k".to_string(), Value::from(k.into_inner()));
        options.insert("b".to_string(), Value::from(b.into_inner()));
        options.insert("avg_len".to_string(), Value::from(avg_len.into_inner()));
        options.insert(
            "tokenizer".to_string(),
            serde_json::to_value(tokenizer)
                .expect("conversion of internal structure to JSON should never fail"),
        );
        options.insert(
            "text_preprocessing_config".to_string(),
            serde_json::to_value(text_preprocessing_config)
                .expect("conversion of internal structure to JSON should never fail"),
        );
        options
    }
}


Review comment (suggestion):
I think this approach might not be the best. Manually serializing the fields into a HashMap makes it very easy to forget new fields or changed attributes in the original struct.

Also if I'm seeing it correctly, this function is never called because we always prefer `DocumentOptions::Common` over `DocumentOptions::Bm25` and this function is only called for `DocumentOptions::Bm25`

Code after:
    pub language: Option<String>,
    /// Lowercase the text before tokenization.
    /// Default is `true`.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub lowercase: Option<bool>,
    /// Configuration of the stopwords filter. Supports list of pre-defined languages and custom stopwords.
    /// Default: initialized for specified `language` or English if not specified.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub stopwords: Option<StopwordsInterface>,
    /// Configuration of the stemmer. Processes tokens to their root form.
    /// Default: initialized Snowball stemmer for specified `language` or English if not specified.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub stemmer: Option<StemmingAlgorithm>,
    /// Minimum token length to keep. If token is shorter than this, it will be discarded.
    /// Default is `None`, which means no minimum length.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub min_token_len: Option<usize>,
    /// Maximum token length to keep. If token is longer than this, it will be discarded.
    /// Default is `None`, which means no maximum length.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub max_token_len: Option<usize>,
}

impl Bm25Config {
    pub fn to_options(&self) -> HashMap<String, Value> {
        debug_assert!(
            false,
            "this code should never be called, it is only for schema generation",
        );

        let value = serde_json::to_value(self)
            .expect("conversion of internal structure to JSON should never fail");

        match value {
            Value::Null
            | Value::Bool(_)
            | Value::Number(_)
            | Value::String(_)
            | Value::Array(_) => HashMap::default(), // not expected
            Value::Object(map) => map.into_iter().collect(),
        }
    }
}

/// Option variants for text documents.
/// Ether general-purpose options or BM25-specific options.
/// BM25-specific will only take effect if the `qdrant/bm25` is specified as a model.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
#[serde(untagged, rename_all = "snake_case")]
pub enum DocumentOptions {
    // This option should go first