Repository: ClickHouse/ClickHouse
File: src/Storages/MergeTree/MergeTreeIndexConditionText.cpp (C++)

Code before:
        || function_name == "mapContainsValueLike"
        || function_name == "has"
        || function_name == "like"
        || function_name == "notLike"
        || function_name == "hasTokenOrNull"
        || function_name == "startsWith"
        || function_name == "endsWith"
        || function_name == "match";
}

TextIndexDirectReadMode MergeTreeIndexConditionText::getHintOrNoneMode() const
{
    const auto & settings = getContext()->getSettingsRef();
    return settings[Setting::query_plan_text_index_add_hint] ? TextIndexDirectReadMode::Hint : TextIndexDirectReadMode::None;
}

TextIndexDirectReadMode MergeTreeIndexConditionText::getDirectReadMode(const String & function_name) const
{
    if (function_name == "hasAnyTokens" || function_name == "hasAllTokens")
    {
        return TextIndexDirectReadMode::Exact;
    }

    if (function_name == "hasToken")
    {
        /// Function "hasToken" assumes the splitByNonAlpha tokenizer while executing brute force search.
        /// Therefore, we can use exact direct read only if tokenizer is splitByNonAlpha and there is no preprocessor.
        bool is_default_tokenizer = typeid_cast<const SplitByNonAlphaTokenExtractor *>(token_extractor);
        bool has_preprocessor = preprocessor && preprocessor->hasActions();
        return is_default_tokenizer && !has_preprocessor ? TextIndexDirectReadMode::Exact : getHintOrNoneMode();
    }

    if (function_name == "equals"
        || function_name == "has"
        || function_name == "mapContainsKey"
        || function_name == "mapContainsValue")
    {
        /// These functions compare the searched token as a whole and therefore
        /// exact direct read is only possible with array token extractor, that doesn't
        /// split documents into tokens. Otherwise we can only use direct read as a hint.
        bool is_array_extractor = typeid_cast<const ArrayTokenExtractor *>(token_extractor);
        bool has_preprocessor = preprocessor && preprocessor->hasActions();
        return is_array_extractor && !has_preprocessor ? TextIndexDirectReadMode::Exact : getHintOrNoneMode();
    }


    if (function_name == "like"
        || function_name == "startsWith"
        || function_name == "endsWith"
        || function_name == "mapContainsKeyLike"
        || function_name == "mapContainsValueLike")

Review comment (suggestion):
I pushed https://github.com/ClickHouse/ClickHouse/pull/93618 which makes more clear that `hasToken` uses the `splitByNonAlpha` tokenizer.

Code after:
        || function_name == "mapContainsValueLike"
        || function_name == "has"
        || function_name == "like"
        || function_name == "notLike"
        || function_name == "hasTokenOrNull"
        || function_name == "startsWith"
        || function_name == "endsWith"
        || function_name == "match";
}

TextIndexDirectReadMode MergeTreeIndexConditionText::getHintOrNoneMode() const
{
    const auto & settings = getContext()->getSettingsRef();
    return settings[Setting::query_plan_text_index_add_hint] ? TextIndexDirectReadMode::Hint : TextIndexDirectReadMode::None;
}

TextIndexDirectReadMode MergeTreeIndexConditionText::getDirectReadMode(const String & function_name) const
{
    if (function_name == "hasToken"
        || function_name == "hasAnyTokens"
        || function_name == "hasAllTokens")
    {
        return TextIndexDirectReadMode::Exact;
    }

    if (function_name == "equals"
        || function_name == "has"
        || function_name == "mapContainsKey"
        || function_name == "mapContainsValue")
    {
        /// These functions compare the searched token as a whole and therefore
        /// exact direct read is only possible with array token extractor, that doesn't
        /// split documents into tokens. Otherwise we can only use direct read as a hint.
        bool is_array_extractor = typeid_cast<const ArrayTokenExtractor *>(token_extractor);
        bool has_preprocessor = preprocessor && preprocessor->hasActions();
        return is_array_extractor && !has_preprocessor ? TextIndexDirectReadMode::Exact : getHintOrNoneMode();
    }


    if (function_name == "like"
        || function_name == "startsWith"
        || function_name == "endsWith"
        || function_name == "mapContainsKeyLike"
        || function_name == "mapContainsValueLike")
    {
        return getHintOrNoneMode();
    }

    return TextIndexDirectReadMode::None;
}