Repository: ClickHouse/ClickHouse
File: src/Interpreters/ITokenizer.h (C/C++)

Code before:
/// Parser extracting tokens which consist of alphanumeric ASCII characters or Unicode characters (not necessarily alphanumeric)
struct SplitByNonAlphaTokenizer final : public ITokenizerHelper<SplitByNonAlphaTokenizer>
{
    SplitByNonAlphaTokenizer() : ITokenizerHelper(Type::SplitByNonAlpha) {}

    static const char * getName() { return "tokenbf_v1"; }
    static const char * getExternalName() { return "splitByNonAlpha"; }
    String getDescription() const override { return "splitByNonAlpha"; }

    bool nextInString(const char * data, size_t length, size_t & __restrict pos, size_t & __restrict token_start, size_t & __restrict token_length) const override;
    bool nextInStringPadded(const char * data, size_t length, size_t & __restrict pos, size_t & __restrict token_start, size_t & __restrict token_length) const override;
    bool nextInStringLike(const char * data, size_t length, size_t & __restrict pos, String & token) const override;
    void substringToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter, bool is_prefix, bool is_suffix) const override;
    void substringToTokens(const char * data, size_t length, std::vector<String> & tokens, bool is_prefix, bool is_suffix) const override;

    bool supportsStringLike() const override { return true; }
};

/// Parser extracting tokens which are separated by certain strings.
/// Allows to emulate e.g. BigQuery's LOG_ANALYZER.
struct SplitByStringTokenizer final : public ITokenizerHelper<SplitByStringTokenizer>
{
    explicit SplitByStringTokenizer(const std::vector<String> & separators_) : ITokenizerHelper(Type::SplitByString), separators(separators_) {}

    static const char * getName() { return "splitByString"; }
    static const char * getExternalName() { return getName(); }
    String getDescription() const override;

    bool nextInString(const char * data, size_t length, size_t & pos, size_t & token_start, size_t & token_length) const override;
    bool nextInStringLike(const char * data, size_t length, size_t & pos, String & token) const override;

    bool supportsStringLike() const override { return false; }
private:
    std::vector<String> separators;
};

/// Parser doing "no operation". Returns the entire input as a single token.
struct ArrayTokenizer final : public ITokenizerHelper<ArrayTokenizer>
{
    ArrayTokenizer() : ITokenizerHelper(Type::Array) {}

    static const char * getName() { return "array"; }
    static const char * getExternalName() { return getName(); }
    String getDescription() const override { return getName(); }

    bool nextInString(const char * data, size_t length, size_t & pos, size_t & token_start, size_t & token_length) const override;
    bool nextInStringLike(const char * data, size_t length, size_t & pos, String & token) const override;

    bool supportsStringLike() const override { return false; }
};


Review comment (refactor):
We'll soon move to x86-v3: https://github.com/ClickHouse/ClickHouse/pull/90043/

I think we can simplify the ifdefs here a bit,. i.e. assume SSE4.2 & remove the SSE 2 path.

Code after:
    String getDescription() const override { return "splitByNonAlpha"; }

    bool nextInString(const char * data, size_t length, size_t & __restrict pos, size_t & __restrict token_start, size_t & __restrict token_length) const override;
    bool nextInStringLike(const char * data, size_t length, size_t & __restrict pos, String & token) const override;
    void substringToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter, bool is_prefix, bool is_suffix) const override;
    void substringToTokens(const char * data, size_t length, std::vector<String> & tokens, bool is_prefix, bool is_suffix) const override;

    bool supportsStringLike() const override { return true; }

    /// High-performance callback-based tokenizer with SSE optimization.
    /// Assumes data is padded from the right with at least 15 bytes (as our Columns provide).
    template <Fn<bool(const char *, size_t)> Callback>
    void forEachTokenImpl(const char * __restrict data, size_t length, Callback && callback) const
    {
        const char * begin = data;
        const char * end = data + length;
        const char * pos = data;

        while (pos < end)
        {
#if defined(__SSE2__) && !defined(MEMORY_SANITIZER) /// We read uninitialized bytes and decide on the calculated mask
            // NOTE: we assume that `data` string is padded from the right with 15 bytes.
            const __m128i haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
            const size_t haystack_length = 16;

#if defined(__SSE4_2__)
            // With the help of https://www.strchr.com/strcmp_and_strlen_using_sse_4.2
            const auto alnum_chars_ranges = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,
                    '\xFF', '\x80', 'z', 'a', 'Z', 'A', '9', '0');
            // Every bit represents if `haystack` character is in the ranges (1) or not (0)
            unsigned result_bitmask = _mm_cvtsi128_si32(_mm_cmpestrm(alnum_chars_ranges, 8, haystack, haystack_length, _SIDD_CMP_RANGES));
#else
            // NOTE: -1 and +1 required since SSE2 has no `>=` and `<=` instructions on packed 8-bit integers (epi8).
            const auto number_begin =      _mm_set1_epi8('0' - 1);
            const auto number_end =        _mm_set1_epi8('9' + 1);
            const auto alpha_lower_begin = _mm_set1_epi8('a' - 1);
            const auto alpha_lower_end =   _mm_set1_epi8('z' + 1);
            const auto alpha_upper_begin = _mm_set1_epi8('A' - 1);
            const auto alpha_upper_end =   _mm_set1_epi8('Z' + 1);
            const auto zero =              _mm_set1_epi8(0);

            // every bit represents if `haystack` character `c` satisfies condition:
            // (c < 0) || (c > '0' - 1 && c < '9' + 1) || (c > 'a' - 1 && c < 'z' + 1) || (c > 'A' - 1 && c < 'Z' + 1)
            // < 0 since _mm_cmplt_epi8 threats chars as SIGNED, and so all chars > 0x80 are negative.
            unsigned result_bitmask = _mm_movemask_epi8(_mm_or_si128(_mm_or_si128(_mm_or_si128(
                    _mm_cmplt_epi8(haystack, zero),
                    _mm_and_si128(_mm_cmpgt_epi8(haystack, number_begin),      _mm_cmplt_epi8(haystack, number_end))),
                    _mm_and_si128(_mm_cmpgt_epi8(haystack, alpha_lower_begin), _mm_cmplt_epi8(haystack, alpha_lower_end))),
                    _mm_and_si128(_mm_cmpgt_epi8(haystack, alpha_upper_begin), _mm_cmplt_epi8(haystack, alpha_upper_end))));
#endif
            const char * next_pos = std::min(end, pos + haystack_length);