Repository: dotnet/runtime
File: src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs (C#)

Code before:

                default:
                    Debug.Assert(FindMode == FindNextStartingPositionMode.NoSearch, $"Unexpected FindMode {FindMode}");
                    return true;
            }
        }
#endif

        /// <summary>
        /// Determines whether the starting characters of the prefixes are frequent enough in typical text
        /// that IndexOfAny with those characters would be a poor filter. When starting characters are common,
        /// multi-string search via SearchValues is preferred because IndexOfAny would match too many
        /// positions. When starting characters are rare, IndexOfAny is an excellent filter and is preferred.
        /// </summary>
        private static bool HasHighFrequencyStartingChars(string[] prefixes)
        {
            ReadOnlySpan<float> frequency = RegexPrefixAnalyzer.Frequency;
            float totalFrequency = 0;
            int count = 0;

            // Use two longs as a 128-bit bitset to track seen ASCII chars.
            long seenLo = 0, seenHi = 0;
            foreach (string prefix in prefixes)
            {
                char c = prefix[0];
                if (c >= 128)
                {
                    // Non-ASCII starting chars have no frequency data; bail out and let
                    // FixedDistanceSets choose a strategy, since it can examine all offsets.
                    return false;
                }

                // Skip duplicate starting chars.
                ref long seen = ref (c < 64 ? ref seenLo : ref seenHi);
                long mask = 1L << (c & 63);
                if ((seen & mask) != 0)
                {
                    continue;
                }
                seen |= mask;

                totalFrequency += frequency[c];
                count++;
            }

            // If the average frequency of starting chars exceeds this threshold, the characters
            // are common enough that IndexOfAny would match many positions, making SearchValues<string>
            // the better choice. Common characters like lowercase letters have frequencies above this
            // threshold, while less frequent characters like uppercase letters fall below it, so
            // IndexOfAny with rare starting chars is already an effective filter and is preferred.
            const float HighFrequencyThreshold = 0.6f;

Review comment (suggestion):
If we're going to use a fixed length like this, we should at least assert that Frequency.Length is 128, such that if we ever changed the table, we'd know to update this as well.

Code after:

                // Nothing special to look for.  Just return true indicating this is a valid position to try to match.

                default:
                    Debug.Assert(FindMode == FindNextStartingPositionMode.NoSearch, $"Unexpected FindMode {FindMode}");
                    return true;
            }
        }
#endif

        /// <summary>
        /// Determines whether the characters in the best <see cref="FixedDistanceSet"/> are frequent enough in typical text
        /// that IndexOfAny with those characters would be a poor filter. When the best set's characters are common,
        /// multi-string search via SearchValues is preferred because IndexOfAny would match too many
        /// positions. When the characters are rare, IndexOfAny is an excellent filter and is preferred.
        /// </summary>
        private static bool HasHighFrequencyChars(FixedDistanceSet set)
        {
            // Negated sets match most characters (e.g. [^a]), so they are inherently high frequency.
            if (set.Negated)
            {
                return true;
            }

            // Sets without extracted chars can't be frequency-analyzed.
            // Single-char sets use IndexOf which has much higher throughput than multi-string search,
            // so we never consider them "high frequency" regardless of which character it is.
            if (set.Chars is not { Length: > 1 } chars)
            {
                return false;
            }

            ReadOnlySpan<float> frequency = RegexPrefixAnalyzer.Frequency;
            float totalFrequency = 0;

            foreach (char c in chars)
            {
                if (c >= frequency.Length)
                {
                    return false;
                }

                totalFrequency += frequency[c];
            }

            // If the average frequency of the set's chars exceeds this threshold, the characters
            // are common enough that IndexOfAny would match many positions, making SearchValues<string>
            // the better choice. Common characters like lowercase letters have frequencies above this
            // threshold, while less frequent characters like uppercase letters fall below it, so
            // IndexOfAny with rare characters is already an effective filter and is preferred.
            const float HighFrequencyThreshold = 0.6f;