Repository: HypothesisWorks/hypothesis

Issue:
`st.from_regex(re.compile("[\u0130_]", re.IGNORECASE))` triggers internal error
`\u0130`, Latin Capital Letter I with Dot Above, lower-cases to `i\u0307`; and in turn if we have this in a regex character set we eventually trigger an internal error by passing it to `ord()` (which requires single-character strings).

I suspect that this is symptomatic of more general mishandling of length-changing case handling and will investigate.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py
@@ -2228,7 +2228,7 @@ def functions(
 ) -> SearchStrategy[Callable[..., Any]]:
     # The proper type signature of `functions()` would have T instead of Any, but mypy
     # disallows default args for generics: https://github.com/python/mypy/issues/3737
-    """functions(*, like=lambda: None, returns=none())
+    """functions(*, like=lambda: None, returns=none(), pure=False)
 
     A strategy for functions, which can be used in callbacks.
 
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/regex.py b/hypothesis-python/src/hypothesis/strategies/_internal/regex.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/regex.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/regex.py
@@ -114,21 +114,29 @@ def __init__(self, negate=False, flags=0):
     def strategy(self):
         """Returns resulting strategy that generates configured char set."""
         max_codepoint = None if self._unicode else 127
+        # Due to the .swapcase() issue described below (and in issue #2657),
+        # self._whitelist_chars may contain strings of len > 1.  We therefore
+        # have some extra logic to filter them out of st.characters() args,
+        # but still generate them if allowed to.
         if self._negate:
             black_chars = self._blacklist_chars - self._whitelist_chars
             return st.characters(
                 blacklist_categories=self._categories | {"Cc", "Cs"},
-                blacklist_characters=self._whitelist_chars,
+                blacklist_characters={c for c in self._whitelist_chars if len(c) == 1},
                 whitelist_characters=black_chars,
                 max_codepoint=max_codepoint,
             )
         white_chars = self._whitelist_chars - self._blacklist_chars
-        return st.characters(
+        multi_chars = {c for c in white_chars if len(c) > 1}
+        char_strategy = st.characters(
             whitelist_categories=self._categories,
             blacklist_characters=self._blacklist_chars,
-            whitelist_characters=white_chars,
+            whitelist_characters=white_chars - multi_chars,
             max_codepoint=max_codepoint,
         )
+        if multi_chars:
+            char_strategy |= st.sampled_from(sorted(multi_chars))
+        return char_strategy
 
     def add_category(self, category):
         """Update unicode state to match sre_parse object ``category``."""
@@ -163,6 +171,7 @@ def add_char(self, char):
             self._ignorecase
             and re.match(re.escape(c), c.swapcase(), flags=re.IGNORECASE) is not None
         ):
+            # Note that it is possible that `len(c.swapcase()) > 1`
             self._whitelist_chars.add(c.swapcase())
 
 
@@ -316,14 +325,8 @@ def recurse(codes):
                     j += 1
 
                 if i + 1 < j:
-                    strategies.append(
-                        st.just(
-                            empty.join(
-                                [to_char(charcode) for (_, charcode) in codes[i:j]]
-                            )
-                        )
-                    )
-
+                    chars = (to_char(charcode) for _, charcode in codes[i:j])
+                    strategies.append(st.just(empty.join(chars)))
                     i = j
                     continue
 
@@ -357,12 +360,28 @@ def recurse(codes):
         elif code == sre.NOT_LITERAL:
             # Regex '[^a]' (negation of a single char)
             c = to_char(value)
-            blacklist = set(c)
+            blacklist = {c}
             if (
                 context.flags & re.IGNORECASE
                 and re.match(re.escape(c), c.swapcase(), re.IGNORECASE) is not None
             ):
-                blacklist |= set(c.swapcase())
+                # There are a few cases where .swapcase() returns two characters,
+                # but is still a case-insensitive match.  In such cases we add *both*
+                # characters to our blacklist, to avoid doing the wrong thing for
+                # patterns such as r"[^\u0130]+" where "i\u0307" matches.
+                #
+                # (that's respectively 'Latin letter capital I with dot above' and
+                # 'latin latter i' + 'combining dot above'; see issue #2657)
+                #
+                # As a final additional wrinkle, "latin letter capital I" *also*
+                # case-insensitive-matches, with or without combining dot character.
+                # We therefore have to chain .swapcase() calls until a fixpoint.
+                stack = [c.swapcase()]
+                while stack:
+                    for char in stack.pop():
+                        blacklist.add(char)
+                        stack.extend(set(char.swapcase()) - blacklist)
+
             if is_unicode:
                 return st.characters(blacklist_characters=blacklist)
             else:


Test patch:
diff --git a/hypothesis-python/tests/cover/test_regex.py b/hypothesis-python/tests/cover/test_regex.py
--- a/hypothesis-python/tests/cover/test_regex.py
+++ b/hypothesis-python/tests/cover/test_regex.py
@@ -453,3 +453,12 @@ def test_fullmatch_must_be_bool():
 
 def test_issue_1786_regression():
     st.from_regex(re.compile("\\\\", flags=re.IGNORECASE)).validate()
+
+
+def test_sets_allow_multichar_output_in_ignorecase_mode():
+    # CharacterBuilder.strategy includes logic to add multi-character strings
+    # via sampled_from(), if any are whitelisted as matching.  See issue #2657.
+    find_any(
+        st.from_regex(re.compile("[\u0130_]", re.IGNORECASE)),
+        lambda s: len(s) > 1,
+    )
diff --git a/hypothesis-python/tests/nocover/test_regex.py b/hypothesis-python/tests/nocover/test_regex.py
--- a/hypothesis-python/tests/nocover/test_regex.py
+++ b/hypothesis-python/tests/nocover/test_regex.py
@@ -85,3 +85,29 @@ def test_fuzz_stuff(data):
 
     ex = data.draw(st.from_regex(regex))
     assert regex.search(ex)
+
+
+# Some preliminaries, to establish what's happening:
+I_WITH_DOT = "\u0130"
+assert I_WITH_DOT.swapcase() == "i\u0307"  # note: string of length two!
+assert re.compile(I_WITH_DOT, flags=re.IGNORECASE).match(I_WITH_DOT.swapcase())
+
+
+@given(st.data())
+def test_case_insensitive_not_literal_never_constructs_multichar_match(data):
+    # So our goal is to confirm that we can never accidentally create a non-matching
+    # string by assembling individually allowed characters.
+    pattern = re.compile(f"[^{I_WITH_DOT}]+", flags=re.IGNORECASE)
+    strategy = st.from_regex(pattern, fullmatch=True)
+    for _ in range(5):
+        s = data.draw(strategy)
+        assert pattern.fullmatch(s) is not None
+        # And to be on the safe side, we implment this stronger property:
+        assert set(s).isdisjoint(I_WITH_DOT.swapcase())
+
+
+@given(st.from_regex(re.compile(f"[^{I_WITH_DOT}_]", re.IGNORECASE), fullmatch=True))
+def test_no_error_converting_negated_sets_to_strategy(s):
+    # CharactersBuilder no longer triggers an internal error converting sets
+    # or negated sets to a strategy when multi-char strings are whitelisted.
+    pass