Repository: HypothesisWorks/hypothesis

Issue:
from_lark fails for grammars with complex regexps
I tried to generate an example of Python code, by loading Lark's builtin Python3 grammar, but I think it failed to generate one of the regexps.

```python
import hypothesis.extra.lark

h = hypothesis.extra.lark.from_lark(Lark.open('lark/examples/python3.lark', start='file_input')) 

h.example()
```
Results in the exception:
```python
  File ".local/lib/python3.6/site-packages/hypothesis/searchstrategy/regex.py", line 185, in add_char
    if self._ignorecase and re.match(c, c.swapcase(), re.IGNORECASE) is not None:
  File "/usr/lib64/python3.6/re.py", line 172, in match
    return _compile(pattern, flags).match(string)
  File "/usr/lib64/python3.6/re.py", line 301, in _compile
    p = sre_compile.compile(pattern, flags)
  File "/usr/lib64/python3.6/sre_compile.py", line 562, in compile
    p = sre_parse.parse(p, flags)
  File "/usr/lib64/python3.6/sre_parse.py", line 855, in parse
    p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0)
  File "/usr/lib64/python3.6/sre_parse.py", line 416, in _parse_sub
    not nested and not items))
  File "/usr/lib64/python3.6/sre_parse.py", line 616, in _parse
    source.tell() - here + len(this))
sre_constants.error: nothing to repeat at position 0
nothing to repeat at position 0

```

Btw, this isn't high-priority. I  just found it while playing around, and thought you might want to know.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
--- a/hypothesis-python/src/hypothesis/extra/pytestplugin.py
+++ b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
@@ -186,7 +186,7 @@ def note_strategy_is_not_test(*args, **kwargs):
                 note_deprecation(
                     "%s is a function that returns a Hypothesis strategy, "
                     "but pytest has collected it as a test function.  This "
-                    "is useless as the function body will never be executed."
+                    "is useless as the function body will never be executed.  "
                     "To define a test function, use @given instead of "
                     "@composite." % (item.nodeid,),
                     since="2018-11-02",
diff --git a/hypothesis-python/src/hypothesis/searchstrategy/regex.py b/hypothesis-python/src/hypothesis/searchstrategy/regex.py
--- a/hypothesis-python/src/hypothesis/searchstrategy/regex.py
+++ b/hypothesis-python/src/hypothesis/searchstrategy/regex.py
@@ -182,7 +182,10 @@ def add_char(self, char):
         """Add given char to the whitelist."""
         c = self.code_to_char(char)
         self._whitelist_chars.add(c)
-        if self._ignorecase and re.match(c, c.swapcase(), re.IGNORECASE) is not None:
+        if (
+            self._ignorecase
+            and re.match(re.escape(c), c.swapcase(), flags=re.IGNORECASE) is not None
+        ):
             self._whitelist_chars.add(c.swapcase())
 
 
@@ -368,7 +371,8 @@ def recurse(codes):
             c = to_char(value)
             if (
                 context.flags & re.IGNORECASE
-                and re.match(c, c.swapcase(), re.IGNORECASE) is not None
+                and c != c.swapcase()
+                and re.match(re.escape(c), c.swapcase(), re.IGNORECASE) is not None
             ):
                 # We do the explicit check for swapped-case matching because
                 # eg 'ß'.upper() == 'SS' and ignorecase doesn't match it.
@@ -381,7 +385,7 @@ def recurse(codes):
             blacklist = set(c)
             if (
                 context.flags & re.IGNORECASE
-                and re.match(c, c.swapcase(), re.IGNORECASE) is not None
+                and re.match(re.escape(c), c.swapcase(), re.IGNORECASE) is not None
             ):
                 blacklist |= set(c.swapcase())
             if is_unicode:


Test patch:
diff --git a/hypothesis-python/tests/cover/test_regex.py b/hypothesis-python/tests/cover/test_regex.py
--- a/hypothesis-python/tests/cover/test_regex.py
+++ b/hypothesis-python/tests/cover/test_regex.py
@@ -463,3 +463,7 @@ def test_fullmatch_matches(pattern, eqiv_pattern):
 def test_fullmatch_must_be_bool():
     with pytest.raises(InvalidArgument):
         st.from_regex("a", fullmatch=None).validate()
+
+
+def test_issue_1786_regression():
+    st.from_regex(re.compile("\\\\", flags=re.IGNORECASE)).validate()
diff --git a/hypothesis-python/tests/nocover/test_regex.py b/hypothesis-python/tests/nocover/test_regex.py
--- a/hypothesis-python/tests/nocover/test_regex.py
+++ b/hypothesis-python/tests/nocover/test_regex.py
@@ -19,6 +19,7 @@
 
 import re
 import string
+from functools import reduce
 
 import hypothesis.strategies as st
 from hypothesis import assume, given, reject
@@ -43,6 +44,7 @@ def conservative_regex(draw):
     result = draw(
         st.one_of(
             st.just(u"."),
+            st.sampled_from([re.escape(c) for c in string.printable]),
             charset(),
             CONSERVATIVE_REGEX.map(lambda s: u"(%s)" % (s,)),
             CONSERVATIVE_REGEX.map(lambda s: s + u"+"),
@@ -59,12 +61,14 @@ def conservative_regex(draw):
 
 
 CONSERVATIVE_REGEX = conservative_regex()
+FLAGS = st.sets(st.sampled_from([getattr(re, "A", 0), re.I, re.M, re.S])).map(
+    lambda flag_set: reduce(int.__or__, flag_set, 0)
+)
 
 
 @given(st.data())
 def test_conservative_regex_are_correct_by_construction(data):
-    pattern = re.compile(data.draw(CONSERVATIVE_REGEX))
-    pattern = re.compile(pattern)
+    pattern = re.compile(data.draw(CONSERVATIVE_REGEX), flags=data.draw(FLAGS))
     result = data.draw(base_regex_strategy(pattern))
     assert pattern.search(result) is not None
 
@@ -76,9 +80,10 @@ def test_fuzz_stuff(data):
         | st.binary(min_size=1, max_size=5)
         | CONSERVATIVE_REGEX.filter(bool)
     )
+    flags = data.draw(FLAGS)
 
     try:
-        regex = re.compile(pattern)
+        regex = re.compile(pattern, flags=flags)
     except re.error:
         reject()