Repository: HypothesisWorks/hypothesis

Issue:
regex strategy ignoring re.VERBOSE
I have a regex expression that uses re.VERBOSE flag for improved readability, and when it is used as a strategy it throws a `FailedHealthCheck`, however the same regex without the flag and compressed on a single line works fine.

I suspect this is due to `from_regex` strategy ignoring the `re.VERBOSE flag` and trying to match the full regex including whitespace and newlines, a quick search in `searchstrategy/regex.py` shows that the VERBOSE flag is not checked anywhere.




Discussion:
Thanks for the report - your diagnosis is correct, this is definitely a bug, and we should fix it.  Notes on that:

- parse things with `sre_parse.parse(regex.pattern, flags=regex.flags)`
- `_strategy` should take a boolean `is_text`, not the pattern itself
- add some tests with `re.VERBOSE`, including patterns where this changes the match
- check whether there are other neglected flags

Resolving patch:
diff --git a/src/hypothesis/searchstrategy/regex.py b/src/hypothesis/searchstrategy/regex.py
--- a/src/hypothesis/searchstrategy/regex.py
+++ b/src/hypothesis/searchstrategy/regex.py
@@ -232,11 +232,11 @@ def maybe_pad(draw, regex, strategy, left_pad_strategy, right_pad_strategy):
 
 def base_regex_strategy(regex, parsed=None):
     if parsed is None:
-        parsed = sre_parse.parse(regex.pattern)
+        parsed = sre_parse.parse(regex.pattern, flags=regex.flags)
     return clear_cache_after_draw(_strategy(
         parsed,
         Context(flags=regex.flags),
-        regex.pattern
+        isinstance(regex.pattern, text_type)
     ))
 
 
@@ -246,7 +246,7 @@ def regex_strategy(regex):
 
     is_unicode = isinstance(regex.pattern, text_type)
 
-    parsed = sre_parse.parse(regex.pattern)
+    parsed = sre_parse.parse(regex.pattern, flags=regex.flags)
 
     if not parsed:
         if is_unicode:
@@ -294,7 +294,7 @@ def regex_strategy(regex):
     return maybe_pad(regex, base, left_pad, right_pad)
 
 
-def _strategy(codes, context, pattern):
+def _strategy(codes, context, is_unicode):
     """Convert SRE regex parse tree to strategy that generates strings matching
     that regex represented by that parse tree.
 
@@ -323,9 +323,9 @@ def _strategy(codes, context, pattern):
 
     """
     def recurse(codes):
-        return _strategy(codes, context, pattern)
+        return _strategy(codes, context, is_unicode)
 
-    if isinstance(pattern, text_type):
+    if is_unicode:
         empty = u''
         to_char = hunichr
     else:
@@ -387,7 +387,7 @@ def recurse(codes):
             if context.flags & re.IGNORECASE and \
                     re.match(c, c.swapcase(), re.IGNORECASE) is not None:
                 blacklist |= set(c.swapcase())
-            if isinstance(pattern, text_type):
+            if is_unicode:
                 return st.characters(blacklist_characters=blacklist)
             else:
                 return binary_char.filter(lambda c: c not in blacklist)
@@ -395,7 +395,7 @@ def recurse(codes):
         elif code == sre.IN:
             # Regex '[abc0-9]' (set of characters)
             negate = value[0][0] == sre.NEGATE
-            if isinstance(pattern, text_type):
+            if is_unicode:
                 builder = CharactersBuilder(negate, context.flags)
             else:
                 builder = BytesBuilder(negate, context.flags)
@@ -425,7 +425,7 @@ def recurse(codes):
 
         elif code == sre.ANY:
             # Regex '.' (any char)
-            if isinstance(pattern, text_type):
+            if is_unicode:
                 if context.flags & re.DOTALL:
                     return st.characters()
                 return st.characters(blacklist_characters=u'\n')
@@ -447,7 +447,7 @@ def recurse(codes):
                 # This feature is available only in specific Python versions
                 context.flags = (context.flags | value[1]) & ~value[2]
 
-            strat = _strategy(value[-1], context, pattern)
+            strat = _strategy(value[-1], context, is_unicode)
 
             context.flags = old_flags
 


Test patch:
diff --git a/tests/cover/test_regex.py b/tests/cover/test_regex.py
--- a/tests/cover/test_regex.py
+++ b/tests/cover/test_regex.py
@@ -375,3 +375,12 @@ def test_shared_union():
     # interesting feature of which is that it contains empty sub-expressions
     # in the branch.
     find_any(st.from_regex('.|.'))
+
+
+@given(st.data())
+def test_issue_992_regression(data):
+    strat = st.from_regex(re.compile(
+        r"""\d +  # the integral part
+            \.    # the decimal point
+            \d *  # some fractional digits""", re.VERBOSE))
+    data.draw(strat)