Repository: neogeny/TatSu

Issue:
Generated Python code doesn't cope with multi-line regexs
Consider:

```
myrule = /(?x) 
   foo
   bar
/ $ ;
```
This generates:

```
self._pattern(r'(?x) \n   foo\n   bar\n')
```
Which will look for literal newlines in the string, rather than being ignored.



Discussion:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 06fc6e4..94de2dd 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -14,6 +14,10 @@ The format of this *Change Log* is inspired by `keeapachangelog.org`_.
 ---------------
 .. _X.Y.Z: https://github.com/apalala/tatsu/compare/v4.2.6...master
 
+*   `#66`_ Fix multiline (`(?x)`) patterns not properly supported in grammar  (`@pdw-mb`_)
+
+.. _#66: https://github.com/neogeny/TatSu/issues/56
+
 `4.2.6`_ @ 2018-0506
 ---------------------
 .. _4.2.6: https://github.com/apalala/tatsu/compare/v4.2.5...v4.2.6
@@ -298,5 +302,6 @@ Added
 .. _gkimbar: https://bitbucket.org/gkimbar
 .. _nehz: https://bitbucket.org/nehz
 .. _neumond: https://bitbucket.org/neumond
+.. _pdw-mb: https://bitbucket.org/pdw-mb
 .. _pgebhard: https://bitbucket.org/pgebhard
 .. _siemer: https://bitbucket.org/siemer
diff --git a/docs/contributors.rst b/docs/contributors.rst
index cafe69a..5e38e83 100644
--- a/docs/contributors.rst
+++ b/docs/contributors.rst
@@ -40,6 +40,7 @@ features, but reports, bug fixes, or suggestions:
     `gkimbar`_,
     `nehz`_ ,
     `neumond`_,
+    `pdw-mb`_,
     `pgebhard`_,
     `siemer`_
 
@@ -83,5 +84,6 @@ Most of the contributions are recorded as **Grako** commits_ or issues_.
 .. _gkimbar: https://bitbucket.org/gkimbar
 .. _nehz: https://bitbucket.org/nehz
 .. _neumond: https://bitbucket.org/neumond
+.. _pdw-mb: https://bitbucket.org/pdw-mb
 .. _pgebhard: https://bitbucket.org/pgebhard
 .. _siemer: https://bitbucket.org/siemer
diff --git a/tatsu/codegen/python.py b/tatsu/codegen/python.py
index 71d606a..2b06af3 100644
--- a/tatsu/codegen/python.py
+++ b/tatsu/codegen/python.py
@@ -98,7 +98,7 @@ class Constant(Base):
 
 class Pattern(Base):
     def render_fields(self, fields):
-        raw_repr = 'r' + urepr(self.node.pattern).replace("\\\\", '\\')
+        raw_repr = urepr(self.node.pattern)
         fields.update(pattern=raw_repr)
 
     template = 'self._pattern({pattern})'


Test patch:
diff --git a/test/grammar/pattern_test.py b/test/grammar/pattern_test.py
index c2a214a..6dd7c0d 100644
--- a/test/grammar/pattern_test.py
+++ b/test/grammar/pattern_test.py
@@ -3,9 +3,10 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 
 import unittest
 
-from tatsu.util import trim
+from tatsu.util import trim, urepr
 from tatsu.tool import compile
 from tatsu.exceptions import FailedParse
+from tatsu.codegen import codegen
 
 
 class PatternTests(unittest.TestCase):
@@ -103,3 +104,30 @@ class PatternTests(unittest.TestCase):
         model = compile(grammar=grammar)
         ast = model.parse('ABcD xYZ')
         self.assertEqual(['ABcD', 'xYZ'], ast)
+
+    def test_multiline_pattern(self):
+        grammar = r'''
+            start =
+            /(?x)
+            foo
+            bar
+            / $ ;
+        '''
+        model = compile(grammar=trim(grammar))
+        print(codegen(model.rules[0].exp.sequence[0]))
+        self.assertEqual(
+            codegen(model.rules[0].exp.sequence[0]),
+            urepr("self._pattern('(?x)\nfoo\nbar\n')").strip('"\'')
+        )
+
+        grammar = r'''
+            start =
+            /(?x)foo\nbar
+            blort/ $ ;
+        '''
+        model = compile(grammar=trim(grammar))
+        print(codegen(model.rules[0].exp.sequence[0]))
+        self.assertEqual(
+            trim(codegen(model.rules[0].exp.sequence[0])),
+            urepr("self._pattern('(?x)foo\\nbar\nblort')").strip('"\.')
+        )