Repository: neogeny/TatSu

Issue:
ModelBuilderSemantics Subclassing
```ebnf
start = expression $;

expression =
    | addition
    | subtraction
    | number;

addition::BinaryOp::Op = left:expression op:'+' ~ right:number;
subtraction::BinaryOp::Op = left:expression op:'-' ~ right:number;

number = /\d+/;
```
This triggers an exception: https://pastebin.com/raw/qMUUXEKb
It works when you remove `left:expression` in addition and subtraction

I'm not sure how this feature is supposed to work in the first place, how does it decide what to put on the base class and what on the inherited one? I suppose the workaround would be to create the base class manually and pass it to the `ModelBuilderSemantics` instance.

In either case, this should be documented better.

Discussion:

Patch:
diff --git a/docs/mini-tutorial.rst b/docs/mini-tutorial.rst
index b32c62c..c73380c 100644
--- a/docs/mini-tutorial.rst
+++ b/docs/mini-tutorial.rst
@@ -305,7 +305,7 @@ Semantics for |TatSu| parsers are not specified in the grammar, but in a separat
         parser = tatsu.compile(grammar)
         ast = parser.parse(
             '3 + 5 * ( 10 - 20 )',
-            parse_with_basic_semantics=CalcBasicSemantics()
+            semantics=CalcBasicSemantics()
         )
 
         print('# BASIC SEMANTICS RESULT')
diff --git a/grammar/tatsu.ebnf b/grammar/tatsu.ebnf
index d087e40..3ea4896 100644
--- a/grammar/tatsu.ebnf
+++ b/grammar/tatsu.ebnf
@@ -378,7 +378,7 @@ token::Token
 
 literal
     =
-    string | raw_string | word | hex | float | int
+    string | raw_string | boolean | word | hex | float | int
     ;
 
 
diff --git a/tatsu/bootstrap.py b/tatsu/bootstrap.py
index 50ecd1f..5fcba66 100644
--- a/tatsu/bootstrap.py
+++ b/tatsu/bootstrap.py
@@ -152,7 +152,7 @@ class EBNFBootstrapParser(Parser):
                                 self._boolean_()
                                 self.name_last_node('value')
                             with self._option():
-                                self._constant('True')
+                                self._constant(True)
                                 self.name_last_node('value')
                             self._error('no available options')
                 with self._option():
@@ -728,7 +728,7 @@ class EBNFBootstrapParser(Parser):
     def _special_(self):  # noqa
         self._token('?(')
         self._cut()
-        self._pattern(r'.*?(?!\)\?)')
+        self._pattern('.*?(?!\\)\\?)')
         self.name_last_node('@')
         self._token(')?')
         self._cut()
@@ -803,11 +803,11 @@ class EBNFBootstrapParser(Parser):
 
     @tatsumasu('Constant')
     def _constant_(self):  # noqa
-        self._pattern(r'`')
+        self._pattern('`')
         self._cut()
         self._literal_()
         self.name_last_node('@')
-        self._pattern(r'`')
+        self._pattern('`')
 
     @tatsumasu('Token')
     def _token_(self):  # noqa
@@ -825,6 +825,8 @@ class EBNFBootstrapParser(Parser):
                 self._string_()
             with self._option():
                 self._raw_string_()
+            with self._option():
+                self._boolean_()
             with self._option():
                 self._word_()
             with self._option():
@@ -851,14 +853,14 @@ class EBNFBootstrapParser(Parser):
             with self._option():
                 self._token('"')
                 self._cut()
-                self._pattern(r'([^"\n]|\\"|\\\\)*')
+                self._pattern('([^"\\n]|\\\\"|\\\\\\\\)*')
                 self.name_last_node('@')
                 self._token('"')
                 self._cut()
             with self._option():
                 self._token("'")
                 self._cut()
-                self._pattern(r"([^'\n]|\\'|\\\\)*")
+                self._pattern("([^'\\n]|\\\\'|\\\\\\\\)*")
                 self.name_last_node('@')
                 self._token("'")
                 self._cut()
@@ -866,23 +868,23 @@ class EBNFBootstrapParser(Parser):
 
     @tatsumasu()
     def _hex_(self):  # noqa
-        self._pattern(r'0[xX](\d|[a-fA-F])+')
+        self._pattern('0[xX](\\d|[a-fA-F])+')
 
     @tatsumasu()
     def _float_(self):  # noqa
-        self._pattern(r'[-+]?(?:\d+\.\d*|\d*\.\d+)(?:[Ee][-+]?\d+)?')
+        self._pattern('[-+]?(?:\\d+\\.\\d*|\\d*\\.\\d+)(?:[Ee][-+]?\\d+)?')
 
     @tatsumasu()
     def _int_(self):  # noqa
-        self._pattern(r'[-+]?\d+')
+        self._pattern('[-+]?\\d+')
 
     @tatsumasu()
     def _path_(self):  # noqa
-        self._pattern(r'(?!\d)\w+(::(?!\d)\w+)+')
+        self._pattern('(?!\\d)\\w+(::(?!\\d)\\w+)+')
 
     @tatsumasu()
     def _word_(self):  # noqa
-        self._pattern(r'(?!\d)\w+')
+        self._pattern('(?!\\d)\\w+')
 
     @tatsumasu('Any')
     def _any_(self):  # noqa
@@ -908,16 +910,16 @@ class EBNFBootstrapParser(Parser):
             with self._option():
                 self._token('/')
                 self._cut()
-                self._pattern(r'([^/\\]|\\/|\\.)+')
+                self._pattern('([^/\\\\]|\\\\/|\\\\.)+')
                 self.name_last_node('@')
                 self._token('/')
                 self._cut()
             with self._option():
                 self._token('?/')
                 self._cut()
-                self._pattern(r'(.|\n)+?(?=/\?)')
+                self._pattern('(.|\\n)+?(?=/\\?)')
                 self.name_last_node('@')
-                self._pattern(r'/\?+')
+                self._pattern('/\\?+')
                 self._cut()
             with self._option():
                 self._token('?')
@@ -1140,14 +1142,16 @@ class EBNFBootstrapSemantics(object):
         return ast
 
 
-def main(filename, start='start', **kwargs):
+def main(filename, start=None, **kwargs):
+    if start is None:
+        start = 'start'
     if not filename or filename == '-':
         text = sys.stdin.read()
     else:
         with open(filename) as f:
             text = f.read()
     parser = EBNFBootstrapParser()
-    return parser.parse(text, start=start, filename=filename, **kwargs)
+    return parser.parse(text, rule_name=start, filename=filename, **kwargs)
 
 
 if __name__ == '__main__':
diff --git a/tatsu/codegen/cgbase.py b/tatsu/codegen/cgbase.py
index c31972e..b858636 100644
--- a/tatsu/codegen/cgbase.py
+++ b/tatsu/codegen/cgbase.py
@@ -90,6 +90,7 @@ class CodeGenerator(object):
     ``ModelRenderer`` class with the same name as each model's node and
     uses it to render the node.
     """
+
     def __init__(self, modules=None):
         self.formatter = DelegatingRenderingFormatter(self)
         self._renderers = {}
diff --git a/tatsu/codegen/objectmodel.py b/tatsu/codegen/objectmodel.py
index d081332..e1f87e7 100644
--- a/tatsu/codegen/objectmodel.py
+++ b/tatsu/codegen/objectmodel.py
@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import inspect
+
 from datetime import datetime
 from collections import namedtuple
 
@@ -15,16 +17,22 @@ from tatsu.objectmodel import BASE_CLASS_TOKEN
 from tatsu.exceptions import CodegenError
 from tatsu.rendering import Renderer
 from tatsu.codegen.cgbase import ModelRenderer, CodeGenerator
+from tatsu.util import PY33
 
 
-NODE_NAME_PATTERN = '(?!\d)\w+(' + BASE_CLASS_TOKEN + '(?!\d)\w+)*'
+NODE_NAME_PATTERN = r'(?!\d)\w+(' + BASE_CLASS_TOKEN + r'(?!\d)\w+)*'
 
 
 TypeSpec = namedtuple('TypeSpec', ['class_name', 'base'])
 
+DEFAULT_BASE_TYPE = '''
+class ModelBase(Node):
+    pass
+'''
+
 
-def codegen(model):
-    return ObjectModelCodeGenerator().render(model)
+def codegen(model, base_type=None):
+    return ObjectModelCodeGenerator().render(model, base_type=base_type)
 
 
 def _get_node_class_name(rule):
@@ -39,28 +47,79 @@ def _get_node_class_name(rule):
     return typespec
 
 
-def _typespec(rule, default_base=True):
+def _typespec(rule):
     if not _get_node_class_name(rule):
-        return TypeSpec(None, None)
+        return []
 
     spec = rule.params[0].split(BASE_CLASS_TOKEN)
-    class_name = safe_name(spec[0])
-    base = None
-    bases = spec[1:]
-    if bases:
-        base = safe_name(bases[0])
-    elif default_base:
-        base = 'ModelBase'
-    return TypeSpec(class_name, base)
+    class_names = [safe_name(n) for n in spec] + ['ModelBase']
+
+    typespec = []
+    for i, class_name in enumerate(class_names[:-1]):
+        base = class_names[i + 1]
+        typespec.append(TypeSpec(class_name, base))
+
+    return typespec
+
+
+def _get_full_name(cls):
+    if not inspect.isclass(cls):
+        raise CodegenError("Base type has to be a class")
+    module = inspect.getmodule(cls)
+    if not module:
+        raise CodegenError("Base type has to be inside a module")
+    modulename = module.__name__
+
+    if PY33:
+        name = cls.__qualname__
+    else:
+        name = cls.__name__
+
+    # Try to reference the class
+    try:
+        idents = name.split('.')
+        _cls = getattr(module, idents[0])
+        for ident in idents[1:]:
+            _cls = getattr(_cls, ident)
+
+        assert _cls == cls
+    except AttributeError:
+        raise CodegenError("Couldn't find base type, it has to be importable")
+
+    return modulename, name
+
+
+class BaseTypeRenderer(Renderer):
+    def __init__(self, base_type):
+        self.base_type = base_type
+
+    def render_fields(self, fields):
+        module, name = _get_full_name(self.base_type)
+        if '.' in name:
+            lookup = "\nModelBase = %s" % name
+            name = name.split('.')[0]
+        else:
+            lookup = " as ModelBase"
+
+        fields.update(
+            module=module,
+            name=name,
+            lookup=lookup
+        )
+
+    template = '''
+        from {module} import {name}{lookup}\
+        '''
 
 
 class BaseClassRenderer(Renderer):
-    def __init__(self, class_name):
-        self.class_name = class_name
+    def __init__(self, spec):
+        self.class_name = spec.class_name
+        self.base = spec.base
 
     template = '''
-        class {class_name}(ModelBase):
-            pass
+        class {class_name}({base}):
+            pass\
         '''
 
 
@@ -80,6 +139,7 @@ class Rule(ModelRenderer):
     def render_fields(self, fields):
         defs = [safe_name(d) for d, l in compress_seq(self.defines())]
         defs = list(sorted(set(defs)))
+        spec = fields["spec"]
 
         kwargs = '\n'.join('%s = None' % d for d in defs)
         if kwargs:
@@ -87,8 +147,6 @@ class Rule(ModelRenderer):
         else:
             kwargs = indent('pass')
 
-        spec = _typespec(self.node)
-
         fields.update(
             class_name=spec.class_name,
             base=spec.base,
@@ -108,24 +166,29 @@ class Grammar(ModelRenderer):
         bases = []
         model_rules = []
         for rule in self.node.rules:
-            spec = _typespec(rule, False)
-            if not spec.class_name:
+            specs = _typespec(rule)
+            if not specs:
                 continue
-            if spec.class_name not in node_class_names:
-                model_rules.append(rule)
-            if spec.base and spec.base not in node_class_names:
-                bases.append(spec.base)
-            node_class_names.add(spec.class_name)
-            node_class_names.add(spec.base)
+
+            node_spec = specs[0]
+            base_spec = reversed(specs[1:])
+
+            if node_spec.class_name not in node_class_names:
+                model_rules.append((rule, node_spec))
+
+            bases.extend(base for base in base_spec
+                         if base.class_name not in node_class_names)
+
+            node_class_names.update(s.class_name for s in specs)
 
         base_class_declarations = [
-            BaseClassRenderer(base).render()
-            for base in bases
+            BaseClassRenderer(spec).render()
+            for spec in bases
         ]
 
         model_class_declarations = [
-            self.get_renderer(rule).render()
-            for rule in model_rules
+            self.get_renderer(rule).render(spec=spec)
+            for rule, spec in model_rules
         ]
 
         base_class_declarations = '\n\n\n'.join(base_class_declarations)
@@ -135,10 +198,13 @@ class Grammar(ModelRenderer):
 
         version = datetime.now().strftime('%Y.%m.%d.%H')
 
+        base_type = fields["base_type"]
+
         fields.update(
             base_class_declarations=base_class_declarations,
             model_class_declarations=model_class_declarations,
             version=version,
+            base_type=BaseTypeRenderer(base_type).render() if base_type else DEFAULT_BASE_TYPE
         )
 
     template = '''\
@@ -159,19 +225,16 @@ class Grammar(ModelRenderer):
                 from tatsu.objectmodel import Node
                 from tatsu.semantics import ModelBuilderSemantics
 
+                {base_type}
 
                 class {name}ModelBuilderSemantics(ModelBuilderSemantics):
-                    def __init__(self):
+                    def __init__(self, context=None, types=None):
                         types = [
                             t for t in globals().values()
                             if type(t) is type and issubclass(t, ModelBase)
-                        ]
-                        super({name}ModelBuilderSemantics, self).__init__(types=types)
-
-
-                class ModelBase(Node):
-                    pass
-
+                        ] + (types or [])
+                        super({name}ModelBuilderSemantics, self).__init__(context=context, types=types)
 
-                {base_class_declarations}{model_class_declarations}
+                {base_class_declarations}
+                {model_class_declarations}
                 '''
diff --git a/tatsu/contexts.py b/tatsu/contexts.py
index e479945..c4ac81d 100644
--- a/tatsu/contexts.py
+++ b/tatsu/contexts.py
@@ -195,7 +195,7 @@ class ParseContext(object):
                 filename=filename,
                 buffer_class=buffer_class,
                 semantics=semantics,
-                trace=trace or self.trace,
+                trace=trace if trace is not None else self.trace,
                 whitespace=whitespace if whitespace is not None else self.whitespace,
                 **kwargs
             )
diff --git a/tatsu/grammars.py b/tatsu/grammars.py
index 9e36ed5..039cb22 100644
--- a/tatsu/grammars.py
+++ b/tatsu/grammars.py
@@ -52,7 +52,7 @@ class EBNFBuffer(EBNFBootstrapBuffer):
             if re.match(PRAGMA_RE, line):
                 directive, arg = line.split('#', 1)[1], ''
                 if '::' in directive:
-                    directive, arg = directive.split('::')
+                    directive, arg = directive.split('::', 1)
                 directive, arg = directive.strip(), arg.strip()
                 i = self.pragma(name, directive, arg, lines, index, i)
             else:
diff --git a/tatsu/objectmodel.py b/tatsu/objectmodel.py
index 5be7337..3cd262c 100644
--- a/tatsu/objectmodel.py
+++ b/tatsu/objectmodel.py
@@ -25,8 +25,8 @@ class Node(object):
             parseinfo = ast.parseinfo if not parseinfo else None
         self._parseinfo = parseinfo
 
-        attributes = ast or {}
-        # asume that kwargs contains node attributes of interest
+        attributes = ast if ast is not None else {}
+        # assume that kwargs contains node attributes of interest
         if isinstance(attributes, MutableMapping):
             attributes.update(kwargs)
 
@@ -39,7 +39,10 @@ class Node(object):
             return
 
         for name in set(ast) - {'parseinfo'}:
-            setattr(self, name, ast[name])
+            try:
+                setattr(self, name, ast[name])
+            except AttributeError:
+                raise AttributeError("'%s' is a reserved name" % name)
 
     @property
     def ast(self):
diff --git a/tatsu/parser_semantics.py b/tatsu/parser_semantics.py
index c15b48b..3d8daf8 100644
--- a/tatsu/parser_semantics.py
+++ b/tatsu/parser_semantics.py
@@ -46,7 +46,7 @@ class EBNFGrammarSemantics(ModelBuilderSemantics):
     def string(self, ast):
         return eval_escapes(ast)
 
-    def hext(self, ast):
+    def hex(self, ast):
         return int(ast, 16)
 
     def float(self, ast):
diff --git a/tatsu/semantics.py b/tatsu/semantics.py
index 6cf88ba..7930f4e 100644
--- a/tatsu/semantics.py
+++ b/tatsu/semantics.py
@@ -31,6 +31,7 @@ class ModelBuilderSemantics(object):
         nodes using the class name given as first parameter to a grammar
         rule, and synthesizes the class/type if it's not known.
     """
+
     def __init__(self, context=None, base_type=Node, types=None):
         self.ctx = context
         self.base_type = base_type
@@ -79,11 +80,11 @@ class ModelBuilderSemantics(object):
 
         typespec = args[0].split(BASE_CLASS_TOKEN)
         typename = typespec[0]
-        bases = typespec[1:]
+        bases = typespec[-1:0:-1]
 
         base = self.base_type
-        for base in bases:
-            base = self._get_constructor(bases[0], base)
+        for base_ in bases:
+            base = self._get_constructor(base_, base)
 
         constructor = self._get_constructor(typename, base)
         try:
diff --git a/tatsu/synth.py b/tatsu/synth.py
index 20a8663..03de139 100644
--- a/tatsu/synth.py
+++ b/tatsu/synth.py
@@ -18,7 +18,7 @@ def synthesize(name, bases):
         bases = (bases,)
 
     if _Synthetic not in bases:
-        bases = (_Synthetic,) + bases
+        bases = bases + (_Synthetic,)
 
     constructor = __REGISTRY.get(typename)
     if not constructor:
diff --git a/tatsu/tool.py b/tatsu/tool.py
index f071974..a146421 100644
--- a/tatsu/tool.py
+++ b/tatsu/tool.py
@@ -8,6 +8,7 @@ import codecs
 import argparse
 import os
 import sys
+import importlib
 
 from tatsu._version import __version__
 from tatsu.util import eval_escapes
@@ -111,6 +112,24 @@ def parse_args():
         help='characters to skip during parsing (use "" to disable)',
     )
 
+    def import_class(path):
+        try:
+            spath = path.rsplit('.', 1)
+            module = importlib.import_module(spath[0])
+
+            return getattr(module, spath[1])
+        except Exception:
+            raise argparse.ArgumentTypeError(
+                "Couldn't find class %s" % path
+            )
+
+    generation_opts.add_argument(
+        '--base-type',
+        metavar='CLASSPATH',
+        help='class to use as base type for the object model, for example "mymodule.MyNode"',
+        type=import_class
+    )
+
     std_args = argparser.add_argument_group('common options')
     std_args.add_argument(
         '--help', '-h',
@@ -157,9 +176,9 @@ def to_python_sourcecode(grammar, name=None, filename=None, **kwargs):
     return pythoncg(model)
 
 
-def to_python_model(grammar, name=None, filename=None, **kwargs):
+def to_python_model(grammar, name=None, filename=None, base_type=None, **kwargs):
     model = compile(grammar, name=name, filename=filename, **kwargs)
-    return objectmodel.codegen(model)
+    return objectmodel.codegen(model, base_type=base_type)
 
 
 # for backwards compatibility. Use `compile()` instead
@@ -223,7 +242,7 @@ def main(codegen=pythoncg):
             elif args.pretty_lean:
                 result = model.pretty_lean()
             elif args.object_model:
-                result = objectmodel.codegen(model)
+                result = objectmodel.codegen(model, base_type=args.base_type)
             else:
                 result = codegen(model)
 
@@ -234,7 +253,7 @@ def main(codegen=pythoncg):
 
         # if requested, always save it
         if args.object_model_outfile:
-            save(args.object_model_outfile, objectmodel.codegen(model))
+            save(args.object_model_outfile, objectmodel.codegen(model, base_type=args.base_type))
 
         print('-' * 72, file=sys.stderr)
         print('{:12,d}  lines in grammar'.format(len(grammar.split())), file=sys.stderr)


Test patch:
diff --git a/test/grammar/parameter_test.py b/test/grammar/parameter_test.py
index 7c7b777..a8c4b94 100644
--- a/test/grammar/parameter_test.py
+++ b/test/grammar/parameter_test.py
@@ -192,7 +192,7 @@ class ParameterTests(unittest.TestCase):
 
     def test_numbers_and_unicode(self):
         grammar = '''
-            rúle(1, -23, 4.56, 7.89e-11, 0xABCDEF, Añez)
+            rúle(1, -23, 4.56, 7.89e-11, Añez)
                 =
                 'a'
                 ;
diff --git a/test/grammar/semantics_test.py b/test/grammar/semantics_test.py
index c23478c..7ee837f 100644
--- a/test/grammar/semantics_test.py
+++ b/test/grammar/semantics_test.py
@@ -7,6 +7,11 @@ from tatsu.tool import compile
 from tatsu.semantics import ModelBuilderSemantics
 
 
+class MyNode(object):
+    def __init__(self, ast):
+        pass
+
+
 class SemanticsTests(unittest.TestCase):
 
     def test_builder_semantics(self):
@@ -34,3 +39,55 @@ class SemanticsTests(unittest.TestCase):
         model = compile(grammar, 'test')
         ast = model.parse(text, semantics=semantics)
         self.assertEqual('5.4.3.2.1', ast)
+
+    def test_builder_subclassing(self):
+        from tatsu import synth
+        from tatsu.model import Node
+
+        registry = getattr(synth, "__REGISTRY")
+
+        grammar = '''
+            @@grammar :: Test
+            start::A::B::C = $ ;
+        '''
+
+        model = compile(grammar, asmodel=True)
+        model.parse("")
+
+        A = registry["A"]
+        B = registry["B"]
+        C = registry["C"]
+
+        self.assertTrue(issubclass(A, B) and issubclass(A, synth._Synthetic) and issubclass(A, Node))
+        self.assertTrue(issubclass(B, C) and issubclass(B, synth._Synthetic) and issubclass(A, Node))
+        self.assertTrue(issubclass(C, synth._Synthetic) and issubclass(C, Node))
+
+    def test_builder_basetype_codegen(self):
+        grammar = '''
+            @@grammar :: Test
+            start::A::B::C = a:() b:() $ ;
+            second::D::A = ();
+            third = ();
+        '''
+
+        from tatsu.tool import to_python_model
+        src = to_python_model(grammar, base_type=MyNode)
+
+        globals = {}
+        exec(src, globals)
+        semantics = globals["TestModelBuilderSemantics"]()
+
+        A = globals["A"]
+        B = globals["B"]
+        C = globals["C"]
+        D = globals["D"]
+
+        model = compile(grammar, semantics=semantics)
+        ast = model.parse("", semantics=semantics)
+
+        self.assertIsInstance(ast, MyNode)
+        self.assertIsInstance(ast, (A, B, C))
+        self.assertTrue(hasattr(ast, "a"))
+        self.assertTrue(hasattr(ast, "b"))
+
+        self.assertTrue(issubclass(D, (A, B, C)))
diff --git a/test/grammar/syntax_test.py b/test/grammar/syntax_test.py
index b733b19..5f007dc 100644
--- a/test/grammar/syntax_test.py
+++ b/test/grammar/syntax_test.py
@@ -320,3 +320,26 @@ class SyntaxTests(unittest.TestCase):
         model = compile(grammar, "start")
         ast = model.parse("1xx 2 yy")
         self.assertEqual(['1', 'xx', ' ', '2', 'yy'], ast)
+
+    def test_constant(self):
+        grammar = '''
+            start = ()
+                _0:`0` _1:`+1` _n123:`-123`
+                _xF:`0xF`
+                _string:`string`
+                _string_space:`'string space'`
+                _true:`True` _false:`False`
+                $;
+        '''
+
+        model = compile(grammar)
+        ast = model.parse("")
+
+        self.assertEqual(ast._0, 0)
+        self.assertEqual(ast._1, 1)
+        self.assertEqual(ast._n123, -123)
+        self.assertEqual(ast._xF, 0xF)
+        self.assertEqual(ast._string, "string")
+        self.assertEqual(ast._string_space, "string space")
+        self.assertEqual(ast._true, True)
+        self.assertEqual(ast._false, False)