Repository: lark-parser/lark
Pull request #356 (2019-04-10 05:57:26)

get_namespace_name now preserves leading underscore on relative import
fixes: https://github.com/lark-parser/lark/issues/353


Patch:
diff --git a/examples/standalone/create_standalone.sh b/examples/standalone/create_standalone.sh
index 141ab89..a4fa879 100755
--- a/examples/standalone/create_standalone.sh
+++ b/examples/standalone/create_standalone.sh
@@ -1,1 +1,1 @@
-PYTHONPATH=../.. python -m lark.tools.standalone json.lark > json_parser.py
+python -m lark.tools.standalone json.lark > json_parser.py
diff --git a/examples/standalone/json_parser.py b/examples/standalone/json_parser.py
index d424f1b..8c51baf 100644
--- a/examples/standalone/json_parser.py
+++ b/examples/standalone/json_parser.py
@@ -1,4 +1,4 @@
-# The file was automatically generated by Lark v0.7.0
+# The file was automatically generated by Lark v0.5.5
 #
 #
 #   Lark Stand-alone Generator Tool
@@ -18,9 +18,6 @@
 #    If you wish to purchase a commercial license for this tool and its
 #    generated code, contact me via email.
 #
-#    If GPL is incompatible with your free or open-source project,
-#    contact me and we'll work it out (for free).
-#
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation, either version 2 of the License, or
@@ -35,218 +32,91 @@
 #
 #
 
-class LarkError(Exception):
-    pass
-
-class GrammarError(LarkError):
-    pass
-
-class ParseError(LarkError):
-    pass
-
-class LexError(LarkError):
-    pass
 
-class UnexpectedInput(LarkError):
-    pos_in_stream = None
-
-    def get_context(self, text, span=40):
-        pos = self.pos_in_stream
-        start = max(pos - span, 0)
-        end = pos + span
-        before = text[start:pos].rsplit('\n', 1)[-1]
-        after = text[pos:end].split('\n', 1)[0]
-        return before + after + '\n' + ' ' * len(before) + '^\n'
-
-    def match_examples(self, parse_fn, examples):
-        """ Given a parser instance and a dictionary mapping some label with
-            some malformed syntax examples, it'll return the label for the
-            example that bests matches the current error.
-        """
-        assert self.state is not None, "Not supported for this exception"
-
-        candidate = None
-        for label, example in examples.items():
-            assert not isinstance(example, STRING_TYPE)
-
-            for malformed in example:
-                try:
-                    parse_fn(malformed)
-                except UnexpectedInput as ut:
-                    if ut.state == self.state:
-                        try:
-                            if ut.token == self.token:  # Try exact match first
-                                return label
-                        except AttributeError:
-                            pass
-                        if not candidate:
-                            candidate = label
+import types
+import functools
+from contextlib import contextmanager
 
-        return candidate
+Str = type(u'')
 
+def inline_args(f):
+    # print '@@', f.__name__, type(f), isinstance(f, types.FunctionType), isinstance(f, types.TypeType), isinstance(f, types.BuiltinFunctionType)
+    if isinstance(f, types.FunctionType):
+        @functools.wraps(f)
+        def _f_func(self, args):
+            return f(self, *args)
+        return _f_func
+    elif isinstance(f, (type, types.BuiltinFunctionType)):
+        @functools.wraps(f)
+        def _f_builtin(_self, args):
+            return f(*args)
+        return _f_builtin
+    elif isinstance(f, types.MethodType):
+        @functools.wraps(f.__func__)
+        def _f(self, args):
+            return f.__func__(self, *args)
+        return _f
+    else:
+        @functools.wraps(f.__call__.__func__)
+        def _f(self, args):
+            return f.__call__.__func__(self, *args)
+        return _f
 
-class UnexpectedCharacters(LexError, UnexpectedInput):
-    def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None):
-        message = "No terminal defined for '%s' at line %d col %d" % (seq[lex_pos], line, column)
 
-        self.line = line
-        self.column = column
-        self.allowed = allowed
-        self.considered_tokens = considered_tokens
-        self.pos_in_stream = lex_pos
-        self.state = state
+try:
+    from contextlib import suppress     # Python 3
+except ImportError:
+    @contextmanager
+    def suppress(*excs):
+        '''Catch and dismiss the provided exception
+
+        >>> x = 'hello'
+        >>> with suppress(IndexError):
+        ...     x = x[10]
+        >>> x
+        'hello'
+        '''
+        try:
+            yield
+        except excs:
+            pass
 
-        message += '\n\n' + self.get_context(seq)
-        if allowed:
-            message += '\nExpecting: %s\n' % allowed
 
-        super(UnexpectedCharacters, self).__init__(message)
+def is_terminal(sym):
+    return sym.isupper()
 
+class GrammarError(Exception):
+    pass
 
+class ParseError(Exception):
+    pass
 
-class UnexpectedToken(ParseError, UnexpectedInput):
-    def __init__(self, token, expected, considered_rules=None, state=None):
+class UnexpectedToken(ParseError):
+    def __init__(self, token, expected, seq, index, considered_rules=None):
         self.token = token
-        self.expected = expected     # XXX str shouldn't necessary
+        self.expected = expected
         self.line = getattr(token, 'line', '?')
         self.column = getattr(token, 'column', '?')
         self.considered_rules = considered_rules
-        self.state = state
-        self.pos_in_stream = getattr(token, 'pos_in_stream', None)
 
+        try:
+            context = ' '.join(['%r(%s)' % (t.value, t.type) for t in seq[index:index+5]])
+        except AttributeError:
+            context = seq[index:index+5]
+        except TypeError:
+            context = "<no context>"
         message = ("Unexpected token %r at line %s, column %s.\n"
-                   "Expected one of: \n\t* %s\n"
-                   % (token, self.line, self.column, '\n\t* '.join(self.expected)))
+                   "Expected: %s\n"
+                   "Context: %s" % (token, self.line, self.column, expected, context))
 
         super(UnexpectedToken, self).__init__(message)
 
-class VisitError(LarkError):
-    def __init__(self, tree, orig_exc):
-        self.tree = tree
-        self.orig_exc = orig_exc
-
-        message = 'Error trying to process rule "%s":\n\n%s' % (tree.data, orig_exc)
-        super(VisitError, self).__init__(message)
-
-def _deserialize(data, namespace, memo):
-    if isinstance(data, dict):
-        if '__type__' in data: # Object
-            class_ = namespace[data['__type__']]
-            return class_.deserialize(data, memo)
-        elif '@' in data:
-            return memo[data['@']]
-        return {key:_deserialize(value, namespace, memo) for key, value in data.items()}
-    elif isinstance(data, list):
-        return [_deserialize(value, namespace, memo) for value in data]
-    return data
-
-
-class Serialize(object):
-    def memo_serialize(self, types_to_memoize):
-        memo = SerializeMemoizer(types_to_memoize)
-        return self.serialize(memo), memo.serialize()
-
-    def serialize(self, memo=None):
-        if memo and memo.in_types(self):
-            return {'@': memo.memoized.get(self)}
-
-        fields = getattr(self, '__serialize_fields__')
-        res = {f: _serialize(getattr(self, f), memo) for f in fields}
-        res['__type__'] = type(self).__name__
-        postprocess = getattr(self, '_serialize', None)
-        if postprocess:
-            postprocess(res, memo)
-        return res
-
-    @classmethod
-    def deserialize(cls, data, memo):
-        namespace = getattr(cls, '__serialize_namespace__', {})
-        namespace = {c.__name__:c for c in namespace}
-
-        fields = getattr(cls, '__serialize_fields__')
-
-        if '@' in data:
-            return memo[data['@']]
-
-        inst = cls.__new__(cls)
-        for f in fields:
-            setattr(inst, f, _deserialize(data[f], namespace, memo))
-        postprocess = getattr(inst, '_deserialize', None)
-        if postprocess:
-            postprocess()
-        return inst
-
-
-class SerializeMemoizer(Serialize):
-    __serialize_fields__ = 'memoized',
-
-    def __init__(self, types_to_memoize):
-        self.types_to_memoize = tuple(types_to_memoize)
-        self.memoized = Enumerator()
-
-    def in_types(self, value):
-        return isinstance(value, self.types_to_memoize)
-
-    def serialize(self):
-        return _serialize(self.memoized.reversed(), None)
-
-    @classmethod
-    def deserialize(cls, data, namespace, memo):
-        return _deserialize(data, namespace, memo)
-
-
-
-try:
-    STRING_TYPE = basestring
-except NameError:   # Python 3
-    STRING_TYPE = str
-
-
-import types
-from functools import wraps, partial
-from contextlib import contextmanager
-
-Str = type(u'')
-try:
-    classtype = types.ClassType # Python2
-except AttributeError:
-    classtype = type    # Python3
-
-def smart_decorator(f, create_decorator):
-    if isinstance(f, types.FunctionType):
-        return wraps(f)(create_decorator(f, True))
-
-    elif isinstance(f, (classtype, type, types.BuiltinFunctionType)):
-        return wraps(f)(create_decorator(f, False))
-
-    elif isinstance(f, types.MethodType):
-        return wraps(f)(create_decorator(f.__func__, True))
-
-    elif isinstance(f, partial):
-        # wraps does not work for partials in 2.7: https://bugs.python.org/issue3445
-        return create_decorator(f.__func__, True)
-
-    else:
-        return create_decorator(f.__func__.__call__, True)
-
-import sys, re
-Py36 = (sys.version_info[:2] >= (3, 6))
 
-class Meta:
-    def __init__(self):
-        self.empty = True
 
 class Tree(object):
-    def __init__(self, data, children, meta=None):
+    def __init__(self, data, children):
         self.data = data
         self.children = children
-        self._meta = meta
-
-    @property
-    def meta(self):
-        if self._meta is None:
-            self._meta = Meta()
-        return self._meta
 
     def __repr__(self):
         return 'Tree(%s, %s)' % (self.data, self.children)
@@ -269,111 +139,33 @@ class Tree(object):
 
     def pretty(self, indent_str='  '):
         return ''.join(self._pretty(0, indent_str))
+class Transformer(object):
+    def _get_func(self, name):
+        return getattr(self, name)
 
-    def __eq__(self, other):
-        try:
-            return self.data == other.data and self.children == other.children
-        except AttributeError:
-            return False
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __hash__(self):
-        return hash((self.data, tuple(self.children)))
-
-from inspect import getmembers, getmro
-
-class Discard(Exception):
-    pass
-
-# Transformers
-
-class Transformer:
-    """Visits the tree recursively, starting with the leaves and finally the root (bottom-up)
-
-    Calls its methods (provided by user via inheritance) according to tree.data
-    The returned value replaces the old one in the structure.
-
-    Can be used to implement map or reduce.
-    """
-
-    def _call_userfunc(self, tree, new_children=None):
-        # Assumes tree is already transformed
-        children = new_children if new_children is not None else tree.children
-        try:
-            f = getattr(self, tree.data)
-        except AttributeError:
-            return self.__default__(tree.data, children, tree.meta)
-        else:
-            try:
-                if getattr(f, 'meta', False):
-                    return f(children, tree.meta)
-                elif getattr(f, 'inline', False):
-                    return f(*children)
-                elif getattr(f, 'whole_tree', False):
-                    if new_children is not None:
-                        raise NotImplementedError("Doesn't work with the base Transformer class")
-                    return f(tree)
-                else:
-                    return f(children)
-            except (GrammarError, Discard):
-                raise
-            except Exception as e:
-                raise VisitError(tree, e)
-
-    def _transform_children(self, children):
-        for c in children:
+    def transform(self, tree):
+        items = []
+        for c in tree.children:
             try:
-                yield self._transform_tree(c) if isinstance(c, Tree) else c
+                items.append(self.transform(c) if isinstance(c, Tree) else c)
             except Discard:
                 pass
+        try:
+            f = self._get_func(tree.data)
+        except AttributeError:
+            return self.__default__(tree.data, items)
+        else:
+            return f(items)
 
-    def _transform_tree(self, tree):
-        children = list(self._transform_children(tree.children))
-        return self._call_userfunc(tree, children)
-
-    def transform(self, tree):
-        return self._transform_tree(tree)
+    def __default__(self, data, children):
+        return Tree(data, children)
 
     def __mul__(self, other):
         return TransformerChain(self, other)
 
-    def __default__(self, data, children, meta):
-        "Default operation on tree (for override)"
-        return Tree(data, children, meta)
-
-    @classmethod
-    def _apply_decorator(cls, decorator, **kwargs):
-        mro = getmro(cls)
-        assert mro[0] is cls
-        libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)}
-        for name, value in getmembers(cls):
-            if name.startswith('_') or name in libmembers:
-                continue
-            if not callable(cls.__dict__[name]):
-                continue
-
-            # Skip if v_args already applied (at the function level)
-            if hasattr(cls.__dict__[name], 'vargs_applied'):
-                continue
-
-            static = isinstance(cls.__dict__[name], (staticmethod, classmethod))
-            setattr(cls, name, decorator(value, static=static, **kwargs))
-        return cls
-
-
-class InlineTransformer(Transformer):   # XXX Deprecated
-    def _call_userfunc(self, tree, new_children=None):
-        # Assumes tree is already transformed
-        children = new_children if new_children is not None else tree.children
-        try:
-            f = getattr(self, tree.data)
-        except AttributeError:
-            return self.__default__(tree.data, children, tree.meta)
-        else:
-            return f(*children)
 
+class Discard(Exception):
+    pass
 
 class TransformerChain(object):
     def __init__(self, *transformers):
@@ -388,57 +180,13 @@ class TransformerChain(object):
         return TransformerChain(*self.transformers + (other,))
 
 
-class Transformer_InPlace(Transformer):
-    "Non-recursive. Changes the tree in-place instead of returning new instances"
-    def _transform_tree(self, tree):           # Cancel recursion
-        return self._call_userfunc(tree)
-
-    def transform(self, tree):
-        for subtree in tree.iter_subtrees():
-            subtree.children = list(self._transform_children(subtree.children))
-
-        return self._transform_tree(tree)
-
-
-class Transformer_InPlaceRecursive(Transformer):
-    "Recursive. Changes the tree in-place instead of returning new instances"
-    def _transform_tree(self, tree):
-        tree.children = list(self._transform_children(tree.children))
-        return self._call_userfunc(tree)
-
-
-
-# Visitors
-
-class VisitorBase:
-    def _call_userfunc(self, tree):
-        return getattr(self, tree.data, self.__default__)(tree)
-
-    def __default__(self, tree):
-        "Default operation on tree (for override)"
-        return tree
-
-
-class Visitor(VisitorBase):
-    """Bottom-up visitor, non-recursive
-
-    Visits the tree, starting with the leaves and finally the root (bottom-up)
-    Calls its methods (provided by user via inheritance) according to tree.data
-    """
-
-
-    def visit(self, tree):
-        for subtree in tree.iter_subtrees():
-            self._call_userfunc(subtree)
-        return tree
 
-class Visitor_Recursive(VisitorBase):
-    """Bottom-up visitor, recursive
+class InlineTransformer(Transformer):
+    def _get_func(self, name):  # use super()._get_func
+        return inline_args(getattr(self, name)).__get__(self)
 
-    Visits the tree, starting with the leaves and finally the root (bottom-up)
-    Calls its methods (provided by user via inheritance) according to tree.data
-    """
 
+class Visitor(object):
     def visit(self, tree):
         for child in tree.children:
             if isinstance(child, Tree):
@@ -448,109 +196,50 @@ class Visitor_Recursive(VisitorBase):
         f(tree)
         return tree
 
-
-
-def visit_children_decor(func):
-    "See Interpreter"
-    @wraps(func)
-    def inner(cls, tree):
-        values = cls.visit_children(tree)
-        return func(cls, values)
-    return inner
-
-
-class Interpreter:
-    """Top-down visitor, recursive
-
-    Visits the tree, starting with the root and finally the leaves (top-down)
-    Calls its methods (provided by user via inheritance) according to tree.data
-
-    Unlike Transformer and Visitor, the Interpreter doesn't automatically visit its sub-branches.
-    The user has to explicitly call visit_children, or use the @visit_children_decor
-    """
-    def visit(self, tree):
-        return getattr(self, tree.data)(tree)
-
-    def visit_children(self, tree):
-        return [self.visit(child) if isinstance(child, Tree) else child
-                for child in tree.children]
-
-    def __getattr__(self, name):
-        return self.__default__
-
     def __default__(self, tree):
-        return self.visit_children(tree)
-
+        pass
 
 
+class Visitor_NoRecurse(Visitor):
+    def visit(self, tree):
+        subtrees = list(tree.iter_subtrees())
 
-# Decorators
-
-def _apply_decorator(obj, decorator, **kwargs):
-    try:
-        _apply = obj._apply_decorator
-    except AttributeError:
-        return decorator(obj, **kwargs)
-    else:
-        return _apply(decorator, **kwargs)
-
-
-
-def _inline_args__func(func):
-    @wraps(func)
-    def create_decorator(_f, with_self):
-        if with_self:
-            def f(self, children):
-                return _f(self, *children)
-        else:
-            def f(self, children):
-                return _f(*children)
-        return f
-
-    return smart_decorator(func, create_decorator)
-
-
-def inline_args(obj):   # XXX Deprecated
-    return _apply_decorator(obj, _inline_args__func)
-
+        for subtree in (subtrees):
+            getattr(self, subtree.data, self.__default__)(subtree)
+        return tree
 
 
-def _visitor_args_func_dec(func, inline=False, meta=False, whole_tree=False, static=False):
-    assert [whole_tree, meta, inline].count(True) <= 1
-    def create_decorator(_f, with_self):
-        if with_self:
-            def f(self, *args, **kwargs):
-                return _f(self, *args, **kwargs)
-        else:
-            def f(self, *args, **kwargs):
-                return _f(*args, **kwargs)
-        return f
+class Transformer_NoRecurse(Transformer):
+    def transform(self, tree):
+        subtrees = list(tree.iter_subtrees())
 
-    if static:
-        f = wraps(func)(create_decorator(func, False))
-    else:
-        f = smart_decorator(func, create_decorator)
-    f.vargs_applied = True
-    f.inline = inline
-    f.meta = meta
-    f.whole_tree = whole_tree
-    return f
+        def _t(t):
+            # Assumes t is already transformed
+            try:
+                f = self._get_func(t.data)
+            except AttributeError:
+                return self.__default__(t)
+            else:
+                return f(t)
 
-def v_args(inline=False, meta=False, tree=False):
-    "A convenience decorator factory, for modifying the behavior of user-supplied visitor methods"
-    if [tree, meta, inline].count(True) > 1:
-        raise ValueError("Visitor functions can either accept tree, or meta, or be inlined. These cannot be combined.")
-    def _visitor_args_dec(obj):
-        return _apply_decorator(obj, _visitor_args_func_dec, inline=inline, meta=meta, whole_tree=tree)
-    return _visitor_args_dec
+        for subtree in subtrees:
+            children = []
+            for c in subtree.children:
+                try:
+                    children.append(_t(c) if isinstance(c, Tree) else c)
+                except Discard:
+                    pass
+            subtree.children = children
 
+        return _t(tree)
 
+    def __default__(self, t):
+        return t
 
 class Indenter:
     def __init__(self):
-        self.paren_level = None
-        self.indent_level = None
-        assert self.tab_len > 0
+        self.paren_level = 0
+        self.indent_level = [0]
 
     def handle_NL(self, token):
         if self.paren_level > 0:
@@ -571,7 +260,7 @@ class Indenter:
 
             assert indent == self.indent_level[-1], '%s != %s' % (indent, self.indent_level[-1])
 
-    def _process(self, stream):
+    def process(self, stream):
         for token in stream:
             if token.type == self.NL_type:
                 for t in self.handle_NL(token):
@@ -591,213 +280,43 @@ class Indenter:
 
         assert self.indent_level == [0], self.indent_level
 
-    def process(self, stream):
-        self.paren_level = 0
-        self.indent_level = [0]
-        return self._process(stream)
-
     # XXX Hack for ContextualLexer. Maybe there's a more elegant solution?
     @property
     def always_accept(self):
         return (self.NL_type,)
 
 
+class LexError(Exception):
+    pass
 
-class Symbol(Serialize):
-    is_term = NotImplemented
-
-    def __init__(self, name):
-        self.name = name
-
-    def __eq__(self, other):
-        assert isinstance(other, Symbol), other
-        return self.is_term == other.is_term and self.name == other.name
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __hash__(self):
-        return hash(self.name)
-
-    def __repr__(self):
-        return '%s(%r)' % (type(self).__name__, self.name)
-
-    fullrepr = property(__repr__)
-
-
-class Terminal(Symbol):
-    __serialize_fields__ = 'name', 'filter_out'
-
-    is_term = True
-
-    def __init__(self, name, filter_out=False):
-        self.name = name
-        self.filter_out = filter_out
-
-    @property
-    def fullrepr(self):
-        return '%s(%r, %r)' % (type(self).__name__, self.name, self.filter_out)
-
-
-
-class NonTerminal(Symbol):
-    __serialize_fields__ = 'name',
-
-    is_term = False
-
-
-
-class RuleOptions(Serialize):
-    __serialize_fields__ = 'keep_all_tokens', 'expand1', 'priority', 'empty_indices'
-
-    def __init__(self, keep_all_tokens=False, expand1=False, priority=None, empty_indices=()):
-        self.keep_all_tokens = keep_all_tokens
-        self.expand1 = expand1
-        self.priority = priority
-        self.empty_indices = empty_indices
-
-    def __repr__(self):
-        return 'RuleOptions(%r, %r, %r)' % (
-            self.keep_all_tokens,
-            self.expand1,
-            self.priority,
-        )
-
-
-class Rule(Serialize):
-    """
-        origin : a symbol
-        expansion : a list of symbols
-        order : index of this expansion amongst all rules of the same name
-    """
-    __slots__ = ('origin', 'expansion', 'alias', 'options', 'order', '_hash')
-
-    __serialize_fields__ = 'origin', 'expansion', 'order', 'alias', 'options'
-    __serialize_namespace__ = Terminal, NonTerminal, RuleOptions
-
-    def __init__(self, origin, expansion, order=0, alias=None, options=None):
-        self.origin = origin
-        self.expansion = expansion
-        self.alias = alias
-        self.order = order
-        self.options = options
-        self._hash = hash((self.origin, tuple(self.expansion)))
-
-    def _deserialize(self):
-        self._hash = hash((self.origin, tuple(self.expansion)))
-
-    def __str__(self):
-        return '<%s : %s>' % (self.origin.name, ' '.join(x.name for x in self.expansion))
-
-    def __repr__(self):
-        return 'Rule(%r, %r, %r, %r)' % (self.origin, self.expansion, self.alias, self.options)
-
-    def __hash__(self):
-        return self._hash
-
-    def __eq__(self, other):
-        if not isinstance(other, Rule):
-            return False
-        return self.origin == other.origin and self.expansion == other.expansion
-
-
-
-
-
-class Pattern(Serialize):
-    __serialize_fields__ = 'value', 'flags'
-
-    def __init__(self, value, flags=()):
-        self.value = value
-        self.flags = frozenset(flags)
-
-    def __repr__(self):
-        return repr(self.to_regexp())
-
-    # Pattern Hashing assumes all subclasses have a different priority!
-    def __hash__(self):
-        return hash((type(self), self.value, self.flags))
-    def __eq__(self, other):
-        return type(self) == type(other) and self.value == other.value and self.flags == other.flags
-
-    def to_regexp(self):
-        raise NotImplementedError()
-
-    if Py36:
-        # Python 3.6 changed syntax for flags in regular expression
-        def _get_flags(self, value):
-            for f in self.flags:
-                value = ('(?%s:%s)' % (f, value))
-            return value
-
-    else:
-        def _get_flags(self, value):
-            for f in self.flags:
-                value = ('(?%s)' % f) + value
-            return value
-
-
-class PatternStr(Pattern):
-    def to_regexp(self):
-        return self._get_flags(re.escape(self.value))
-
-    @property
-    def min_width(self):
-        return len(self.value)
-    max_width = min_width
-
-class PatternRE(Pattern):
-    def to_regexp(self):
-        return self._get_flags(self.value)
-
-    @property
-    def min_width(self):
-        return get_regexp_width(self.to_regexp())[0]
-    @property
-    def max_width(self):
-        return get_regexp_width(self.to_regexp())[1]
-
-
-class TerminalDef(Serialize):
-    __serialize_fields__ = 'name', 'pattern', 'priority'
-    __serialize_namespace__ = PatternStr, PatternRE
-
-    def __init__(self, name, pattern, priority=1):
-        assert isinstance(pattern, Pattern), pattern
-        self.name = name
-        self.pattern = pattern
-        self.priority = priority
-
-    def __repr__(self):
-        return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern)
-
-
+class UnexpectedInput(LexError):
+    def __init__(self, seq, lex_pos, line, column, allowed=None, considered_rules=None):
+        context = seq[lex_pos:lex_pos+5]
+        message = "No token defined for: '%s' in %r at line %d col %d" % (seq[lex_pos], context, line, column)
+        if allowed:
+            message += '\n\nExpecting: %s\n' % allowed
 
-class Token(Str):
-    __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column')
+        super(UnexpectedInput, self).__init__(message)
 
-    def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None, end_line=None, end_column=None):
-        try:
-            self = super(Token, cls).__new__(cls, value)
-        except UnicodeDecodeError:
-            value = value.decode('latin1')
-            self = super(Token, cls).__new__(cls, value)
-
-        self.type = type_
-        self.pos_in_stream = pos_in_stream
-        self.value = value
         self.line = line
         self.column = column
-        self.end_line = end_line
-        self.end_column = end_column
-        return self
+        self.context = context
+        self.allowed = allowed
+        self.considered_rules = considered_rules
+
+class Token(Str):
+    def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
+        inst = Str.__new__(cls, value)
+        inst.type = type_
+        inst.pos_in_stream = pos_in_stream
+        inst.value = value
+        inst.line = line
+        inst.column = column
+        return inst
 
     @classmethod
     def new_borrow_pos(cls, type_, value, borrow_t):
-        return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column)
-
-    def __reduce__(self):
-        return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column, ))
+        return cls(type_, value, borrow_t.pos_in_stream, line=borrow_t.line, column=borrow_t.column)
 
     def __repr__(self):
         return 'Token(%s, %r)' % (self.type, self.value)
@@ -819,7 +338,7 @@ class LineCounter:
         self.newline_char = '\n'
         self.char_pos = 0
         self.line = 1
-        self.column = 1
+        self.column = 0
         self.line_start_pos = 0
 
     def feed(self, token, test_newline=True):
@@ -834,51 +353,45 @@ class LineCounter:
                 self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1
 
         self.char_pos += len(token)
-        self.column = self.char_pos - self.line_start_pos + 1
+        self.column = self.char_pos - self.line_start_pos
 
 class _Lex:
     "Built to serve both Lexer and ContextualLexer"
-    def __init__(self, lexer, state=None):
+    def __init__(self, lexer):
         self.lexer = lexer
-        self.state = state
 
     def lex(self, stream, newline_types, ignore_types):
-        newline_types = frozenset(newline_types)
-        ignore_types = frozenset(ignore_types)
+        newline_types = list(newline_types)
+        ignore_types = list(ignore_types)
         line_ctr = LineCounter()
 
-        while line_ctr.char_pos < len(stream):
+        t = None
+        while True:
             lexer = self.lexer
             for mre, type_from_index in lexer.mres:
                 m = mre.match(stream, line_ctr.char_pos)
-                if not m:
-                    continue
-
-                t = None
-                value = m.group(0)
-                type_ = type_from_index[m.lastindex]
-                if type_ not in ignore_types:
-                    t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
-                    if t.type in lexer.callback:
-                        t = lexer.callback[t.type](t)
-                        if not isinstance(t, Token):
-                            raise ValueError("Callbacks must return a token (returned %r)" % t)
-                    yield t
-                else:
-                    if type_ in lexer.callback:
+                if m:
+                    value = m.group(0)
+                    type_ = type_from_index[m.lastindex]
+                    if type_ not in ignore_types:
                         t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
-                        lexer.callback[type_](t)
-
-                line_ctr.feed(value, type_ in newline_types)
-                if t:
-                    t.end_line = line_ctr.line
-                    t.end_column = line_ctr.column
-
-                break
+                        if t.type in lexer.callback:
+                            t = lexer.callback[t.type](t)
+                        yield t
+                    else:
+                        if type_ in lexer.callback:
+                            t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
+                            lexer.callback[type_](t)
+
+                    line_ctr.feed(value, type_ in newline_types)
+                    if t:
+                        t.end_line = line_ctr.line
+                        t.end_column = line_ctr.column
+                    break
             else:
-                allowed = [v for m, tfi in lexer.mres for v in tfi.values()]
-                raise UnexpectedCharacters(stream, line_ctr.char_pos, line_ctr.line, line_ctr.column, allowed=allowed, state=self.state)
-
+                if line_ctr.char_pos < len(stream):
+                    raise UnexpectedInput(stream, line_ctr.char_pos, line_ctr.line, line_ctr.column)
+                break
 
 class UnlessCallback:
     def __init__(self, mres):
@@ -888,183 +401,14 @@ class UnlessCallback:
         for mre, type_from_index in self.mres:
             m = mre.match(t.value)
             if m:
+                value = m.group(0)
                 t.type = type_from_index[m.lastindex]
                 break
         return t
 
-class CallChain:
-    def __init__(self, callback1, callback2, cond):
-        self.callback1 = callback1
-        self.callback2 = callback2
-        self.cond = cond
-
-    def __call__(self, t):
-        t2 = self.callback1(t)
-        return self.callback2(t) if self.cond(t2) else t2
-
-
-
-
-
-def _create_unless(terminals):
-    tokens_by_type = classify(terminals, lambda t: type(t.pattern))
-    assert len(tokens_by_type) <= 2, tokens_by_type.keys()
-    embedded_strs = set()
-    callback = {}
-    for retok in tokens_by_type.get(PatternRE, []):
-        unless = [] # {}
-        for strtok in tokens_by_type.get(PatternStr, []):
-            if strtok.priority > retok.priority:
-                continue
-            s = strtok.pattern.value
-            m = re.match(retok.pattern.to_regexp(), s)
-            if m and m.group(0) == s:
-                unless.append(strtok)
-                if strtok.pattern.flags <= retok.pattern.flags:
-                    embedded_strs.add(strtok)
-        if unless:
-            callback[retok.name] = UnlessCallback(build_mres(unless, match_whole=True))
-
-    terminals = [t for t in terminals if t not in embedded_strs]
-    return terminals, callback
-
-
-def _build_mres(terminals, max_size, match_whole):
-    # Python sets an unreasonable group limit (currently 100) in its re module
-    # Worse, the only way to know we reached it is by catching an AssertionError!
-    # This function recursively tries less and less groups until it's successful.
-    postfix = '$' if match_whole else ''
-    mres = []
-    while terminals:
-        try:
-            mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in terminals[:max_size]))
-        except AssertionError:  # Yes, this is what Python provides us.. :/
-            return _build_mres(terminals, max_size//2, match_whole)
-
-        # terms_from_name = {t.name: t for t in terminals[:max_size]}
-        mres.append((mre, {i:n for n,i in mre.groupindex.items()} ))
-        terminals = terminals[max_size:]
-    return mres
-
-def build_mres(terminals, match_whole=False):
-    return _build_mres(terminals, len(terminals), match_whole)
-
-def _regexp_has_newline(r):
-    """Expressions that may indicate newlines in a regexp:
-        - newlines (\n)
-        - escaped newline (\\n)
-        - anything but ([^...])
-        - any-char (.) when the flag (?s) exists
-    """
-    return '\n' in r or '\\n' in r or '[^' in r or ('(?s' in r and '.' in r)
-
-class Lexer(Serialize):
-    """Lexer interface
-
-    Method Signatures:
-        lex(self, stream) -> Iterator[Token]
-
-        set_parser_state(self, state)   # Optional
-    """
-    set_parser_state = NotImplemented
-    lex = NotImplemented
-
-
-class TraditionalLexer(Lexer):
-    __serialize_fields__ = 'terminals', 'ignore_types', 'newline_types'
-    __serialize_namespace__ = TerminalDef,
-
-    def _deserialize(self):
-        self.mres = build_mres(self.terminals)
-        self.callback = {}  # TODO implement
-
-
-    def __init__(self, terminals, ignore=(), user_callbacks={}):
-        assert all(isinstance(t, TerminalDef) for t in terminals), terminals
-
-        terminals = list(terminals)
-
-        # Sanitization
-        for t in terminals:
-            try:
-                re.compile(t.pattern.to_regexp())
-            except:
-                raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))
-
-            if t.pattern.min_width == 0:
-                raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern))
-
-        assert set(ignore) <= {t.name for t in terminals}
-
-        # Init
-        self.newline_types = [t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())]
-        self.ignore_types = list(ignore)
-
-        terminals.sort(key=lambda x:(-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name))
-
-        terminals, self.callback = _create_unless(terminals)
-        assert all(self.callback.values())
-
-        for type_, f in user_callbacks.items():
-            if type_ in self.callback:
-                # Already a callback there, probably UnlessCallback
-                self.callback[type_] = CallChain(self.callback[type_], f, lambda t: t.type == type_)
-            else:
-                self.callback[type_] = f
-
-        self.terminals = terminals
-
-        self.mres = build_mres(terminals)
-
-
-    def lex(self, stream):
-        return _Lex(self).lex(stream, self.newline_types, self.ignore_types)
-
-
-
 
-class ContextualLexer(Lexer):
-    __serialize_fields__ = 'root_lexer', 'lexers'
-    __serialize_namespace__ = TraditionalLexer,
-
-    def __init__(self, terminals, states, ignore=(), always_accept=(), user_callbacks={}):
-        tokens_by_name = {}
-        for t in terminals:
-            assert t.name not in tokens_by_name, t
-            tokens_by_name[t.name] = t
-
-        lexer_by_tokens = {}
-        self.lexers = {}
-        for state, accepts in states.items():
-            key = frozenset(accepts)
-            try:
-                lexer = lexer_by_tokens[key]
-            except KeyError:
-                accepts = set(accepts) | set(ignore) | set(always_accept)
-                state_tokens = [tokens_by_name[n] for n in accepts if n and n in tokens_by_name]
-                lexer = TraditionalLexer(state_tokens, ignore=ignore, user_callbacks=user_callbacks)
-                lexer_by_tokens[key] = lexer
-
-            self.lexers[state] = lexer
-
-        self.root_lexer = TraditionalLexer(terminals, ignore=ignore, user_callbacks=user_callbacks)
-
-        self.set_parser_state(None) # Needs to be set on the outside
-
-    def set_parser_state(self, state):
-        self.parser_state = state
-
-    def lex(self, stream):
-        l = _Lex(self.lexers[self.parser_state], self.parser_state)
-        for x in l.lex(stream, self.root_lexer.newline_types, self.root_lexer.ignore_types):
-            yield x
-            l.lexer = self.lexers[self.parser_state]
-            l.state = self.parser_state
-
-
-from functools import partial, wraps
-from itertools import repeat, product
 
+from functools import partial
 
 class ExpandSingleChild:
     def __init__(self, node_builder):
@@ -1076,90 +420,42 @@ class ExpandSingleChild:
         else:
             return self.node_builder(children)
 
-class PropagatePositions:
-    def __init__(self, node_builder):
-        self.node_builder = node_builder
-
-    def __call__(self, children):
-        res = self.node_builder(children)
-
-        if isinstance(res, Tree):
-            for c in children:
-                if isinstance(c, Tree) and c.children and not c.meta.empty:
-                    res.meta.line = c.meta.line
-                    res.meta.column = c.meta.column
-                    res.meta.start_pos = c.meta.start_pos
-                    res.meta.empty = False
-                    break
-                elif isinstance(c, Token):
-                    res.meta.line = c.line
-                    res.meta.column = c.column
-                    res.meta.start_pos = c.pos_in_stream
-                    res.meta.empty = False
-                    break
 
-            for c in reversed(children):
-                if isinstance(c, Tree) and c.children and not c.meta.empty:
-                    res.meta.end_line = c.meta.end_line
-                    res.meta.end_column = c.meta.end_column
-                    res.meta.end_pos = c.meta.end_pos
-                    res.meta.empty = False
-                    break
-                elif isinstance(c, Token):
-                    res.meta.end_line = c.end_line
-                    res.meta.end_column = c.end_column
-                    res.meta.end_pos = c.pos_in_stream + len(c.value)
-                    res.meta.empty = False
-                    break
+class CreateToken:
+    "Used for fixing the results of scanless parsing"
 
-        return res
-
-
-class ChildFilter:
-    def __init__(self, to_include, append_none, node_builder):
+    def __init__(self, token_name, node_builder):
         self.node_builder = node_builder
-        self.to_include = to_include
-        self.append_none = append_none
+        self.token_name = token_name
 
     def __call__(self, children):
-        filtered = []
+        return self.node_builder( [Token(self.token_name, ''.join(children))] )
 
-        for i, to_expand, add_none in self.to_include:
-            if add_none:
-                filtered += [None] * add_none
-            if to_expand:
-                filtered += children[i].children
-            else:
-                filtered.append(children[i])
 
-        if self.append_none:
-            filtered += [None] * self.append_none
+class PropagatePositions:
+    def __init__(self, node_builder):
+        self.node_builder = node_builder
 
-        return self.node_builder(filtered)
+    def __call__(self, children):
+        res = self.node_builder(children)
 
-class ChildFilterLALR(ChildFilter):
-    "Optimized childfilter for LALR (assumes no duplication in parse tree, so it's safe to change it)"
+        if children:
+            for a in children:
+                with suppress(AttributeError):
+                    res.line = a.line
+                    res.column = a.column
+                break
 
-    def __call__(self, children):
-        filtered = []
-        for i, to_expand, add_none in self.to_include:
-            if add_none:
-                filtered += [None] * add_none
-            if to_expand:
-                if filtered:
-                    filtered += children[i].children
-                else:   # Optimize for left-recursion
-                    filtered = children[i].children
-            else:
-                filtered.append(children[i])
+            for a in reversed(children):
+                with suppress(AttributeError):
+                    res.end_line = a.end_line
+                    res.end_column = a.end_column
+                break
 
-        if self.append_none:
-            filtered += [None] * self.append_none
+        return res
 
-        return self.node_builder(filtered)
 
-class ChildFilterLALR_NoPlaceholders(ChildFilter):
-    "Optimized childfilter for LALR (assumes no duplication in parse tree, so it's safe to change it)"
+class ChildFilter:
     def __init__(self, to_include, node_builder):
         self.node_builder = node_builder
         self.to_include = to_include
@@ -1174,158 +470,77 @@ class ChildFilterLALR_NoPlaceholders(ChildFilter):
                     filtered = children[i].children
             else:
                 filtered.append(children[i])
+
         return self.node_builder(filtered)
 
 def _should_expand(sym):
-    return not sym.is_term and sym.name.startswith('_')
-
-def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices):
-    # Prepare empty_indices as: How many Nones to insert at each index?
-    if _empty_indices:
-        assert _empty_indices.count(False) == len(expansion)
-        s = ''.join(str(int(b)) for b in _empty_indices)
-        empty_indices = [len(ones) for ones in s.split('0')]
-        assert len(empty_indices) == len(expansion)+1, (empty_indices, len(expansion))
-    else:
-        empty_indices = [0] * (len(expansion)+1)
-
-    to_include = []
-    nones_to_add = 0
-    for i, sym in enumerate(expansion):
-        nones_to_add += empty_indices[i]
-        if keep_all_tokens or not (sym.is_term and sym.filter_out):
-            to_include.append((i, _should_expand(sym), nones_to_add))
-            nones_to_add = 0
-
-    nones_to_add += empty_indices[len(expansion)]
-
-    if _empty_indices or len(to_include) < len(expansion) or any(to_expand for i, to_expand,_ in to_include):
-        if _empty_indices or ambiguous:
-            return partial(ChildFilter if ambiguous else ChildFilterLALR, to_include, nones_to_add)
-        else:
-            # LALR without placeholders
-            return partial(ChildFilterLALR_NoPlaceholders, [(i, x) for i,x,_ in to_include])
-
-class AmbiguousExpander:
-    """Deal with the case where we're expanding children ('_rule') into a parent but the children
-       are ambiguous. i.e. (parent->_ambig->_expand_this_rule). In this case, make the parent itself
-       ambiguous with as many copies as their are ambiguous children, and then copy the ambiguous children
-       into the right parents in the right places, essentially shifting the ambiguiuty up the tree."""
-    def __init__(self, to_expand, tree_class, node_builder):
-        self.node_builder = node_builder
-        self.tree_class = tree_class
-        self.to_expand = to_expand
+    return not is_terminal(sym) and sym.startswith('_')
 
-    def __call__(self, children):
-        def _is_ambig_tree(child):
-            return hasattr(child, 'data') and child.data == '_ambig'
-
-        #### When we're repeatedly expanding ambiguities we can end up with nested ambiguities.
-        #    All children of an _ambig node should be a derivation of that ambig node, hence
-        #    it is safe to assume that if we see an _ambig node nested within an ambig node
-        #    it is safe to simply expand it into the parent _ambig node as an alternative derivation.
-        ambiguous = []
-        for i, child in enumerate(children):
-            if _is_ambig_tree(child):
-                if i in self.to_expand:
-                    ambiguous.append(i)
-
-                to_expand = [j for j, grandchild in enumerate(child.children) if _is_ambig_tree(grandchild)]
-                child.expand_kids_by_index(*to_expand)
-
-        if not ambiguous:
-            return self.node_builder(children)
+def maybe_create_child_filter(expansion, filter_out):
+    to_include = [(i, _should_expand(sym)) for i, sym in enumerate(expansion) if sym not in filter_out]
 
-        expand = [ iter(child.children) if i in ambiguous else repeat(child) for i, child in enumerate(children) ]
-        return self.tree_class('_ambig', [self.node_builder(list(f[0])) for f in product(zip(*expand))])
+    if len(to_include) < len(expansion) or any(to_expand for i, to_expand in to_include):
+        return partial(ChildFilter, to_include)
 
-def maybe_create_ambiguous_expander(tree_class, expansion, keep_all_tokens):
-    to_expand = [i for i, sym in enumerate(expansion)
-                 if keep_all_tokens or ((not (sym.is_term and sym.filter_out)) and _should_expand(sym))]
-    if to_expand:
-        return partial(AmbiguousExpander, to_expand, tree_class)
 
-def ptb_inline_args(func):
-    @wraps(func)
-    def f(children):
-        return func(*children)
-    return f
+class Callback(object):
+    pass
 
 class ParseTreeBuilder:
-    def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False, maybe_placeholders=False):
+    def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False):
         self.tree_class = tree_class
         self.propagate_positions = propagate_positions
         self.always_keep_all_tokens = keep_all_tokens
-        self.ambiguous = ambiguous
-        self.maybe_placeholders = maybe_placeholders
 
         self.rule_builders = list(self._init_builders(rules))
 
+        self.user_aliases = {}
+
     def _init_builders(self, rules):
+        filter_out = {rule.origin for rule in rules if rule.options and rule.options.filter_out}
+        filter_out |= {sym for rule in rules for sym in rule.expansion if is_terminal(sym) and sym.startswith('_')}
+        assert all(x.startswith('_') for x in filter_out)
+
         for rule in rules:
             options = rule.options
             keep_all_tokens = self.always_keep_all_tokens or (options.keep_all_tokens if options else False)
             expand_single_child = options.expand1 if options else False
+            create_token = options.create_token if options else False
 
-            wrapper_chain = list(filter(None, [
+            wrapper_chain = filter(None, [
+                create_token and partial(CreateToken, create_token),
                 (expand_single_child and not rule.alias) and ExpandSingleChild,
-                maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous, options.empty_indices if self.maybe_placeholders and options else None),
+                maybe_create_child_filter(rule.expansion, () if keep_all_tokens else filter_out),
                 self.propagate_positions and PropagatePositions,
-                self.ambiguous and maybe_create_ambiguous_expander(self.tree_class, rule.expansion, keep_all_tokens),
-            ]))
+            ])
 
             yield rule, wrapper_chain
 
 
     def create_callback(self, transformer=None):
-        callbacks = {}
+        callback = Callback()
 
         for rule, wrapper_chain in self.rule_builders:
+            internal_callback_name = '_callback_%s_%s' % (rule.origin, '_'.join(rule.expansion))
 
-            user_callback_name = rule.alias or rule.origin.name
+            user_callback_name = rule.alias or rule.origin
             try:
-                f = getattr(transformer, user_callback_name)
-                assert not getattr(f, 'meta', False), "Meta args not supported for internal transformer"
-                # XXX InlineTransformer is deprecated!
-                if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer):
-                    f = ptb_inline_args(f)
+                f = transformer._get_func(user_callback_name)
             except AttributeError:
                 f = partial(self.tree_class, user_callback_name)
 
+            self.user_aliases[rule] = rule.alias
+            rule.alias = internal_callback_name
+
             for w in wrapper_chain:
                 f = w(f)
 
-            if rule in callbacks:
+            if hasattr(callback, internal_callback_name):
                 raise GrammarError("Rule '%s' already exists" % (rule,))
+            setattr(callback, internal_callback_name, f)
 
-            callbacks[rule] = f
-
-        return callbacks
-
+        return callback
 
-class LALR_Parser(object):
-    def __init__(self, parser_conf, debug=False):
-        assert all(r.options is None or r.options.priority is None
-                   for r in parser_conf.rules), "LALR doesn't yet support prioritization"
-        analysis = LALR_Analyzer(parser_conf, debug=debug)
-        analysis.compute_lookahead()
-        callbacks = parser_conf.callbacks
-
-        self._parse_table = analysis.parse_table
-        self.parser_conf = parser_conf
-        self.parser = _Parser(analysis.parse_table, callbacks)
-
-    @classmethod
-    def deserialize(cls, data, memo, callbacks):
-        inst = cls.__new__(cls)
-        inst.parser = _Parser(IntParseTable.deserialize(data, memo), callbacks)
-        return inst
-
-    def serialize(self, memo):
-        return self._parse_table.serialize(memo)
-
-    def parse(self, *args):
-        return self.parser.parse(*args)
 
 
 class _Parser:
@@ -1336,6 +551,7 @@ class _Parser:
         self.callbacks = callbacks
 
     def parse(self, seq, set_state=None):
+        i = 0
         token = None
         stream = iter(seq)
         states = self.states
@@ -1345,13 +561,14 @@ class _Parser:
 
         if set_state: set_state(self.start_state)
 
-        def get_action(token):
+        def get_action(key):
             state = state_stack[-1]
             try:
-                return states[state][token.type]
+                return states[state][key]
             except KeyError:
-                expected = [s for s in states[state].keys() if s.isupper()]
-                raise UnexpectedToken(token, expected, state=state)
+                expected = states[state].keys()
+
+                raise UnexpectedToken(token, expected, seq, i)
 
         def reduce(rule):
             size = len(rule.expansion)
@@ -1364,15 +581,15 @@ class _Parser:
 
             value = self.callbacks[rule](s)
 
-            _action, new_state = states[state_stack[-1]][rule.origin.name]
+            _action, new_state = get_action(rule.origin)
             assert _action is Shift
             state_stack.append(new_state)
             value_stack.append(value)
 
         # Main LALR-parser loop
-        for token in stream:
+        for i, token in enumerate(stream):
             while True:
-                action, arg = get_action(token)
+                action, arg = get_action(token.type)
                 assert arg != self.end_state
 
                 if action is Shift:
@@ -1383,9 +600,8 @@ class _Parser:
                 else:
                     reduce(arg)
 
-        token = Token.new_borrow_pos('$END', '', token) if token else Token('$END', '', 0, 1, 1)
         while True:
-            _action, arg = get_action(token)
+            _action, arg = get_action('$END')
             if _action is Shift:
                 assert arg == self.end_state
                 val ,= value_stack
@@ -1395,444 +611,169 @@ class _Parser:
 
 
 
-class Action:
-    def __init__(self, name):
-        self.name = name
-    def __str__(self):
-        return self.name
-    def __repr__(self):
-        return str(self)
-
-Shift = Action('Shift')
-Reduce = Action('Reduce')
-
-class ParseTable:
-    def __init__(self, states, start_state, end_state):
-        self.states = states
-        self.start_state = start_state
-        self.end_state = end_state
-
-    def serialize(self, memo):
-        tokens = Enumerator()
-        rules = Enumerator()
-
-        states = {
-            state: {tokens.get(token): ((1, arg.serialize(memo)) if action is Reduce else (0, arg))
-                    for token, (action, arg) in actions.items()}
-            for state, actions in self.states.items()
-        }
-
-        return {
-            'tokens': tokens.reversed(),
-            'states': states,
-            'start_state': self.start_state,
-            'end_state': self.end_state,
-        }
-
-    @classmethod
-    def deserialize(cls, data, memo):
-        tokens = data['tokens']
-        states = {
-            state: {tokens[token]: ((Reduce, Rule.deserialize(arg, memo)) if action==1 else (Shift, arg))
-                    for token, (action, arg) in actions.items()}
-            for state, actions in data['states'].items()
-        }
-        return cls(states, data['start_state'], data['end_state'])
-
-
-class IntParseTable(ParseTable):
-
-    @classmethod
-    def from_ParseTable(cls, parse_table):
-        enum = list(parse_table.states)
-        state_to_idx = {s:i for i,s in enumerate(enum)}
-        int_states = {}
-
-        for s, la in parse_table.states.items():
-            la = {k:(v[0], state_to_idx[v[1]]) if v[0] is Shift else v
-                  for k,v in la.items()}
-            int_states[ state_to_idx[s] ] = la
-
-
-        start_state = state_to_idx[parse_table.start_state]
-        end_state = state_to_idx[parse_table.end_state]
-        return cls(int_states, start_state, end_state)
-
-
-
-def get_frontend(parser, lexer):
-    if parser=='lalr':
-        if lexer is None:
-            raise ValueError('The LALR parser requires use of a lexer')
-        elif lexer == 'standard':
-            return LALR_TraditionalLexer
-        elif lexer == 'contextual':
-            return LALR_ContextualLexer
-        elif issubclass(lexer, Lexer):
-            return partial(LALR_CustomLexer, lexer)
-        else:
-            raise ValueError('Unknown lexer: %s' % lexer)
-    elif parser=='earley':
-        if lexer=='standard':
-            return Earley
-        elif lexer=='dynamic':
-            return XEarley
-        elif lexer=='dynamic_complete':
-            return XEarley_CompleteLex
-        elif lexer=='contextual':
-            raise ValueError('The Earley parser does not support the contextual parser')
-        else:
-            raise ValueError('Unknown lexer: %s' % lexer)
-    elif parser == 'cyk':
-        if lexer == 'standard':
-            return CYK
-        else:
-            raise ValueError('CYK parser requires using standard parser.')
-    else:
-        raise ValueError('Unknown parser: %s' % parser)
-
-
-
-
-class WithLexer(Serialize):
-    lexer = None
-    parser = None
-    lexer_conf = None
-
-    __serialize_fields__ = 'parser', 'lexer'
-    __serialize_namespace__ = Rule, ContextualLexer, TraditionalLexer
-
-    @classmethod
-    def deserialize(cls, data, memo, callbacks, postlex):
-        inst = super(WithLexer, cls).deserialize(data, memo)
-        inst.postlex = postlex
-        inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks)
-        return inst
-    
-    def _serialize(self, data, memo):
-        data['parser'] = data['parser'].serialize(memo)
-
-    def init_traditional_lexer(self, lexer_conf):
-        self.lexer_conf = lexer_conf
-        self.lexer = TraditionalLexer(lexer_conf.tokens, ignore=lexer_conf.ignore, user_callbacks=lexer_conf.callbacks)
-        self.postlex = lexer_conf.postlex
-
-    def init_contextual_lexer(self, lexer_conf):
-        self.lexer_conf = lexer_conf
-        self.postlex = lexer_conf.postlex
-        states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()}
-        always_accept = self.postlex.always_accept if self.postlex else ()
-        self.lexer = ContextualLexer(lexer_conf.tokens, states,
-                                     ignore=lexer_conf.ignore,
-                                     always_accept=always_accept,
-                                     user_callbacks=lexer_conf.callbacks)
-
-    def lex(self, text):
-        stream = self.lexer.lex(text)
-        return self.postlex.process(stream) if self.postlex else stream
-
-    def parse(self, text):
-        token_stream = self.lex(text)
-        sps = self.lexer.set_parser_state
-        return self.parser.parse(token_stream, *[sps] if sps is not NotImplemented else [])
-
-
-class LALR_TraditionalLexer(WithLexer):
-    def __init__(self, lexer_conf, parser_conf, options=None):
-        debug = options.debug if options else False
-        self.parser = LALR_Parser(parser_conf, debug=debug)
-        self.init_traditional_lexer(lexer_conf)
-
-class LALR_ContextualLexer(WithLexer):
-    def __init__(self, lexer_conf, parser_conf, options=None):
-        debug = options.debug if options else False
-        self.parser = LALR_Parser(parser_conf, debug=debug)
-        self.init_contextual_lexer(lexer_conf)
-
-
-
-class LarkOptions(Serialize):
-    """Specifies the options for Lark
-
+class Rule(object):
     """
-    OPTIONS_DOC = """
-        parser - Decides which parser engine to use, "earley" or "lalr". (Default: "earley")
-                 Note: "lalr" requires a lexer
-
-        lexer - Decides whether or not to use a lexer stage
-            "standard": Use a standard lexer
-            "contextual": Stronger lexer (only works with parser="lalr")
-            "dynamic": Flexible and powerful (only with parser="earley")
-            "dynamic_complete": Same as dynamic, but tries *every* variation
-                                of tokenizing possible. (only with parser="earley")
-            "auto" (default): Choose for me based on grammar and parser
-
-        ambiguity - Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
-            "resolve": The parser will automatically choose the simplest derivation
-                       (it chooses consistently: greedy for tokens, non-greedy for rules)
-            "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
-
-        transformer - Applies the transformer to every parse tree
-        debug - Affects verbosity (default: False)
-        keep_all_tokens - Don't automagically remove "punctuation" tokens (default: False)
-        cache_grammar - Cache the Lark grammar (Default: False)
-        postlex - Lexer post-processing (Default: None) Only works with the standard and contextual lexers.
-        start - The start symbol (Default: start)
-        profile - Measure run-time usage in Lark. Read results from the profiler proprety (Default: False)
-        priority - How priorities should be evaluated - auto, none, normal, invert (Default: auto)
-        propagate_positions - Propagates [line, column, end_line, end_column] attributes into all tree branches.
-        lexer_callbacks - Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
-        maybe_placeholders - Experimental feature. Instead of omitting optional rules (i.e. rule?), replace them with None
+        origin : a symbol
+        expansion : a list of symbols
     """
-    if __doc__:
-        __doc__ += OPTIONS_DOC
-
-    _defaults = {
-        'debug': False,
-        'keep_all_tokens': False,
-        'tree_class': None,
-        'cache_grammar': False,
-        'postlex': None,
-        'parser': 'earley',
-        'lexer': 'auto',
-        'transformer': None,
-        'start': 'start',
-        'profile': False,
-        'priority': 'auto',
-        'ambiguity': 'auto',
-        'propagate_positions': False,
-        'lexer_callbacks': {},
-        'maybe_placeholders': False,
-    }
-
-    def __init__(self, options_dict):
-        o = dict(options_dict)
-
-        options = {}
-        for name, default in self._defaults.items():
-            if name in o:
-                value = o.pop(name)
-                if isinstance(default, bool):
-                    value = bool(value)
-            else:
-                value = default
-
-            options[name] = value
-
-        self.__dict__['options'] = options
-
-        assert self.parser in ('earley', 'lalr', 'cyk', None)
-
-        if self.parser == 'earley' and self.transformer:
-            raise ValueError('Cannot specify an embedded transformer when using the Earley algorithm.'
-                             'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)')
-
-        if o:
-            raise ValueError("Unknown options: %s" % o.keys())
-
-    def __getattr__(self, name):
-        return self.options[name]
-    def __setattr__(self, name, value):
-        assert name in self.options
-        self.options[name] = value
-
-    def serialize(self, memo):
-        return self.options
-
-    @classmethod
-    def deserialize(cls, data, memo):
-        return cls(data)
-
-
-class Profiler:
-    def __init__(self):
-        self.total_time = defaultdict(float)
-        self.cur_section = '__init__'
-        self.last_enter_time = time.time()
-
-    def enter_section(self, name):
-        cur_time = time.time()
-        self.total_time[self.cur_section] += cur_time - self.last_enter_time
-        self.last_enter_time = cur_time
-        self.cur_section = name
-
-    def make_wrapper(self, name, f):
-        def wrapper(*args, **kwargs):
-            last_section = self.cur_section
-            self.enter_section(name)
-            try:
-                return f(*args, **kwargs)
-            finally:
-                self.enter_section(last_section)
-
-        return wrapper
-
-
-class Lark(Serialize):
-    def __init__(self, grammar, **options):
-        """
-            grammar : a string or file-object containing the grammar spec (using Lark's ebnf syntax)
-            options : a dictionary controlling various aspects of Lark.
-        """
-        self.options = LarkOptions(options)
-
-        # Some, but not all file-like objects have a 'name' attribute
-        try:
-            self.source = grammar.name
-        except AttributeError:
-            self.source = '<string>'
-
-        # Drain file-like objects to get their contents
-        try:
-            read = grammar.read
-        except AttributeError:
-            pass
-        else:
-            grammar = read()
-
-        assert isinstance(grammar, STRING_TYPE)
-
-        if self.options.cache_grammar:
-            raise NotImplementedError("Not available yet")
-
-        assert not self.options.profile, "Feature temporarily disabled"
-        # self.profiler = Profiler() if self.options.profile else None
-
-        if self.options.lexer == 'auto':
-            if self.options.parser == 'lalr':
-                self.options.lexer = 'contextual'
-            elif self.options.parser == 'earley':
-                self.options.lexer = 'dynamic'
-            elif self.options.parser == 'cyk':
-                self.options.lexer = 'standard'
-            else:
-                assert False, self.options.parser
-        lexer = self.options.lexer
-        assert lexer in ('standard', 'contextual', 'dynamic', 'dynamic_complete') or issubclass(lexer, Lexer)
-
-        if self.options.ambiguity == 'auto':
-            if self.options.parser == 'earley':
-                self.options.ambiguity = 'resolve'
-        else:
-            disambig_parsers = ['earley', 'cyk']
-            assert self.options.parser in disambig_parsers, (
-                'Only %s supports disambiguation right now') % ', '.join(disambig_parsers)
-
-        if self.options.priority == 'auto':
-            if self.options.parser in ('earley', 'cyk', ):
-                self.options.priority = 'normal'
-            elif self.options.parser in ('lalr', ):
-                self.options.priority = None
-        elif self.options.priority in ('invert', 'normal'):
-            assert self.options.parser in ('earley', 'cyk'), "priorities are not supported for LALR at this time"
-
-        assert self.options.priority in ('auto', None, 'normal', 'invert'), 'invalid priority option specified: {}. options are auto, none, normal, invert.'.format(self.options.priority)
-        assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"'
-        assert self.options.ambiguity in ('resolve', 'explicit', 'auto', )
-
-        # Parse the grammar file and compose the grammars (TODO)
-        self.grammar = load_grammar(grammar, self.source)
-
-        # Compile the EBNF grammar into BNF
-        self.terminals, self.rules, self.ignore_tokens = self.grammar.compile()
-
-        # If the user asked to invert the priorities, negate them all here.
-        # This replaces the old 'resolve__antiscore_sum' option.
-        if self.options.priority == 'invert':
-            for rule in self.rules:
-                if rule.options and rule.options.priority is not None:
-                    rule.options.priority = -rule.options.priority
-        # Else, if the user asked to disable priorities, strip them from the
-        # rules. This allows the Earley parsers to skip an extra forest walk
-        # for improved performance, if you don't need them (or didn't specify any).
-        elif self.options.priority == None:
-            for rule in self.rules:
-                if rule.options and rule.options.priority is not None:
-                    rule.options.priority = None
-        self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, self.options.lexer_callbacks)
-
-        if self.options.parser:
-            self.parser = self._build_parser()
-        elif lexer:
-            self.lexer = self._build_lexer()
-
-    if __init__.__doc__:
-        __init__.__doc__ += "\nOPTIONS:" + LarkOptions.OPTIONS_DOC
-
-    __serialize_fields__ = 'parser', 'rules', 'options'
-
-    def _build_lexer(self):
-        return TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks)
-
-    def _prepare_callbacks(self):
-        self.parser_class = get_frontend(self.options.parser, self.options.lexer)
-        self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class or Tree, self.options.propagate_positions, self.options.keep_all_tokens, self.options.parser!='lalr' and self.options.ambiguity=='explicit', self.options.maybe_placeholders)
-        self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
-
-    def _build_parser(self):
-        self._prepare_callbacks()
-        parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
-        return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
-
-    @classmethod
-    def deserialize(cls, data, namespace, memo, transformer=None, postlex=None):
-        if memo:
-            memo = SerializeMemoizer.deserialize(memo, namespace, {})
-        inst = cls.__new__(cls)
-        options = dict(data['options'])
-        options['transformer'] = transformer
-        options['postlex'] = postlex
-        inst.options = LarkOptions.deserialize(options, memo)
-        inst.rules = [Rule.deserialize(r, memo) for r in data['rules']]
-        inst._prepare_callbacks()
-        inst.parser = inst.parser_class.deserialize(data['parser'], memo, inst._callbacks, inst.options.postlex)
-        return inst
-
-
-    @classmethod
-    def open(cls, grammar_filename, rel_to=None, **options):
-        """Create an instance of Lark with the grammar given by its filename
-
-        If rel_to is provided, the function will find the grammar filename in relation to it.
-
-        Example:
-
-            >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
-            Lark(...)
+    def __init__(self, origin, expansion, alias=None, options=None):
+        self.origin = origin
+        self.expansion = expansion
+        self.alias = alias
+        self.options = options
 
-        """
-        if rel_to:
-            basepath = os.path.dirname(rel_to)
-            grammar_filename = os.path.join(basepath, grammar_filename)
-        with open(grammar_filename, encoding='utf8') as f:
-            return cls(f, **options)
+    def __str__(self):
+        return '<%s : %s>' % (self.origin, ' '.join(map(str,self.expansion)))
 
     def __repr__(self):
-        return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source, self.options.parser, self.options.lexer)
+        return 'Rule(%r, %r, %r, %r)' % (self.origin, self.expansion, self.alias, self.options)
 
 
-    def lex(self, text):
-        "Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'"
-        if not hasattr(self, 'lexer'):
-            self.lexer = self._build_lexer()
-        stream = self.lexer.lex(text)
-        if self.options.postlex:
-            return self.options.postlex.process(stream)
-        return stream
+class RuleOptions:
+    def __init__(self, keep_all_tokens=False, expand1=False, create_token=None, filter_out=False, priority=None):
+        self.keep_all_tokens = keep_all_tokens
+        self.expand1 = expand1
+        self.create_token = create_token  # used for scanless postprocessing
+        self.priority = priority
 
-    def parse(self, text):
-        "Parse the given text, according to the options provided. Returns a tree, unless specified otherwise."
-        return self.parser.parse(text)
+        self.filter_out = filter_out        # remove this rule from the tree
+                                            # used for "token"-rules in scanless
 
+    def __repr__(self):
+        return 'RuleOptions(%r, %r, %r, %r, %r)' % (
+            self.keep_all_tokens,
+            self.expand1,
+            self.create_token,
+            self.priority,
+            self.filter_out
+        )
 
-DATA = (
-{'rules': [{'@': 27}, {'@': 31}, {'@': 26}, {'@': 13}, {'@': 24}, {'@': 18}, {'@': 16}, {'@': 23}, {'@': 21}, {'@': 17}, {'@': 28}, {'@': 30}, {'@': 25}, {'@': 29}, {'@': 20}, {'@': 22}, {'@': 15}, {'@': 19}, {'@': 12}, {'@': 14}], 'parser': {'parser': {'tokens': {0: 'COMMA', 1: 'RBRACE', 2: u'pair', 3: u'ESCAPED_STRING', 4: u'string', 5: 'COLON', 6: 'RSQB', 7: '$END', 8: 'LBRACE', 9: u'FALSE', 10: u'object', 11: u'SIGNED_NUMBER', 12: u'value', 13: 'LSQB', 14: u'NULL', 15: u'TRUE', 16: u'array', 17: '__anon_star_1', 18: '__anon_star_0', 19: 'start'}, 'states': {0: {0: (0, 1), 1: (0, 32)}, 1: {2: (0, 5), 3: (0, 21), 4: (0, 3)}, 2: {0: (1, {'@': 12}), 1: (1, {'@': 12})}, 3: {5: (0, 13)}, 4: {0: (1, {'@': 13}), 1: (1, {'@': 13}), 6: (1, {'@': 13}), 7: (1, {'@': 13})}, 5: {0: (1, {'@': 14}), 1: (1, {'@': 14})}, 6: {0: (1, {'@': 15}), 6: (1, {'@': 15})}, 7: {0: (1, {'@': 16}), 1: (1, {'@': 16}), 6: (1, {'@': 16}), 7: (1, {'@': 16})}, 8: {3: (0, 21), 4: (0, 4), 8: (0, 34), 9: (0, 7), 10: (0, 33), 11: (0, 25), 12: (0, 12), 13: (0, 14), 14: (0, 24), 15: (0, 11), 16: (0, 27)}, 9: {0: (1, {'@': 17}), 1: (1, {'@': 17}), 6: (1, {'@': 17}), 7: (1, {'@': 17})}, 10: {0: (0, 22), 17: (0, 0), 1: (0, 26)}, 11: {0: (1, {'@': 18}), 1: (1, {'@': 18}), 6: (1, {'@': 18}), 7: (1, {'@': 18})}, 12: {0: (1, {'@': 19}), 6: (1, {'@': 19})}, 13: {3: (0, 21), 4: (0, 4), 8: (0, 34), 9: (0, 7), 10: (0, 33), 11: (0, 25), 12: (0, 15), 13: (0, 14), 14: (0, 24), 15: (0, 11), 16: (0, 27)}, 14: {3: (0, 21), 4: (0, 4), 6: (0, 30), 8: (0, 34), 9: (0, 7), 10: (0, 33), 11: (0, 25), 12: (0, 23), 13: (0, 14), 14: (0, 24), 15: (0, 11), 16: (0, 27)}, 15: {0: (1, {'@': 20}), 1: (1, {'@': 20})}, 16: {0: (1, {'@': 21}), 1: (1, {'@': 21}), 6: (1, {'@': 21}), 7: (1, {'@': 21})}, 17: {3: (0, 21), 4: (0, 4), 8: (0, 34), 9: (0, 7), 10: (0, 33), 11: (0, 25), 12: (0, 6), 13: (0, 14), 14: (0, 24), 15: (0, 11), 16: (0, 27)}, 18: {}, 19: {7: (0, 18)}, 20: {0: (0, 8), 6: (0, 16)}, 21: {0: (1, {'@': 22}), 1: (1, {'@': 22}), 5: (1, {'@': 22}), 6: (1, {'@': 22}), 7: (1, {'@': 22})}, 22: {2: (0, 2), 3: (0, 21), 4: (0, 3)}, 23: {0: (0, 17), 18: (0, 20), 6: (0, 9)}, 24: {0: (1, {'@': 23}), 1: (1, {'@': 23}), 6: (1, {'@': 23}), 7: (1, {'@': 23})}, 25: {0: (1, {'@': 24}), 1: (1, {'@': 24}), 6: (1, {'@': 24}), 7: (1, {'@': 24})}, 26: {0: (1, {'@': 25}), 1: (1, {'@': 25}), 6: (1, {'@': 25}), 7: (1, {'@': 25})}, 27: {0: (1, {'@': 26}), 1: (1, {'@': 26}), 6: (1, {'@': 26}), 7: (1, {'@': 26})}, 28: {3: (0, 21), 4: (0, 4), 8: (0, 34), 9: (0, 7), 10: (0, 33), 11: (0, 25), 12: (0, 29), 13: (0, 14), 14: (0, 24), 15: (0, 11), 16: (0, 27), 19: (0, 19)}, 29: {7: (1, {'@': 27})}, 30: {0: (1, {'@': 28}), 1: (1, {'@': 28}), 6: (1, {'@': 28}), 7: (1, {'@': 28})}, 31: {0: (1, {'@': 29}), 1: (1, {'@': 29}), 6: (1, {'@': 29}), 7: (1, {'@': 29})}, 32: {0: (1, {'@': 30}), 1: (1, {'@': 30}), 6: (1, {'@': 30}), 7: (1, {'@': 30})}, 33: {0: (1, {'@': 31}), 1: (1, {'@': 31}), 6: (1, {'@': 31}), 7: (1, {'@': 31})}, 34: {1: (0, 31), 2: (0, 10), 3: (0, 21), 4: (0, 3)}}, 'end_state': 18, 'start_state': 28}, '__type__': 'LALR_TraditionalLexer', 'lexer': {'ignore_types': [u'WS'], 'terminals': [{'@': 0}, {'@': 1}, {'@': 2}, {'@': 3}, {'@': 4}, {'@': 5}, {'@': 6}, {'@': 7}, {'@': 8}, {'@': 9}, {'@': 10}, {'@': 11}], '__type__': 'TraditionalLexer', 'newline_types': [u'WS']}}, '__type__': 'Lark', 'options': {'profile': False, 'transformer': None, 'lexer': 'standard', 'lexer_callbacks': {}, 'postlex': None, 'parser': 'lalr', 'cache_grammar': False, 'tree_class': None, 'priority': None, 'start': 'start', 'keep_all_tokens': False, 'ambiguity': 'auto', 'debug': False, 'propagate_positions': False, 'maybe_placeholders': False}}
-)
-MEMO = (
-{0: {'priority': 1, 'pattern': {'__type__': 'PatternRE', 'flags': [], 'value': u'(?:(?:\\+|\\-))?(?:(?:(?:[0-9])+(?:e|E)(?:(?:\\+|\\-))?(?:[0-9])+|(?:(?:[0-9])+\\.(?:(?:[0-9])+)?|\\.(?:[0-9])+)(?:(?:e|E)(?:(?:\\+|\\-))?(?:[0-9])+)?)|(?:[0-9])+)'}, '__type__': 'TerminalDef', 'name': u'SIGNED_NUMBER'}, 1: {'priority': 1, 'pattern': {'__type__': 'PatternRE', 'flags': [], 'value': u'\\".*?(?<!\\\\)(\\\\\\\\)*?\\"'}, '__type__': 'TerminalDef', 'name': u'ESCAPED_STRING'}, 2: {'priority': 1, 'pattern': {'__type__': 'PatternRE', 'flags': [], 'value': u'(?:[ \t\x0c\r\n])+'}, '__type__': 'TerminalDef', 'name': u'WS'}, 3: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u'false'}, '__type__': 'TerminalDef', 'name': u'FALSE'}, 4: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u'null'}, '__type__': 'TerminalDef', 'name': u'NULL'}, 5: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u'true'}, '__type__': 'TerminalDef', 'name': u'TRUE'}, 6: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u':'}, '__type__': 'TerminalDef', 'name': 'COLON'}, 7: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u','}, '__type__': 'TerminalDef', 'name': 'COMMA'}, 8: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u'{'}, '__type__': 'TerminalDef', 'name': 'LBRACE'}, 9: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u'['}, '__type__': 'TerminalDef', 'name': 'LSQB'}, 10: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u'}'}, '__type__': 'TerminalDef', 'name': 'RBRACE'}, 11: {'priority': 1, 'pattern': {'__type__': 'PatternStr', 'flags': [], 'value': u']'}, '__type__': 'TerminalDef', 'name': 'RSQB'}, 12: {'origin': {'__type__': 'NonTerminal', 'name': '__anon_star_1'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': 'COMMA'}, {'__type__': 'NonTerminal', 'name': u'pair'}], 'options': None, 'alias': None, 'order': 1}, 13: {'origin': {'__type__': 'NonTerminal', 'name': u'value'}, '__type__': 'Rule', 'expansion': [{'__type__': 'NonTerminal', 'name': u'string'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': True}, 'alias': None, 'order': 0}, 14: {'origin': {'__type__': 'NonTerminal', 'name': '__anon_star_1'}, '__type__': 'Rule', 'expansion': [{'__type__': 'NonTerminal', 'name': '__anon_star_1'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'COMMA'}, {'__type__': 'NonTerminal', 'name': u'pair'}], 'options': None, 'alias': None, 'order': 2}, 15: {'origin': {'__type__': 'NonTerminal', 'name': '__anon_star_0'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': 'COMMA'}, {'__type__': 'NonTerminal', 'name': u'value'}], 'options': None, 'alias': None, 'order': 1}, 16: {'origin': {'__type__': 'NonTerminal', 'name': u'value'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': u'FALSE'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': True}, 'alias': u'false', 'order': 0}, 17: {'origin': {'__type__': 'NonTerminal', 'name': u'array'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': 'LSQB'}, {'__type__': 'NonTerminal', 'name': u'value'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'RSQB'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': False}, 'alias': None, 'order': 2}, 18: {'origin': {'__type__': 'NonTerminal', 'name': u'value'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': u'TRUE'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': True}, 'alias': u'true', 'order': 0}, 19: {'origin': {'__type__': 'NonTerminal', 'name': '__anon_star_0'}, '__type__': 'Rule', 'expansion': [{'__type__': 'NonTerminal', 'name': '__anon_star_0'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'COMMA'}, {'__type__': 'NonTerminal', 'name': u'value'}], 'options': None, 'alias': None, 'order': 2}, 20: {'origin': {'__type__': 'NonTerminal', 'name': u'pair'}, '__type__': 'Rule', 'expansion': [{'__type__': 'NonTerminal', 'name': u'string'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'COLON'}, {'__type__': 'NonTerminal', 'name': u'value'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': False}, 'alias': None, 'order': 2}, 21: {'origin': {'__type__': 'NonTerminal', 'name': u'array'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': 'LSQB'}, {'__type__': 'NonTerminal', 'name': u'value'}, {'__type__': 'NonTerminal', 'name': '__anon_star_0'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'RSQB'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': False}, 'alias': None, 'order': 3}, 22: {'origin': {'__type__': 'NonTerminal', 'name': u'string'}, '__type__': 'Rule', 'expansion': [{'filter_out': False, '__type__': 'Terminal', 'name': u'ESCAPED_STRING'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': False}, 'alias': None, 'order': 0}, 23: {'origin': {'__type__': 'NonTerminal', 'name': u'value'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': u'NULL'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': True}, 'alias': u'null', 'order': 0}, 24: {'origin': {'__type__': 'NonTerminal', 'name': u'value'}, '__type__': 'Rule', 'expansion': [{'filter_out': False, '__type__': 'Terminal', 'name': u'SIGNED_NUMBER'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': True}, 'alias': u'number', 'order': 0}, 25: {'origin': {'__type__': 'NonTerminal', 'name': u'object'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': 'LBRACE'}, {'__type__': 'NonTerminal', 'name': u'pair'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'RBRACE'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': False}, 'alias': None, 'order': 2}, 26: {'origin': {'__type__': 'NonTerminal', 'name': u'value'}, '__type__': 'Rule', 'expansion': [{'__type__': 'NonTerminal', 'name': u'array'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': True}, 'alias': None, 'order': 0}, 27: {'origin': {'__type__': 'NonTerminal', 'name': u'start'}, '__type__': 'Rule', 'expansion': [{'__type__': 'NonTerminal', 'name': u'value'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': True}, 'alias': None, 'order': 0}, 28: {'origin': {'__type__': 'NonTerminal', 'name': u'array'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': 'LSQB'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'RSQB'}], 'options': {'priority': None, 'empty_indices': [False, True, False], 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': False}, 'alias': None, 'order': 2}, 29: {'origin': {'__type__': 'NonTerminal', 'name': u'object'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': 'LBRACE'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'RBRACE'}], 'options': {'priority': None, 'empty_indices': [False, True, False], 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': False}, 'alias': None, 'order': 2}, 30: {'origin': {'__type__': 'NonTerminal', 'name': u'object'}, '__type__': 'Rule', 'expansion': [{'filter_out': True, '__type__': 'Terminal', 'name': 'LBRACE'}, {'__type__': 'NonTerminal', 'name': u'pair'}, {'__type__': 'NonTerminal', 'name': '__anon_star_1'}, {'filter_out': True, '__type__': 'Terminal', 'name': 'RBRACE'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': False}, 'alias': None, 'order': 3}, 31: {'origin': {'__type__': 'NonTerminal', 'name': u'value'}, '__type__': 'Rule', 'expansion': [{'__type__': 'NonTerminal', 'name': u'object'}], 'options': {'priority': None, 'empty_indices': (), 'keep_all_tokens': False, '__type__': 'RuleOptions', 'expand1': True}, 'alias': None, 'order': 0}}
-)
 Shift = 0
 Reduce = 1
-def Lark_StandAlone(transformer=None, postlex=None):
-  namespace = {'Rule': Rule, 'TerminalDef': TerminalDef}
-  return Lark.deserialize(DATA, namespace, MEMO, transformer=transformer, postlex=postlex)
+import re
+MRES = (
+[(u'(?P<SIGNED_NUMBER>(?:(?:\\+|\\-))?(?:(?:(?:[0-9])+(?:e|E)(?:(?:\\+|\\-))?(?:[0-9])+|(?:(?:[0-9])+\\.(?:(?:[0-9])+)?|\\.(?:[0-9])+)(?:(?:e|E)(?:(?:\\+|\\-))?(?:[0-9])+)?)|(?:[0-9])+))|(?P<ESCAPED_STRING>\\"(?:(?:\\\\\\"|[^"]))*\\")|(?P<WS>(?:[ \t\x0c\r\n])+)|(?P<__FALSE1>false)|(?P<__NULL2>null)|(?P<__TRUE0>true)|(?P<__COLON>\\:)|(?P<__COMMA>\\,)|(?P<__LBRACE>\\{)|(?P<__LSQB>\\[)|(?P<__RBRACE>\\})|(?P<__RSQB>\\])',
+  {1: u'SIGNED_NUMBER',
+   2: u'ESCAPED_STRING',
+   3: u'WS',
+   4: u'__FALSE1',
+   5: u'__NULL2',
+   6: u'__TRUE0',
+   7: u'__COLON',
+   8: u'__COMMA',
+   9: u'__LBRACE',
+   10: u'__LSQB',
+   11: u'__RBRACE',
+   12: u'__RSQB'})]
+)
+LEXER_CALLBACK = (
+{}
+)
+NEWLINE_TYPES = [u'WS']
+IGNORE_TYPES = [u'WS']
+class LexerRegexps: pass
+lexer_regexps = LexerRegexps()
+lexer_regexps.mres = [(re.compile(p), d) for p, d in MRES]
+lexer_regexps.callback = {n: UnlessCallback([(re.compile(p), d) for p, d in mres])
+                          for n, mres in LEXER_CALLBACK.items()}
+lexer = _Lex(lexer_regexps)
+def lex(stream):
+    return lexer.lex(stream, NEWLINE_TYPES, IGNORE_TYPES)
+RULES = {
+  0: Rule(u'start', [u'value'], None, RuleOptions(False, True, None, None, False)),
+  1: Rule(u'value', [u'string'], None, RuleOptions(False, True, None, None, False)),
+  2: Rule(u'value', [u'__TRUE0'], u'true', RuleOptions(False, True, None, None, False)),
+  3: Rule(u'value', [u'array'], None, RuleOptions(False, True, None, None, False)),
+  4: Rule(u'value', [u'__NULL2'], u'null', RuleOptions(False, True, None, None, False)),
+  5: Rule(u'value', [u'SIGNED_NUMBER'], u'number', RuleOptions(False, True, None, None, False)),
+  6: Rule(u'value', [u'object'], None, RuleOptions(False, True, None, None, False)),
+  7: Rule(u'value', [u'__FALSE1'], u'false', RuleOptions(False, True, None, None, False)),
+  8: Rule(u'array', ['__LSQB', u'value', '__RSQB'], None, RuleOptions(False, False, None, None, False)),
+  9: Rule(u'array', ['__LSQB', u'value', '__anon_star_0', '__RSQB'], None, RuleOptions(False, False, None, None, False)),
+  10: Rule(u'array', ['__LSQB', '__RSQB'], None, RuleOptions(False, False, None, None, False)),
+  11: Rule(u'object', ['__LBRACE', u'pair', '__anon_star_1', '__RBRACE'], None, RuleOptions(False, False, None, None, False)),
+  12: Rule(u'object', ['__LBRACE', '__RBRACE'], None, RuleOptions(False, False, None, None, False)),
+  13: Rule(u'object', ['__LBRACE', u'pair', '__RBRACE'], None, RuleOptions(False, False, None, None, False)),
+  14: Rule(u'pair', [u'string', '__COLON', u'value'], None, RuleOptions(False, False, None, None, False)),
+  15: Rule(u'string', [u'ESCAPED_STRING'], None, RuleOptions(False, False, None, None, False)),
+  16: Rule('__anon_star_0', ['__anon_star_0', '__COMMA', u'value'], None, None),
+  17: Rule('__anon_star_0', ['__COMMA', u'value'], None, None),
+  18: Rule('__anon_star_1', ['__COMMA', u'pair'], None, None),
+  19: Rule('__anon_star_1', ['__anon_star_1', '__COMMA', u'pair'], None, None),
+}
+parse_tree_builder = ParseTreeBuilder(RULES.values(), Tree)
+class ParseTable: pass
+parse_table = ParseTable()
+STATES = {
+  0: {0: (1, 4), 1: (1, 4), 2: (1, 4), 3: (1, 4)},
+  1: {1: (1, 14), 2: (1, 14)},
+  2: {0: (0, 29), 1: (0, 32), 4: (0, 9)},
+  3: {1: (0, 13), 2: (0, 12)},
+  4: {0: (1, 1), 1: (1, 1), 2: (1, 1), 3: (1, 1)},
+  5: {0: (1, 10), 1: (1, 10), 2: (1, 10), 3: (1, 10)},
+  6: {2: (0, 15), 5: (0, 27), 6: (0, 16), 7: (0, 26)},
+  7: {5: (0, 34), 6: (0, 16), 7: (0, 26)},
+  8: {0: (1, 2), 1: (1, 2), 2: (1, 2), 3: (1, 2)},
+  9: {0: (0, 11), 1: (0, 22)},
+  10: {0: (1, 6), 1: (1, 6), 2: (1, 6), 3: (1, 6)},
+  11: {0: (1, 9), 1: (1, 9), 2: (1, 9), 3: (1, 9)},
+  12: {0: (1, 11), 1: (1, 11), 2: (1, 11), 3: (1, 11)},
+  13: {5: (0, 20), 6: (0, 16), 7: (0, 26)},
+  14: {6: (0, 16), 7: (0, 4), 8: (0, 6), 9: (0, 31), 10: (0, 24), 11: (0, 10), 12: (0, 21), 13: (0, 17), 14: (0, 33), 15: (0, 0), 16: (0, 19), 17: (0, 8)},
+  15: {0: (1, 12), 1: (1, 12), 2: (1, 12), 3: (1, 12)},
+  16: {0: (1, 15), 1: (1, 15), 2: (1, 15), 3: (1, 15), 18: (1, 15)},
+  17: {3: (1, 0)},
+  18: {},
+  19: {0: (1, 3), 1: (1, 3), 2: (1, 3), 3: (1, 3)},
+  20: {1: (1, 19), 2: (1, 19)},
+  21: {0: (1, 5), 1: (1, 5), 2: (1, 5), 3: (1, 5)},
+  22: {6: (0, 16), 7: (0, 4), 8: (0, 6), 9: (0, 31), 10: (0, 24), 11: (0, 10), 12: (0, 21), 13: (0, 30), 15: (0, 0), 16: (0, 19), 17: (0, 8)},
+  23: {6: (0, 16), 7: (0, 4), 8: (0, 6), 9: (0, 31), 10: (0, 24), 11: (0, 10), 12: (0, 21), 13: (0, 1), 15: (0, 0), 16: (0, 19), 17: (0, 8)},
+  24: {0: (0, 5), 6: (0, 16), 7: (0, 4), 8: (0, 6), 9: (0, 31), 10: (0, 24), 11: (0, 10), 12: (0, 21), 13: (0, 2), 15: (0, 0), 16: (0, 19), 17: (0, 8)},
+  25: {0: (1, 13), 1: (1, 13), 2: (1, 13), 3: (1, 13)},
+  26: {18: (0, 23)},
+  27: {1: (0, 7), 2: (0, 25), 19: (0, 3)},
+  28: {0: (1, 17), 1: (1, 17)},
+  29: {0: (1, 8), 1: (1, 8), 2: (1, 8), 3: (1, 8)},
+  30: {0: (1, 16), 1: (1, 16)},
+  31: {0: (1, 7), 1: (1, 7), 2: (1, 7), 3: (1, 7)},
+  32: {6: (0, 16), 7: (0, 4), 8: (0, 6), 9: (0, 31), 10: (0, 24), 11: (0, 10), 12: (0, 21), 13: (0, 28), 15: (0, 0), 16: (0, 19), 17: (0, 8)},
+  33: {3: (0, 18)},
+  34: {1: (1, 18), 2: (1, 18)},
+}
+TOKEN_TYPES = (
+{0: '__RSQB',
+ 1: '__COMMA',
+ 2: '__RBRACE',
+ 3: '$END',
+ 4: '__anon_star_0',
+ 5: u'pair',
+ 6: u'ESCAPED_STRING',
+ 7: u'string',
+ 8: '__LBRACE',
+ 9: u'__FALSE1',
+ 10: '__LSQB',
+ 11: u'object',
+ 12: u'SIGNED_NUMBER',
+ 13: u'value',
+ 14: 'start',
+ 15: u'__NULL2',
+ 16: u'array',
+ 17: u'__TRUE0',
+ 18: '__COLON',
+ 19: '__anon_star_1'}
+)
+parse_table.states = {s: {TOKEN_TYPES[t]: (a, RULES[x] if a is Reduce else x) for t, (a, x) in acts.items()}
+                      for s, acts in STATES.items()}
+parse_table.start_state = 14
+parse_table.end_state = 18
+class Lark_StandAlone:
+  def __init__(self, transformer=None, postlex=None):
+     callback = parse_tree_builder.create_callback(transformer=transformer)
+     callbacks = {rule: getattr(callback, rule.alias or rule.origin, None) for rule in RULES.values()}
+     self.parser = _Parser(parse_table, callbacks)
+     self.postlex = postlex
+  def parse(self, stream):
+     tokens = lex(stream)
+     if self.postlex: tokens = self.postlex.process(tokens)
+     return self.parser.parse(tokens)
diff --git a/lark/grammar.py b/lark/grammar.py
index 14893fb..e171d52 100644
--- a/lark/grammar.py
+++ b/lark/grammar.py
@@ -1,8 +1,4 @@
-from .utils import Serialize
-
-###{standalone
-
-class Symbol(Serialize):
+class Symbol(object):
     is_term = NotImplemented
 
     def __init__(self, name):
@@ -23,10 +19,7 @@ class Symbol(Serialize):
 
     fullrepr = property(__repr__)
 
-
 class Terminal(Symbol):
-    __serialize_fields__ = 'name', 'filter_out'
-
     is_term = True
 
     def __init__(self, name, filter_out=False):
@@ -38,42 +31,16 @@ class Terminal(Symbol):
         return '%s(%r, %r)' % (type(self).__name__, self.name, self.filter_out)
 
 
-
 class NonTerminal(Symbol):
-    __serialize_fields__ = 'name',
-
     is_term = False
 
-
-
-class RuleOptions(Serialize):
-    __serialize_fields__ = 'keep_all_tokens', 'expand1', 'priority', 'empty_indices'
-
-    def __init__(self, keep_all_tokens=False, expand1=False, priority=None, empty_indices=()):
-        self.keep_all_tokens = keep_all_tokens
-        self.expand1 = expand1
-        self.priority = priority
-        self.empty_indices = empty_indices
-
-    def __repr__(self):
-        return 'RuleOptions(%r, %r, %r)' % (
-            self.keep_all_tokens,
-            self.expand1,
-            self.priority,
-        )
-
-
-class Rule(Serialize):
+class Rule(object):
     """
         origin : a symbol
         expansion : a list of symbols
         order : index of this expansion amongst all rules of the same name
     """
     __slots__ = ('origin', 'expansion', 'alias', 'options', 'order', '_hash')
-
-    __serialize_fields__ = 'origin', 'expansion', 'order', 'alias', 'options'
-    __serialize_namespace__ = Terminal, NonTerminal, RuleOptions
-
     def __init__(self, origin, expansion, order=0, alias=None, options=None):
         self.origin = origin
         self.expansion = expansion
@@ -82,8 +49,6 @@ class Rule(Serialize):
         self.options = options
         self._hash = hash((self.origin, tuple(self.expansion)))
 
-    def _deserialize(self):
-        self._hash = hash((self.origin, tuple(self.expansion)))
 
     def __str__(self):
         return '<%s : %s>' % (self.origin.name, ' '.join(x.name for x in self.expansion))
@@ -100,5 +65,16 @@ class Rule(Serialize):
         return self.origin == other.origin and self.expansion == other.expansion
 
 
+class RuleOptions:
+    def __init__(self, keep_all_tokens=False, expand1=False, priority=None):
+        self.keep_all_tokens = keep_all_tokens
+        self.expand1 = expand1
+        self.priority = priority
+        self.empty_indices = ()
 
-###}
\ No newline at end of file
+    def __repr__(self):
+        return 'RuleOptions(%r, %r, %r)' % (
+            self.keep_all_tokens,
+            self.expand1,
+            self.priority,
+        )
diff --git a/lark/lark.py b/lark/lark.py
index daeed31..178141c 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -5,7 +5,7 @@ import time
 from collections import defaultdict
 from io import open
 
-from .utils import STRING_TYPE, Serialize, SerializeMemoizer
+from .utils import STRING_TYPE
 from .load_grammar import load_grammar
 from .tree import Tree
 from .common import LexerConf, ParserConf
@@ -13,11 +13,9 @@ from .common import LexerConf, ParserConf
 from .lexer import Lexer, TraditionalLexer
 from .parse_tree_builder import ParseTreeBuilder
 from .parser_frontends import get_frontend
-from .grammar import Rule
 
-###{standalone
 
-class LarkOptions(Serialize):
+class LarkOptions(object):
     """Specifies the options for Lark
 
     """
@@ -53,39 +51,24 @@ class LarkOptions(Serialize):
     if __doc__:
         __doc__ += OPTIONS_DOC
 
-    _defaults = {
-        'debug': False,
-        'keep_all_tokens': False,
-        'tree_class': None,
-        'cache_grammar': False,
-        'postlex': None,
-        'parser': 'earley',
-        'lexer': 'auto',
-        'transformer': None,
-        'start': 'start',
-        'profile': False,
-        'priority': 'auto',
-        'ambiguity': 'auto',
-        'propagate_positions': False,
-        'lexer_callbacks': {},
-        'maybe_placeholders': False,
-    }
-
     def __init__(self, options_dict):
         o = dict(options_dict)
 
-        options = {}
-        for name, default in self._defaults.items():
-            if name in o:
-                value = o.pop(name)
-                if isinstance(default, bool):
-                    value = bool(value)
-            else:
-                value = default
-
-            options[name] = value
-
-        self.__dict__['options'] = options
+        self.debug = bool(o.pop('debug', False))
+        self.keep_all_tokens = bool(o.pop('keep_all_tokens', False))
+        self.tree_class = o.pop('tree_class', Tree)
+        self.cache_grammar = o.pop('cache_grammar', False)
+        self.postlex = o.pop('postlex', None)
+        self.parser = o.pop('parser', 'earley')
+        self.lexer = o.pop('lexer', 'auto')
+        self.transformer = o.pop('transformer', None)
+        self.start = o.pop('start', 'start')
+        self.profile = o.pop('profile', False)
+        self.priority = o.pop('priority', 'auto')
+        self.ambiguity = o.pop('ambiguity', 'auto')
+        self.propagate_positions = o.pop('propagate_positions', False)
+        self.lexer_callbacks = o.pop('lexer_callbacks', {})
+        self.maybe_placeholders = o.pop('maybe_placeholders', False)
 
         assert self.parser in ('earley', 'lalr', 'cyk', None)
 
@@ -96,19 +79,6 @@ class LarkOptions(Serialize):
         if o:
             raise ValueError("Unknown options: %s" % o.keys())
 
-    def __getattr__(self, name):
-        return self.options[name]
-    def __setattr__(self, name, value):
-        assert name in self.options
-        self.options[name] = value
-
-    def serialize(self, memo):
-        return self.options
-
-    @classmethod
-    def deserialize(cls, data, memo):
-        return cls(data)
-
 
 class Profiler:
     def __init__(self):
@@ -134,7 +104,7 @@ class Profiler:
         return wrapper
 
 
-class Lark(Serialize):
+class Lark:
     def __init__(self, grammar, **options):
         """
             grammar : a string or file-object containing the grammar spec (using Lark's ebnf syntax)
@@ -225,35 +195,18 @@ class Lark(Serialize):
     if __init__.__doc__:
         __init__.__doc__ += "\nOPTIONS:" + LarkOptions.OPTIONS_DOC
 
-    __serialize_fields__ = 'parser', 'rules', 'options'
-
     def _build_lexer(self):
         return TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks)
 
-    def _prepare_callbacks(self):
+    def _build_parser(self):
         self.parser_class = get_frontend(self.options.parser, self.options.lexer)
-        self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class or Tree, self.options.propagate_positions, self.options.keep_all_tokens, self.options.parser!='lalr' and self.options.ambiguity=='explicit', self.options.maybe_placeholders)
-        self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
 
-    def _build_parser(self):
-        self._prepare_callbacks()
-        parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
-        return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
+        self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class, self.options.propagate_positions, self.options.keep_all_tokens, self.options.parser!='lalr' and self.options.ambiguity=='explicit', self.options.maybe_placeholders)
+        callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
 
-    @classmethod
-    def deserialize(cls, data, namespace, memo, transformer=None, postlex=None):
-        if memo:
-            memo = SerializeMemoizer.deserialize(memo, namespace, {})
-        inst = cls.__new__(cls)
-        options = dict(data['options'])
-        options['transformer'] = transformer
-        options['postlex'] = postlex
-        inst.options = LarkOptions.deserialize(options, memo)
-        inst.rules = [Rule.deserialize(r, memo) for r in data['rules']]
-        inst._prepare_callbacks()
-        inst.parser = inst.parser_class.deserialize(data['parser'], memo, inst._callbacks, inst.options.postlex)
-        return inst
+        parser_conf = ParserConf(self.rules, callbacks, self.options.start)
 
+        return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
 
     @classmethod
     def open(cls, grammar_filename, rel_to=None, **options):
@@ -290,4 +243,14 @@ class Lark(Serialize):
         "Parse the given text, according to the options provided. Returns a tree, unless specified otherwise."
         return self.parser.parse(text)
 
-###}
\ No newline at end of file
+        # if self.profiler:
+        #     self.profiler.enter_section('lex')
+        #     l = list(self.lex(text))
+        #     self.profiler.enter_section('parse')
+        #     try:
+        #         return self.parser.parse(l)
+        #     finally:
+        #         self.profiler.enter_section('outside_lark')
+        # else:
+        #     l = list(self.lex(text))
+        #     return self.parser.parse(l)
diff --git a/lark/lexer.py b/lark/lexer.py
index a3fb244..82a04e8 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -2,14 +2,10 @@
 
 import re
 
-from .utils import Str, classify, get_regexp_width, Py36, Serialize
+from .utils import Str, classify, get_regexp_width, Py36
 from .exceptions import UnexpectedCharacters, LexError
 
-###{standalone
-
-class Pattern(Serialize):
-    __serialize_fields__ = 'value', 'flags'
-
+class Pattern(object):
     def __init__(self, value, flags=()):
         self.value = value
         self.flags = frozenset(flags)
@@ -39,7 +35,6 @@ class Pattern(Serialize):
                 value = ('(?%s)' % f) + value
             return value
 
-
 class PatternStr(Pattern):
     def to_regexp(self):
         return self._get_flags(re.escape(self.value))
@@ -60,11 +55,7 @@ class PatternRE(Pattern):
     def max_width(self):
         return get_regexp_width(self.to_regexp())[1]
 
-
-class TerminalDef(Serialize):
-    __serialize_fields__ = 'name', 'pattern', 'priority'
-    __serialize_namespace__ = PatternStr, PatternRE
-
+class TerminalDef(object):
     def __init__(self, name, pattern, priority=1):
         assert isinstance(pattern, Pattern), pattern
         self.name = name
@@ -76,6 +67,7 @@ class TerminalDef(Serialize):
 
 
 
+###{standalone
 class Token(Str):
     __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column')
 
@@ -206,6 +198,7 @@ class CallChain:
         return self.callback2(t) if self.cond(t2) else t2
 
 
+###}
 
 
 
@@ -261,7 +254,7 @@ def _regexp_has_newline(r):
     """
     return '\n' in r or '\\n' in r or '[^' in r or ('(?s' in r and '.' in r)
 
-class Lexer(Serialize):
+class Lexer:
     """Lexer interface
 
     Method Signatures:
@@ -272,16 +265,7 @@ class Lexer(Serialize):
     set_parser_state = NotImplemented
     lex = NotImplemented
 
-
 class TraditionalLexer(Lexer):
-    __serialize_fields__ = 'terminals', 'ignore_types', 'newline_types'
-    __serialize_namespace__ = TerminalDef,
-
-    def _deserialize(self):
-        self.mres = build_mres(self.terminals)
-        self.callback = {}  # TODO implement
-
-
     def __init__(self, terminals, ignore=(), user_callbacks={}):
         assert all(isinstance(t, TerminalDef) for t in terminals), terminals
 
@@ -324,12 +308,7 @@ class TraditionalLexer(Lexer):
         return _Lex(self).lex(stream, self.newline_types, self.ignore_types)
 
 
-
-
 class ContextualLexer(Lexer):
-    __serialize_fields__ = 'root_lexer', 'lexers'
-    __serialize_namespace__ = TraditionalLexer,
-
     def __init__(self, terminals, states, ignore=(), always_accept=(), user_callbacks={}):
         tokens_by_name = {}
         for t in terminals:
@@ -364,4 +343,4 @@ class ContextualLexer(Lexer):
             l.lexer = self.lexers[self.parser_state]
             l.state = self.parser_state
 
-###}
\ No newline at end of file
+
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index b638a71..3608fcb 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -597,6 +597,8 @@ def import_from_grammar_into_namespace(grammar, namespace, aliases):
         try:
             return aliases[name].value
         except KeyError:
+            if name[0] == '_':
+                return '_%s__%s' % (namespace, name[1:])
             return '%s__%s' % (namespace, name)
 
     to_import = list(bfs(aliases, rule_dependencies))
diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py
index 977c371..ca12d5f 100644
--- a/lark/parse_tree_builder.py
+++ b/lark/parse_tree_builder.py
@@ -209,12 +209,12 @@ class ParseTreeBuilder:
             keep_all_tokens = self.always_keep_all_tokens or (options.keep_all_tokens if options else False)
             expand_single_child = options.expand1 if options else False
 
-            wrapper_chain = list(filter(None, [
+            wrapper_chain = filter(None, [
                 (expand_single_child and not rule.alias) and ExpandSingleChild,
                 maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous, options.empty_indices if self.maybe_placeholders and options else None),
                 self.propagate_positions and PropagatePositions,
                 self.ambiguous and maybe_create_ambiguous_expander(self.tree_class, rule.expansion, keep_all_tokens),
-            ]))
+            ])
 
             yield rule, wrapper_chain
 
diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py
index 73c4611..c351ddc 100644
--- a/lark/parser_frontends.py
+++ b/lark/parser_frontends.py
@@ -1,78 +1,25 @@
 import re
 from functools import partial
 
-from .utils import get_regexp_width, Serialize
+from .utils import get_regexp_width
 from .parsers.grammar_analysis import GrammarAnalyzer
 from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token
-from .parsers import earley, xearley, cyk
-from .parsers.lalr_parser import LALR_Parser
-from .grammar import Rule
+from .parsers import lalr_parser, earley, xearley, cyk
 from .tree import Tree
 
-###{standalone
-
-def get_frontend(parser, lexer):
-    if parser=='lalr':
-        if lexer is None:
-            raise ValueError('The LALR parser requires use of a lexer')
-        elif lexer == 'standard':
-            return LALR_TraditionalLexer
-        elif lexer == 'contextual':
-            return LALR_ContextualLexer
-        elif issubclass(lexer, Lexer):
-            return partial(LALR_CustomLexer, lexer)
-        else:
-            raise ValueError('Unknown lexer: %s' % lexer)
-    elif parser=='earley':
-        if lexer=='standard':
-            return Earley
-        elif lexer=='dynamic':
-            return XEarley
-        elif lexer=='dynamic_complete':
-            return XEarley_CompleteLex
-        elif lexer=='contextual':
-            raise ValueError('The Earley parser does not support the contextual parser')
-        else:
-            raise ValueError('Unknown lexer: %s' % lexer)
-    elif parser == 'cyk':
-        if lexer == 'standard':
-            return CYK
-        else:
-            raise ValueError('CYK parser requires using standard parser.')
-    else:
-        raise ValueError('Unknown parser: %s' % parser)
-
-
-
-
-class WithLexer(Serialize):
+class WithLexer:
     lexer = None
     parser = None
     lexer_conf = None
 
-    __serialize_fields__ = 'parser', 'lexer'
-    __serialize_namespace__ = Rule, ContextualLexer, TraditionalLexer
-
-    @classmethod
-    def deserialize(cls, data, memo, callbacks, postlex):
-        inst = super(WithLexer, cls).deserialize(data, memo)
-        inst.postlex = postlex
-        inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks)
-        return inst
-    
-    def _serialize(self, data, memo):
-        data['parser'] = data['parser'].serialize(memo)
-
     def init_traditional_lexer(self, lexer_conf):
         self.lexer_conf = lexer_conf
         self.lexer = TraditionalLexer(lexer_conf.tokens, ignore=lexer_conf.ignore, user_callbacks=lexer_conf.callbacks)
-        self.postlex = lexer_conf.postlex
 
     def init_contextual_lexer(self, lexer_conf):
         self.lexer_conf = lexer_conf
-        self.postlex = lexer_conf.postlex
         states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()}
-        always_accept = self.postlex.always_accept if self.postlex else ()
+        always_accept = lexer_conf.postlex.always_accept if lexer_conf.postlex else ()
         self.lexer = ContextualLexer(lexer_conf.tokens, states,
                                      ignore=lexer_conf.ignore,
                                      always_accept=always_accept,
@@ -80,31 +27,30 @@ class WithLexer(Serialize):
 
     def lex(self, text):
         stream = self.lexer.lex(text)
-        return self.postlex.process(stream) if self.postlex else stream
+        if self.lexer_conf.postlex:
+            return self.lexer_conf.postlex.process(stream)
+        return stream
 
     def parse(self, text):
         token_stream = self.lex(text)
         sps = self.lexer.set_parser_state
         return self.parser.parse(token_stream, *[sps] if sps is not NotImplemented else [])
 
-
 class LALR_TraditionalLexer(WithLexer):
     def __init__(self, lexer_conf, parser_conf, options=None):
         debug = options.debug if options else False
-        self.parser = LALR_Parser(parser_conf, debug=debug)
+        self.parser = lalr_parser.Parser(parser_conf, debug=debug)
         self.init_traditional_lexer(lexer_conf)
 
 class LALR_ContextualLexer(WithLexer):
     def __init__(self, lexer_conf, parser_conf, options=None):
         debug = options.debug if options else False
-        self.parser = LALR_Parser(parser_conf, debug=debug)
+        self.parser = lalr_parser.Parser(parser_conf, debug=debug)
         self.init_contextual_lexer(lexer_conf)
 
-###}
-
 class LALR_CustomLexer(WithLexer):
     def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None):
-        self.parser = LALR_Parser(parser_conf)
+        self.parser = lalr_parser.Parser(parser_conf)
         self.lexer_conf = lexer_conf
         self.lexer = lexer_cls(lexer_conf)
 
@@ -195,3 +141,37 @@ class CYK(WithLexer):
     def _apply_callback(self, tree):
         return self.callbacks[tree.rule](tree.children)
 
+
+def get_frontend(parser, lexer):
+    if parser=='lalr':
+        if lexer is None:
+            raise ValueError('The LALR parser requires use of a lexer')
+        elif lexer == 'standard':
+            return LALR_TraditionalLexer
+        elif lexer == 'contextual':
+            return LALR_ContextualLexer
+        elif issubclass(lexer, Lexer):
+            return partial(LALR_CustomLexer, lexer)
+        else:
+            raise ValueError('Unknown lexer: %s' % lexer)
+    elif parser=='earley':
+        if lexer=='standard':
+            return Earley
+        elif lexer=='dynamic':
+            return XEarley
+        elif lexer=='dynamic_complete':
+            return XEarley_CompleteLex
+        elif lexer=='contextual':
+            raise ValueError('The Earley parser does not support the contextual parser')
+        else:
+            raise ValueError('Unknown lexer: %s' % lexer)
+    elif parser == 'cyk':
+        if lexer == 'standard':
+            return CYK
+        else:
+            raise ValueError('CYK parser requires using standard parser.')
+    else:
+        raise ValueError('Unknown parser: %s' % parser)
+
+
+
diff --git a/lark/parsers/lalr_analysis.py b/lark/parsers/lalr_analysis.py
index cceaa45..950fb27 100644
--- a/lark/parsers/lalr_analysis.py
+++ b/lark/parsers/lalr_analysis.py
@@ -9,13 +9,10 @@ For now, shift/reduce conflicts are automatically resolved as shifts.
 import logging
 from collections import defaultdict
 
-from ..utils import classify, classify_bool, bfs, fzset, Serialize, Enumerator
+from ..utils import classify, classify_bool, bfs, fzset
 from ..exceptions import GrammarError
 
 from .grammar_analysis import GrammarAnalyzer, Terminal
-from ..grammar import Rule
-
-###{standalone
 
 class Action:
     def __init__(self, name):
@@ -34,34 +31,6 @@ class ParseTable:
         self.start_state = start_state
         self.end_state = end_state
 
-    def serialize(self, memo):
-        tokens = Enumerator()
-        rules = Enumerator()
-
-        states = {
-            state: {tokens.get(token): ((1, arg.serialize(memo)) if action is Reduce else (0, arg))
-                    for token, (action, arg) in actions.items()}
-            for state, actions in self.states.items()
-        }
-
-        return {
-            'tokens': tokens.reversed(),
-            'states': states,
-            'start_state': self.start_state,
-            'end_state': self.end_state,
-        }
-
-    @classmethod
-    def deserialize(cls, data, memo):
-        tokens = data['tokens']
-        states = {
-            state: {tokens[token]: ((Reduce, Rule.deserialize(arg, memo)) if action==1 else (Shift, arg))
-                    for token, (action, arg) in actions.items()}
-            for state, actions in data['states'].items()
-        }
-        return cls(states, data['start_state'], data['end_state'])
-
-
 class IntParseTable(ParseTable):
 
     @classmethod
@@ -80,7 +49,8 @@ class IntParseTable(ParseTable):
         end_state = state_to_idx[parse_table.end_state]
         return cls(int_states, start_state, end_state)
 
-###}
+
+
 
 class LALR_Analyzer(GrammarAnalyzer):
 
diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py
index 5510e3d..c30a92e 100644
--- a/lark/parsers/lalr_parser.py
+++ b/lark/parsers/lalr_parser.py
@@ -4,13 +4,10 @@
 # Email : erezshin@gmail.com
 from ..exceptions import UnexpectedToken
 from ..lexer import Token
-from ..utils import Enumerator, Serialize
 
-from .lalr_analysis import LALR_Analyzer, Shift, IntParseTable
+from .lalr_analysis import LALR_Analyzer, Shift
 
-
-###{standalone
-class LALR_Parser(object):
+class Parser:
     def __init__(self, parser_conf, debug=False):
         assert all(r.options is None or r.options.priority is None
                    for r in parser_conf.rules), "LALR doesn't yet support prioritization"
@@ -21,19 +18,9 @@ class LALR_Parser(object):
         self._parse_table = analysis.parse_table
         self.parser_conf = parser_conf
         self.parser = _Parser(analysis.parse_table, callbacks)
+        self.parse = self.parser.parse
 
-    @classmethod
-    def deserialize(cls, data, memo, callbacks):
-        inst = cls.__new__(cls)
-        inst.parser = _Parser(IntParseTable.deserialize(data, memo), callbacks)
-        return inst
-
-    def serialize(self, memo):
-        return self._parse_table.serialize(memo)
-
-    def parse(self, *args):
-        return self.parser.parse(*args)
-
+###{standalone
 
 class _Parser:
     def __init__(self, parse_table, callbacks):
diff --git a/lark/tools/standalone.py b/lark/tools/standalone.py
index ab334b5..3e6cc5d 100644
--- a/lark/tools/standalone.py
+++ b/lark/tools/standalone.py
@@ -36,7 +36,6 @@
 #
 ###}
 
-import pprint
 import codecs
 import sys
 import os
@@ -48,10 +47,6 @@ import lark
 from lark import Lark
 from lark.parsers.lalr_analysis import Reduce
 
-
-from lark.grammar import RuleOptions, Rule
-from lark.lexer import TerminalDef
-
 _dir = path.dirname(__file__)
 _larkdir = path.join(_dir, path.pardir)
 
@@ -63,15 +58,12 @@ EXTRACT_STANDALONE_FILES = [
     'tree.py',
     'visitors.py',
     'indenter.py',
-    'grammar.py',
     'lexer.py',
     'parse_tree_builder.py',
     'parsers/lalr_parser.py',
-    'parsers/lalr_analysis.py',
-    'parser_frontends.py',
-    'lark.py',
 ]
 
+
 def extract_sections(lines):
     section = None
     text = []
@@ -91,33 +83,152 @@ def extract_sections(lines):
 
     return {name:''.join(text) for name, text in sections.items()}
 
+def _prepare_mres(mres):
+    return [(p.pattern,{i: t for i, t in d.items()}) for p,d in mres]
+
+class TraditionalLexerAtoms:
+    def __init__(self, lexer):
+        self.mres = _prepare_mres(lexer.mres)
+        self.newline_types = lexer.newline_types
+        self.ignore_types = lexer.ignore_types
+        self.callback = {name:_prepare_mres(c.mres)
+                         for name, c in lexer.callback.items()}
+
+    def print_python(self):
+        print('import re')
+        print('class LexerRegexps: pass')
+        print('NEWLINE_TYPES = %s' % self.newline_types)
+        print('IGNORE_TYPES = %s' % self.ignore_types)
+        self._print_python('lexer')
+
+    def _print_python(self, var_name):
+        print('MRES = (')
+        pprint(self.mres)
+        print(')')
+        print('LEXER_CALLBACK = (')
+        pprint(self.callback)
+        print(')')
+        print('lexer_regexps = LexerRegexps()')
+        print('lexer_regexps.mres = [(re.compile(p), d) for p, d in MRES]')
+        print('lexer_regexps.callback = {n: UnlessCallback([(re.compile(p), d) for p, d in mres])')
+        print('                          for n, mres in LEXER_CALLBACK.items()}')
+        print('%s = (lexer_regexps)' % var_name)
+
+
+class ContextualLexerAtoms:
+    def __init__(self, lexer):
+        self.lexer_atoms = {state: TraditionalLexerAtoms(lexer) for state, lexer in lexer.lexers.items()}
+        self.root_lexer_atoms = TraditionalLexerAtoms(lexer.root_lexer)
+
+    def print_python(self):
+        print('import re')
+        print('class LexerRegexps: pass')
+        print('NEWLINE_TYPES = %s' % self.root_lexer_atoms.newline_types)
+        print('IGNORE_TYPES = %s' % self.root_lexer_atoms.ignore_types)
+
+        print('LEXERS = {}')
+        for state, lexer_atoms in self.lexer_atoms.items():
+            lexer_atoms._print_python('LEXERS[%d]' % state)
+
+        print('class ContextualLexer:')
+        print('    def __init__(self):')
+        print('        self.lexers = LEXERS')
+        print('        self.set_parser_state(None)')
+        print('    def set_parser_state(self, state):')
+        print('        self.parser_state = state')
+        print('    def lex(self, stream):')
+        print('        newline_types = NEWLINE_TYPES')
+        print('        ignore_types = IGNORE_TYPES')
+        print('        lexers = LEXERS')
+        print('        l = _Lex(lexers[self.parser_state], self.parser_state)')
+        print('        for x in l.lex(stream, newline_types, ignore_types):')
+        print('            yield x')
+        print('            l.lexer = lexers[self.parser_state]')
+        print('            l.state = self.parser_state')
+
+        print('CON_LEXER = ContextualLexer()')
+        print('def lex(stream):')
+        print('    return CON_LEXER.lex(stream)')
+
+class GetRule:
+    def __init__(self, rule_id):
+        self.rule_id = rule_id
+
+    def __repr__(self):
+        return 'RULES[%d]' % self.rule_id
+
+rule_ids = {}
+token_types = {}
+
+def _get_token_type(token_type):
+    if token_type not in token_types:
+        token_types[token_type] = len(token_types)
+    return token_types[token_type]
+
+class ParserAtoms:
+    def __init__(self, parser):
+        self.parse_table = parser._parse_table
+
+    def print_python(self):
+        print('class ParseTable: pass')
+        print('parse_table = ParseTable()')
+        print('STATES = {')
+        for state, actions in self.parse_table.states.items():
+            print('  %r: %r,' % (state, {_get_token_type(token): ((1, rule_ids[arg]) if action is Reduce else (0, arg))
+                            for token, (action, arg) in actions.items()}))
+        print('}')
+        print('TOKEN_TYPES = (')
+        pprint({v:k for k, v in token_types.items()})
+        print(')')
+        print('parse_table.states = {s: {TOKEN_TYPES[t]: (a, RULES[x] if a is Reduce else x) for t, (a, x) in acts.items()}')
+        print('                      for s, acts in STATES.items()}')
+        print('parse_table.start_state = %s' % self.parse_table.start_state)
+        print('parse_table.end_state = %s' % self.parse_table.end_state)
+        print('class Lark_StandAlone:')
+        print('  def __init__(self, transformer=None, postlex=None):')
+        print('     callbacks = parse_tree_builder.create_callback(transformer=transformer)')
+        print('     self.parser = _Parser(parse_table, callbacks)')
+        print('     self.postlex = postlex')
+        print('  def parse(self, stream):')
+        print('     tokens = lex(stream)')
+        print('     sps = CON_LEXER.set_parser_state')
+        print('     if self.postlex: tokens = self.postlex.process(tokens)')
+        print('     return self.parser.parse(tokens, sps)')
+
+class TreeBuilderAtoms:
+    def __init__(self, lark):
+        self.rules = lark.rules
+
+    def print_python(self):
+        # print('class InlineTransformer: pass')
+        print('RULES = {')
+        for i, r in enumerate(self.rules):
+            rule_ids[r] = i
+            print('  %d: Rule(%r, [%s], alias=%r, options=%r),' % (i, r.origin, ', '.join(s.fullrepr for s in r.expansion), r.alias, r.options ))
+        print('}')
+        print('parse_tree_builder = ParseTreeBuilder(RULES.values(), Tree)')
 
 def main(fobj, start):
     lark_inst = Lark(fobj, parser="lalr", lexer="contextual", start=start)
 
+    lexer_atoms = ContextualLexerAtoms(lark_inst.parser.lexer)
+    parser_atoms = ParserAtoms(lark_inst.parser.parser)
+    tree_builder_atoms = TreeBuilderAtoms(lark_inst)
+
     print('# The file was automatically generated by Lark v%s' % lark.__version__)
 
     for pyfile in EXTRACT_STANDALONE_FILES:
         with open(os.path.join(_larkdir, pyfile)) as f:
             print (extract_sections(f)['standalone'])
 
-    data, m = lark_inst.memo_serialize([TerminalDef, Rule])
-    print( 'DATA = (' )
-    # pprint(data, width=160)
-    print(data)
-    print(')')
-    print( 'MEMO = (')
-    print(m)
-    print(')')
-
+    with open(os.path.join(_larkdir, 'grammar.py')) as grammar_py:
+        print(grammar_py.read())
 
     print('Shift = 0')
     print('Reduce = 1')
-    print("def Lark_StandAlone(transformer=None, postlex=None):")
-    print("  namespace = {'Rule': Rule, 'TerminalDef': TerminalDef}")
-    print("  return Lark.deserialize(DATA, namespace, MEMO, transformer=transformer, postlex=postlex)")
-
-
+    lexer_atoms.print_python()
+    tree_builder_atoms.print_python()
+    parser_atoms.print_python()
 
 if __name__ == '__main__':
     if len(sys.argv) < 2:
diff --git a/lark/utils.py b/lark/utils.py
index cf1042b..ea1eb21 100644
--- a/lark/utils.py
+++ b/lark/utils.py
@@ -1,6 +1,8 @@
 import sys
 from collections import deque
 
+Py36 = (sys.version_info[:2] >= (3, 6))
+
 class fzset(frozenset):
     def __repr__(self):
         return '{%s}' % ', '.join(map(repr, self))
@@ -43,89 +45,7 @@ def bfs(initial, expand):
 
 
 
-def _serialize(value, memo):
-    # if memo and memo.in_types(value):
-    #     return {'__memo__': memo.memoized.get(value)}
-
-    if isinstance(value, Serialize):
-        return value.serialize(memo)
-    elif isinstance(value, list):
-        return [_serialize(elem, memo) for elem in value]
-    elif isinstance(value, frozenset):
-        return list(value)  # TODO reversible?
-    elif isinstance(value, dict):
-        return {key:_serialize(elem, memo) for key, elem in value.items()}
-    return value
-
 ###{standalone
-def _deserialize(data, namespace, memo):
-    if isinstance(data, dict):
-        if '__type__' in data: # Object
-            class_ = namespace[data['__type__']]
-            return class_.deserialize(data, memo)
-        elif '@' in data:
-            return memo[data['@']]
-        return {key:_deserialize(value, namespace, memo) for key, value in data.items()}
-    elif isinstance(data, list):
-        return [_deserialize(value, namespace, memo) for value in data]
-    return data
-
-
-class Serialize(object):
-    def memo_serialize(self, types_to_memoize):
-        memo = SerializeMemoizer(types_to_memoize)
-        return self.serialize(memo), memo.serialize()
-
-    def serialize(self, memo=None):
-        if memo and memo.in_types(self):
-            return {'@': memo.memoized.get(self)}
-
-        fields = getattr(self, '__serialize_fields__')
-        res = {f: _serialize(getattr(self, f), memo) for f in fields}
-        res['__type__'] = type(self).__name__
-        postprocess = getattr(self, '_serialize', None)
-        if postprocess:
-            postprocess(res, memo)
-        return res
-
-    @classmethod
-    def deserialize(cls, data, memo):
-        namespace = getattr(cls, '__serialize_namespace__', {})
-        namespace = {c.__name__:c for c in namespace}
-
-        fields = getattr(cls, '__serialize_fields__')
-
-        if '@' in data:
-            return memo[data['@']]
-
-        inst = cls.__new__(cls)
-        for f in fields:
-            setattr(inst, f, _deserialize(data[f], namespace, memo))
-        postprocess = getattr(inst, '_deserialize', None)
-        if postprocess:
-            postprocess()
-        return inst
-
-
-class SerializeMemoizer(Serialize):
-    __serialize_fields__ = 'memoized',
-
-    def __init__(self, types_to_memoize):
-        self.types_to_memoize = tuple(types_to_memoize)
-        self.memoized = Enumerator()
-
-    def in_types(self, value):
-        return isinstance(value, self.types_to_memoize)
-
-    def serialize(self):
-        return _serialize(self.memoized.reversed(), None)
-
-    @classmethod
-    def deserialize(cls, data, namespace, memo):
-        return _deserialize(data, namespace, memo)
-
-
-
 try:
     STRING_TYPE = basestring
 except NameError:   # Python 3
@@ -159,11 +79,6 @@ def smart_decorator(f, create_decorator):
     else:
         return create_decorator(f.__func__.__call__, True)
 
-import sys, re
-Py36 = (sys.version_info[:2] >= (3, 6))
-###}
-
-
 def dedup_list(l):
     """Given a list (l) will removing duplicates from the list,
        preserving the original order of the list. Assumes that
@@ -171,7 +86,7 @@ def dedup_list(l):
     dedup = set()
     return [ x for x in l if not (x in dedup or dedup.add(x))]
 
-
+###}
 
 
 try:
@@ -213,22 +128,3 @@ def get_regexp_width(regexp):
         return sre_parse.parse(regexp).getwidth()
     except sre_constants.error:
         raise ValueError(regexp)
-
-
-class Enumerator(Serialize):
-    def __init__(self):
-        self.enums = {}
-
-    def get(self, item):
-        if item not in self.enums:
-            self.enums[item] = len(self.enums)
-        return self.enums[item]
-
-    def __len__(self):
-        return len(self.enums)
-
-    def reversed(self):
-        r = {v: k for k, v in self.enums.items()}
-        assert len(r) == len(self.enums)
-        return r
-
diff --git a/lark/visitors.py b/lark/visitors.py
index 53847f9..71dfd42 100644
--- a/lark/visitors.py
+++ b/lark/visitors.py
@@ -72,9 +72,7 @@ class Transformer:
         assert mro[0] is cls
         libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)}
         for name, value in getmembers(cls):
-
-            # Make sure the function isn't inherited (unless it's overwritten)
-            if name.startswith('_') or (name in libmembers and name not in cls.__dict__):
+            if name.startswith('_') or name in libmembers:
                 continue
             if not callable(cls.__dict__[name]):
                 continue


Test patch:
diff --git a/tests/grammars/leading_underscore_grammar.lark b/tests/grammars/leading_underscore_grammar.lark
new file mode 100644
index 0000000..b09a2f4
--- /dev/null
+++ b/tests/grammars/leading_underscore_grammar.lark
@@ -0,0 +1,6 @@
+A: "A"
+
+_SEP: "x"
+_a: A
+
+c: _a _SEP
\ No newline at end of file
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 92cda02..92f2f06 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -21,8 +21,6 @@ from lark.lark import Lark
 from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters
 from lark.tree import Tree
 from lark.visitors import Transformer
-from lark.grammar import Rule
-from lark.lexer import TerminalDef
 
 __path__ = os.path.dirname(__file__)
 def _read(n, *args):
@@ -1101,6 +1099,11 @@ def _make_parser_test(LEXER, PARSER):
             x = l.parse('12 capybaras')
             self.assertEqual(x.children, ['12', 'capybaras'])
 
+        def test_relative_import_preserves_leading_underscore(self):
+            l = _Lark_open("test_relative_import_preserves_leading_underscore.lark", rel_to=__file__)
+            x = l.parse('Ax')
+            self.assertEqual(next(x.find_data('c')).children, ['A'])
+
         def test_import_errors(self):
             grammar = """
             start: NUMBER WORD
@@ -1431,23 +1434,6 @@ def _make_parser_test(LEXER, PARSER):
 
             parser.parse(r'"That" "And a \"b"')
 
-        @unittest.skipIf(PARSER!='lalr', "Serialize currently only works for LALR parsers (though it should be easy to extend)")
-        def test_serialize(self):
-            grammar = """
-                start: "A" b "C"
-                b: "B"
-            """
-            parser = _Lark(grammar)
-            d = parser.serialize()
-            parser2 = Lark.deserialize(d, {}, {})
-            self.assertEqual(parser2.parse('ABC'), Tree('start', [Tree('b', [])]) )
-
-            namespace = {'Rule': Rule, 'TerminalDef': TerminalDef}
-            d, m = parser.memo_serialize(namespace.values())
-            parser3 = Lark.deserialize(d, namespace, m)
-            self.assertEqual(parser3.parse('ABC'), Tree('start', [Tree('b', [])]) )
-
-
 
     _NAME = "Test" + PARSER.capitalize() + LEXER.capitalize()
     _TestParser.__name__ = _NAME
diff --git a/tests/test_relative_import_preserves_leading_underscore.lark b/tests/test_relative_import_preserves_leading_underscore.lark
new file mode 100644
index 0000000..92c08c6
--- /dev/null
+++ b/tests/test_relative_import_preserves_leading_underscore.lark
@@ -0,0 +1,3 @@
+start: c
+
+%import .grammars.leading_underscore_grammar.c
\ No newline at end of file
diff --git a/tests/test_tools.py b/tests/test_tools.py
index e1c49c4..5965788 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -70,10 +70,6 @@ class TestStandalone(TestCase):
         x = T().transform(x)
         self.assertEqual(x, ['a', 'b'])
 
-        l2 = _Lark(transformer=T())
-        x = l2.parse('ABAB')
-        self.assertEqual(x, ['a', 'b'])
-
 
 if __name__ == '__main__':
     unittest.main()