Repository: lark-parser/lark
Pull request #711 (2020-10-05 21:38:36)

Allows the standalone parser to take more options at the command line
This adds most of the options that can be passed to a normal Lark instantiation to the standalone generator and the serializer.

Based on #710 and #707.

Note: breaks backwards compatibility by deprecating `python -m lark.tools.standalone grammar-file start` in favor of `python -m lark.tools.standalone -s start grammar-file`

Also adds an overview where new options to `Lark` need to (potentially) be added.

erezsh: Breaking compatibility isn't great!

Unless you want it to wait for v1.0
MegaIng: Well, it does have backwards compatibility. It isn't breaking. The only problem being that I use the `warn` function that outputs to stderr that the syntax is no longer allowed. This _should_ not mess with the normal usecase since only stdout _should_ be redirected. Therefor it _should_ not be breaking. If stderr also gets put in the file, python will simply not read it.
erezsh: Ah, ok. Well, you can output to stderr starting with `#` just to be safe.

Also, can you squash some of those commits?
MegaIng: Yeah, but then I can't use `warn`.

Also, squashing would be easier if #707 is merged first.
MegaIng: Btw, is there anywhere a list which breaking changes will be made with 1.0?
erezsh: #707 Merged.

I keep a list of v1.0 changes locally, but I'm not sure I'll do all of them:

- change "standard" lexer into "simple" or "basic"
- new grammar syntax?
- lexer_conf.tokens -> terminals
- unify position attribute names
- propagate positions on every node with a meta attribute?
    - tokens with meta attribute instead of direct position attributes? 


Patch:
diff --git a/lark/lark.py b/lark/lark.py
index 770b821..776469a 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -95,7 +95,16 @@ class LarkOptions(Serialize):
     """
     if __doc__:
         __doc__ += OPTIONS_DOC
-
+    
+    
+    # Adding a new option needs to be done in multiple places:
+    # - In the dictionary below. This is the primary truth which options `Lark.__init__` takes
+    # - In the doc string above. It is used both for the docstring of `LarkOptions` and `Lark`
+    # - In `lark-stubs/lark.pyi`:
+    #   - As attribute to `LarkOptions`
+    #   - As parameter to `Lark.__init__`
+    # - potentially in `_LOAD_ALLOWED_OPTIONS` below this class, when the option doesn't change how the grammar is loaded
+    # - potentially in `lark.tools.__init__`, when it can easily be passed as a cmd argument and makes sense
     _defaults = {
         'debug': False,
         'keep_all_tokens': False,
@@ -163,8 +172,9 @@ class LarkOptions(Serialize):
         return cls(data)
 
 
-_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'use_bytes', 'debug', 'g_regex_flags',
-                         'regex', 'propagate_positions', 'keep_all_tokens', 'tree_class'}
+# Options that can be passed to the Lark parser, even when it was loaded from cache/standalone.
+# These option are only used outside of `load_grammar`.
+_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
 
 
 class Lark(Serialize):
diff --git a/lark/tools/__init__.py b/lark/tools/__init__.py
index e69de29..4ecf13d 100644
--- a/lark/tools/__init__.py
+++ b/lark/tools/__init__.py
@@ -0,0 +1,65 @@
+import sys
+from argparse import ArgumentParser, FileType
+try:
+    from textwrap import indent
+except ImportError:
+    def indent(text, prefix):
+        return ''.join(prefix + line for line in text.splitlines(True))
+from logging import DEBUG, INFO, WARN, ERROR
+import warnings
+
+from lark import Lark, logger
+
+lalr_argparser = ArgumentParser(add_help=False, epilog='Look at the Lark documentation for more info on the options')
+
+flags = [
+    ('d', 'debug'),
+    'keep_all_tokens',
+    'regex',
+    'propagate_positions',
+    'maybe_placeholders',
+    'use_bytes'
+]
+
+options = ['start', 'lexer']
+
+lalr_argparser.add_argument('-v', '--verbose', action='count', default=0, help="Increase Logger output level, up to three times")
+lalr_argparser.add_argument('-s', '--start', action='append', default=[])
+lalr_argparser.add_argument('-l', '--lexer', default='contextual', choices=('standard', 'contextual'))
+k = {'encoding': 'utf-8'} if sys.version_info > (3, 4) else {}
+lalr_argparser.add_argument('-o', '--out', type=FileType('w', **k), default=sys.stdout, help='the output file (default=stdout)')
+lalr_argparser.add_argument('grammar_file', type=FileType('r', **k), help='A valid .lark file')
+
+for f in flags:
+    if isinstance(f, tuple):
+        options.append(f[1])
+        lalr_argparser.add_argument('-' + f[0], '--' + f[1], action='store_true')
+    else:
+        options.append(f)
+        lalr_argparser.add_argument('--' + f, action='store_true')
+
+
+def build_lalr(namespace):
+    logger.setLevel((ERROR, WARN, INFO, DEBUG)[min(namespace.verbose, 3)])
+    if len(namespace.start) == 0:
+        namespace.start.append('start')
+    kwargs = {n: getattr(namespace, n) for n in options}
+    return Lark(namespace.grammar_file, parser='lalr', **kwargs), namespace.out
+
+
+def showwarning_as_comment(message, category, filename, lineno, file=None, line=None):
+    # Based on warnings._showwarnmsg_impl
+    text = warnings.formatwarning(message, category, filename, lineno, line)
+    text = indent(text, '# ')
+    if file is None:
+        file = sys.stderr
+        if file is None:
+            return 
+    try:
+        file.write(text)
+    except OSError:
+        pass
+
+
+def make_warnings_comments():
+    warnings.showwarning = showwarning_as_comment
diff --git a/lark/tools/serialize.py b/lark/tools/serialize.py
index fb69d35..10884eb 100644
--- a/lark/tools/serialize.py
+++ b/lark/tools/serialize.py
@@ -5,20 +5,16 @@ import json
 from lark import Lark
 from lark.grammar import RuleOptions, Rule
 from lark.lexer import TerminalDef
+from lark.tools import lalr_argparser, build_lalr
 
 import argparse
 
-argparser = argparse.ArgumentParser(prog='python -m lark.tools.serialize') #description='''Lark Serialization Tool -- Stores Lark's internal state & LALR analysis as a convenient JSON file''')
+argparser = argparse.ArgumentParser(prog='python -m lark.tools.serialize', parents=[lalr_argparser],
+                                    description="Lark Serialization Tool - Stores Lark's internal state & LALR analysis as a JSON file",
+                                    epilog='Look at the Lark documentation for more info on the options')
 
-argparser.add_argument('grammar_file', type=argparse.FileType('r'), help='A valid .lark file')
-argparser.add_argument('-o', '--out', type=argparse.FileType('w'), default=sys.stdout, help='json file path to create (default=stdout)')
-argparser.add_argument('-s', '--start', default='start', help='start symbol (default="start")', nargs='+')
-argparser.add_argument('-l', '--lexer', default='standard', choices=['standard', 'contextual'], help='lexer type (default="standard")')
-
-
-def serialize(infile, outfile, lexer, start):
-    lark_inst = Lark(infile, parser="lalr", lexer=lexer, start=start)    # TODO contextual
 
+def serialize(lark_inst, outfile):
     data, memo = lark_inst.memo_serialize([TerminalDef, Rule])
     outfile.write('{\n')
     outfile.write('  "data": %s,\n' % json.dumps(data))
@@ -27,13 +23,9 @@ def serialize(infile, outfile, lexer, start):
 
 
 def main():
-    if len(sys.argv) == 1 or '-h' in sys.argv or '--help' in sys.argv:
-        print("Lark Serialization Tool - Stores Lark's internal state & LALR analysis as a JSON file")
-        print("")
-        argparser.print_help()
-    else:
-        args = argparser.parse_args()
-        serialize(args.grammar_file, args.out, args.lexer, args.start)
+    ns = argparser.parse_args()
+    serialize(*build_lalr(ns))
+
 
 if __name__ == '__main__':
-    main()
\ No newline at end of file
+    main()
diff --git a/lark/tools/standalone.py b/lark/tools/standalone.py
index f2af015..4ca57f6 100644
--- a/lark/tools/standalone.py
+++ b/lark/tools/standalone.py
@@ -26,22 +26,21 @@ from __future__ import print_function
 #
 #
 
-import os
 from io import open
 ###}
 
-import codecs
 import sys
 import token, tokenize
 import os
-from pprint import pprint
 from os import path
 from collections import defaultdict
 from functools import partial
+from argparse import ArgumentParser, SUPPRESS
+from warnings import warn
 
 import lark
 from lark import Lark
-from lark.parsers.lalr_analysis import Reduce
+from lark.tools import lalr_argparser, build_lalr, make_warnings_comments
 
 
 from lark.grammar import RuleOptions, Rule
@@ -120,48 +119,53 @@ def strip_docstrings(line_gen):
 
 
 def main(fobj, start, print=print):
+    warn('`lark.tools.standalone.main` is being redesigned. Use `gen_standalone`', DeprecationWarning)
     lark_inst = Lark(fobj, parser="lalr", lexer="contextual", start=start)
+    gen_standalone(lark_inst, print)
+
+def gen_standalone(lark_inst, output=None, out=sys.stdout):
+    if output is None:
+        output = partial(print, file=out)
 
-    print('# The file was automatically generated by Lark v%s' % lark.__version__)
-    print('__version__ = "%s"' % lark.__version__)
-    print()
+    output('# The file was automatically generated by Lark v%s' % lark.__version__)
+    output('__version__ = "%s"' % lark.__version__)
+    output()
 
     for i, pyfile in enumerate(EXTRACT_STANDALONE_FILES):
         with open(os.path.join(_larkdir, pyfile)) as f:
             code = extract_sections(f)['standalone']
             if i:   # if not this file
                 code = strip_docstrings(partial(next, iter(code.splitlines(True))))
-            print(code)
+            output(code)
 
     data, m = lark_inst.memo_serialize([TerminalDef, Rule])
-    print( 'DATA = (' )
+    output('DATA = (')
     # pprint(data, width=160)
-    print(data)
-    print(')')
-    print( 'MEMO = (')
-    print(m)
-    print(')')
+    output(data)
+    output(')')
+    output('MEMO = (')
+    output(m)
+    output(')')
+
 
+    output('Shift = 0')
+    output('Reduce = 1')
+    output("def Lark_StandAlone(**kwargs):")
+    output("  return Lark._load_from_dict(DATA, MEMO, **kwargs)")
 
-    print('Shift = 0')
-    print('Reduce = 1')
-    print("def Lark_StandAlone(**kwargs):")
-    print("  return Lark._load_from_dict(DATA, MEMO, **kwargs)")
 
 
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print("Lark Stand-alone Generator Tool")
-        print("Usage: python -m lark.tools.standalone <grammar-file> [<start>]")
-        sys.exit(1)
-
-    if len(sys.argv) == 3:
-        fn, start = sys.argv[1:]
-    elif len(sys.argv) == 2:
-        fn, start = sys.argv[1], 'start'
-    else:
-        assert False, sys.argv
-
-    with codecs.open(fn, encoding='utf8') as f:
-        main(f, start)
+    make_warnings_comments()
+    parser = ArgumentParser(prog="prog='python -m lark.tools.standalone'", description="Lark Stand-alone Generator Tool",
+                            parents=[lalr_argparser], epilog='Look at the Lark documentation for more info on the options')
+    parser.add_argument("old_start", nargs='?', help=SUPPRESS)
+    ns = parser.parse_args()
+    if ns.old_start is not None:
+        warn('The syntax `python -m lark.tools.standalone <grammar-file> <start>` is deprecated. Use the -s option')
+        ns.start.append(ns.old_start)
+    
+    lark_inst, out = build_lalr(ns)
+    gen_standalone(lark_inst, out=out)
+    


Test patch:
diff --git a/tests/test_tools.py b/tests/test_tools.py
index ce995d8..0972f8f 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -3,7 +3,7 @@ from __future__ import absolute_import, print_function
 import sys
 from unittest import TestCase, main
 
-from functools import partial
+from lark import Lark
 from lark.tree import Tree
 from lark.tools import standalone
 
@@ -21,8 +21,7 @@ class TestStandalone(TestCase):
 
     def _create_standalone(self, grammar):
         code_buf = StringIO()
-        pr = partial(print, file=code_buf)
-        standalone.main(StringIO(grammar), 'start', print=pr)
+        standalone.gen_standalone(Lark(grammar, parser='lalr'), out=code_buf)
         code = code_buf.getvalue()
 
         context = {'__doc__': None}