Repository: lark-parser/lark

Issue:
maybe_placeholders and anonymous terminal omits the None
Doc says [here](https://lark-parser.readthedocs.io/en/latest/grammar.html#rules) that `maybe_placeholders=True` makes ` rule : some1 [item] some2` to produce a `None` if item is not found, but anonymous token seems to not produce `None`.

Minimal reproduction: 
~~~python
from lark import Lark

grammar = r"""
UINT : /\d+/

math_atom : UINT 

LESS : "<"

math_script : math_atom ("_" ["<"] math_atom)*
"""

parser = Lark(
    grammar= grammar,
    parser="lalr",
    keep_all_tokens=True,
    maybe_placeholders=True,
    start="math_script"
     )

test_file= "1_1"
tree = parser.parse(test_file)
print(tree.pretty())
~~~

Produces: 
~~~
math_script
  math_atom     1
  _
  math_atom     1
~~~

If you replace `math_atom` to `UINT` it works. 
if you replace `["<"]` with `[LESS]` it works. 

~~~
math_script
  math_atom     1
  _
  None
  math_atom     1
~~~


I have both a global lark installed and a fresh environment with just lark


global install `Version: 0.9.0`
environment install  `Version: 0.10.0`


Discussion:

Patch:
diff --git a/lark/__init__.py b/lark/__init__.py
index 0e0ba4d..33a68cb 100644
--- a/lark/__init__.py
+++ b/lark/__init__.py
@@ -7,4 +7,4 @@ from .exceptions import (ParseError, LexError, GrammarError, UnexpectedToken,
 from .lexer import Token
 from .lark import Lark
 
-__version__ = "0.10.1"
+__version__ = "0.10.0"
diff --git a/lark/lark.py b/lark/lark.py
index e749138..72c8ad0 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -267,7 +267,7 @@ class Lark(Serialize):
         assert self.options.ambiguity in ('resolve', 'explicit', 'forest', 'auto', )
 
         # Parse the grammar file and compose the grammars (TODO)
-        self.grammar = load_grammar(grammar, self.source, re_module)
+        self.grammar = load_grammar(grammar, self.source, re_module, self.options.keep_all_tokens)
 
         # Compile the EBNF grammar into BNF
         self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start)
@@ -322,7 +322,7 @@ class Lark(Serialize):
         self._callbacks = None
         # we don't need these callbacks if we aren't building a tree
         if self.options.ambiguity != 'forest':
-            self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class or Tree, self.options.propagate_positions, self.options.keep_all_tokens, self.options.parser!='lalr' and self.options.ambiguity=='explicit', self.options.maybe_placeholders)
+            self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class or Tree, self.options.propagate_positions, self.options.parser!='lalr' and self.options.ambiguity=='explicit', self.options.maybe_placeholders)
             self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
 
     def _build_parser(self):
@@ -336,7 +336,7 @@ class Lark(Serialize):
         Useful for caching and multiprocessing.
         """
         data, m = self.memo_serialize([TerminalDef, Rule])
-        pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL)
+        pickle.dump({'data': data, 'memo': m}, f)
 
     @classmethod
     def load(cls, f):
@@ -373,8 +373,7 @@ class Lark(Serialize):
             self._callbacks,
             self.options.postlex,
             self.options.transformer,
-            re_module,
-            self.options.debug
+            re_module
         )
         self.terminals = self.parser.lexer_conf.tokens
         self._terminals_dict = {t.name: t for t in self.terminals}
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index cd36e4b..08990f2 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -650,7 +650,7 @@ class Grammar:
 
 
 _imported_grammars = {}
-def import_grammar(grammar_path, re_, base_paths=[]):
+def import_grammar(grammar_path, loader, base_paths=[]):
     if grammar_path not in _imported_grammars:
         import_paths = base_paths + IMPORT_PATHS
         for import_path in import_paths:
@@ -658,7 +658,7 @@ def import_grammar(grammar_path, re_, base_paths=[]):
                 joined_path = os.path.join(import_path, grammar_path)
                 with open(joined_path, encoding='utf8') as f:
                     text = f.read()
-                grammar = load_grammar(text, joined_path, re_)
+                grammar = loader.load_grammar(text, joined_path)
                 _imported_grammars[grammar_path] = grammar
                 break
         else:
@@ -803,7 +803,7 @@ class GrammarLoader:
         ('%ignore expects a value', ['%ignore %import\n']),
     ]
 
-    def __init__(self, re_module):
+    def __init__(self, re_module, always_keep_all_tokens):
         terminals = [TerminalDef(name, PatternRE(value)) for name, value in TERMINALS.items()]
 
         rules = [options_from_rule(name, None, x) for name, x in  RULES.items()]
@@ -816,6 +816,7 @@ class GrammarLoader:
 
         self.canonize_tree = CanonizeTree()
         self.re_module = re_module
+        self.always_keep_all_tokens = always_keep_all_tokens
 
     def load_grammar(self, grammar_text, grammar_name='<?>'):
         "Parse grammar_text, verify, and create Grammar object. Display nice messages on error."
@@ -901,7 +902,7 @@ class GrammarLoader:
         # import grammars
         for dotted_path, (base_paths, aliases) in imports.items():
             grammar_path = os.path.join(*dotted_path) + EXT
-            g = import_grammar(grammar_path, self.re_module, base_paths=base_paths)
+            g = import_grammar(grammar_path, self, base_paths=base_paths)
             new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases)
 
             term_defs += new_td
@@ -946,7 +947,11 @@ class GrammarLoader:
         rules = rule_defs
 
         rule_names = {}
-        for name, params, _x, _o in rules:
+        for name, params, _x, option in rules:
+            # We can't just simply not throw away the tokens later, we need option.keep_all_tokens to correctly generate maybe_placeholders
+            if self.always_keep_all_tokens: 
+                option.keep_all_tokens = True
+            
             if name.startswith('__'):
                 raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
             if name in rule_names:
@@ -981,5 +986,5 @@ class GrammarLoader:
 
 
 
-def load_grammar(grammar, source, re_):
-    return GrammarLoader(re_).load_grammar(grammar, source)
+def load_grammar(grammar, source, re_, always_keep_all_tokens):
+    return GrammarLoader(re_, always_keep_all_tokens).load_grammar(grammar, source)
diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py
index 8b81d29..a4c4330 100644
--- a/lark/parse_tree_builder.py
+++ b/lark/parse_tree_builder.py
@@ -299,10 +299,9 @@ def apply_visit_wrapper(func, name, wrapper):
 
 
 class ParseTreeBuilder:
-    def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False, maybe_placeholders=False):
+    def __init__(self, rules, tree_class, propagate_positions=False, ambiguous=False, maybe_placeholders=False):
         self.tree_class = tree_class
         self.propagate_positions = propagate_positions
-        self.always_keep_all_tokens = keep_all_tokens
         self.ambiguous = ambiguous
         self.maybe_placeholders = maybe_placeholders
 
@@ -311,7 +310,7 @@ class ParseTreeBuilder:
     def _init_builders(self, rules):
         for rule in rules:
             options = rule.options
-            keep_all_tokens = self.always_keep_all_tokens or options.keep_all_tokens
+            keep_all_tokens = options.keep_all_tokens
             expand_single_child = options.expand1
 
             wrapper_chain = list(filter(None, [
diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py
index d433d09..eb2b615 100644
--- a/lark/parser_frontends.py
+++ b/lark/parser_frontends.py
@@ -82,11 +82,11 @@ class WithLexer(_ParserFrontend):
         self.postlex = lexer_conf.postlex
 
     @classmethod
-    def deserialize(cls, data, memo, callbacks, postlex, transformer, re_module, debug):
+    def deserialize(cls, data, memo, callbacks, postlex, transformer, re_module):
         inst = super(WithLexer, cls).deserialize(data, memo)
 
         inst.postlex = postlex
-        inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks, debug)
+        inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks)
 
         terminals = [item for item in memo.values() if isinstance(item, TerminalDef)]
         inst.lexer_conf.callbacks = _get_lexer_callbacks(transformer, terminals)
diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py
index 433f3ef..cf6a4bf 100644
--- a/lark/parsers/lalr_parser.py
+++ b/lark/parsers/lalr_parser.py
@@ -23,10 +23,10 @@ class LALR_Parser(object):
         self.parser = _Parser(analysis.parse_table, callbacks, debug)
 
     @classmethod
-    def deserialize(cls, data, memo, callbacks, debug=False):
+    def deserialize(cls, data, memo, callbacks):
         inst = cls.__new__(cls)
         inst._parse_table = IntParseTable.deserialize(data, memo)
-        inst.parser = _Parser(inst._parse_table, callbacks, debug)
+        inst.parser = _Parser(inst._parse_table, callbacks)
         return inst
 
     def serialize(self, memo):


Test patch:
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 4a07d7a..ca4d781 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -86,12 +86,6 @@ class TestCache(TestCase):
             parser = Lark(g, parser='lalr', lexer=CustomLexer, cache=True)
             assert len(mock_fs.files) == 1
             assert parser.parse('a') == Tree('start', [])
-
-            # Test options persistence
-            mock_fs.files = {}
-            Lark(g, parser="lalr", debug=True, cache=True)
-            parser = Lark(g, parser="lalr", debug=True, cache=True)
-            assert parser.options.options['debug']
         finally:
             lark_module.FS = fs
 
@@ -99,3 +93,6 @@ class TestCache(TestCase):
 
 if __name__ == '__main__':
     main()
+
+
+
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 83336c5..d97c810 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -2041,6 +2041,10 @@ def _make_parser_test(LEXER, PARSER):
             # Anonymous tokens shouldn't count
             p = _Lark("""start: ["a"] ["b"] ["c"] """, maybe_placeholders=True)
             self.assertEqual(p.parse("").children, [])
+            
+            # Unless keep_all_tokens=True
+            p = _Lark("""start: ["a"] ["b"] ["c"] """, maybe_placeholders=True, keep_all_tokens=True)
+            self.assertEqual(p.parse("").children, [None, None, None])
 
             # All invisible constructs shouldn't count
             p = _Lark("""start: [A] ["b"] [_c] ["e" "f" _c]
diff --git a/tests/test_trees.py b/tests/test_trees.py
index 905ad5a..cca92f9 100644
--- a/tests/test_trees.py
+++ b/tests/test_trees.py
@@ -20,7 +20,7 @@ class TestTrees(TestCase):
 
     def test_pickle(self):
         s = copy.deepcopy(self.tree1)
-        data = pickle.dumps(s, protocol=pickle.HIGHEST_PROTOCOL)
+        data = pickle.dumps(s)
         assert pickle.loads(data) == s
 
     def test_repr_runnable(self):