Repository: lark-parser/lark
Pull request #963 (2021-08-17 11:19:39)

V1.0 merge master2
It seems I forgot to include the latest commits.


Patch:
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 792c6bd..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-dist: xenial
-language: python
-python:
-  - "3.4"
-  - "3.5"
-  - "3.6"
-  - "3.7"
-  - "3.8"
-  - "3.9-dev"
-  - "pypy3.5-6.0"
-install: pip install tox-travis
-script:
-  - tox
diff --git a/README.md b/README.md
index 82f6148..70be4fe 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ Most importantly, Lark will save you time and prevent you from getting parsing h
 
 Lark has no dependencies.
 
-[![Build Status](https://travis-ci.org/lark-parser/lark.svg?branch=master)](https://travis-ci.org/lark-parser/lark)
+[![Tests](https://github.com/lark-parser/lark/actions/workflows/tests.yml/badge.svg)](https://github.com/lark-parser/lark/actions/workflows/tests.yml)
 
 ### Syntax Highlighting
 
diff --git a/lark/lexer.py b/lark/lexer.py
index 7906a70..aa34a30 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -282,7 +282,6 @@ def _create_unless(terminals, g_regex_flags, re_, use_bytes):
     return new_terminals, callback
 
 
-
 class Scanner:
     def __init__(self, terminals, g_regex_flags, re_, use_bytes, match_whole=False):
         self.terminals = terminals
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index 3b46426..c0503e6 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -739,7 +739,10 @@ class Grammar:
                 else:
                     exp_options = options
 
-                assert all(isinstance(x, Symbol) for x in expansion), expansion
+                for sym in expansion:
+                    assert isinstance(sym, Symbol)
+                    if sym.is_term and exp_options and exp_options.keep_all_tokens:
+                        sym.filter_out = False
                 rule = Rule(NonTerminal(name), expansion, i, alias, exp_options)
                 compiled_rules.append(rule)
 
diff --git a/setup.cfg b/setup.cfg
index 6ddead9..6d71f28 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,6 +5,4 @@ zip_safe=
 universal = 1
 
 [metadata]
-description-file = README.md
 license_file = LICENSE
-
diff --git a/tox.ini b/tox.ini
index 33f9f64..04840eb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -2,13 +2,6 @@
 envlist = py36, py37, py38, py39, pypy, pypy3
 skip_missing_interpreters=true
 
-[travis]
-3.6 = py36
-3.7 = py37
-3.8 = py38
-3.9 = py39
-pypy3 = pypy3
-
 [testenv]
 whitelist_externals = git
 deps =


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a635e58..a98a00b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9.0-rc - 3.9, pypy3]
+        python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-rc - 3.10, pypy3]
 
     steps:
       - uses: actions/checkout@v2
diff --git a/tests/test_reconstructor.py b/tests/test_reconstructor.py
index f132312..e2f2dbe 100644
--- a/tests/test_reconstructor.py
+++ b/tests/test_reconstructor.py
@@ -3,6 +3,7 @@
 import json
 import sys
 import unittest
+from itertools import product
 from unittest import TestCase
 
 from lark import Lark
@@ -20,8 +21,8 @@ def _remove_ws(s):
 
 class TestReconstructor(TestCase):
 
-    def assert_reconstruct(self, grammar, code):
-        parser = Lark(grammar, parser='lalr', maybe_placeholders=False)
+    def assert_reconstruct(self, grammar, code, **options):
+        parser = Lark(grammar, parser='lalr', maybe_placeholders=False, **options)
         tree = parser.parse(code)
         new = Reconstructor(parser).reconstruct(tree)
         self.assertEqual(_remove_ws(code), _remove_ws(new))
@@ -142,6 +143,17 @@ class TestReconstructor(TestCase):
         new_json = Reconstructor(json_parser).reconstruct(tree)
         self.assertEqual(json.loads(new_json), json.loads(test_json))
 
+    def test_keep_all_tokens(self):
+        g = """
+        start: "a"? _B? c? _d?
+        _B: "b"
+        c: "c"
+        _d: "d"
+        """
+        examples = list(map(''.join, product(('', 'a'), ('', 'b'), ('', 'c'), ('', 'd'), )))
+        for code in examples:
+            self.assert_reconstruct(g, code, keep_all_tokens=True)
+
     @unittest.skipIf(sys.version_info < (3, 0), "Python 2 does not play well with Unicode.")
     def test_switch_grammar_unicode_terminal(self):
         """