Repository: neogeny/TatSu

Issue:
@@keyword confusion
This here is valid:
```
@@namechars :: '_'
@@keyword :: if else not and or false true var let type
```

But if you swap those two lines:
```
@@keyword :: if else not and or false true var let type
@@namechars :: '_'
```

It gives the following error:
```
no available options :
@@namechars :: '_'
 ^
decorator
rule
grammar
start
```


Discussion:

Patch:
diff --git a/examples/g2e/python.ebnf b/examples/g2e/python.ebnf
index 46a8804..e69de29 100644
--- a/examples/g2e/python.ebnf
+++ b/examples/g2e/python.ebnf
@@ -1,741 +0,0 @@
-single_input
-    =
-    newline | simple_stmt | compound_stmt newline
-    ;
-
-
-file_input
-    =
-    {newline | stmt}
-    ;
-
-
-eval_input
-    =
-    {newline} testlist {newline}
-    ;
-
-
-decorators
-    =
-    {decorator}+
-    ;
-
-
-decorator
-    =
-    '@' dotted_attr ['(' [arglist] ')'] newline
-    ;
-
-
-dotted_attr
-    =
-    name {'.' name}
-    ;
-
-
-funcdef
-    =
-    [decorators] 'def' name parameters ':' suite
-    ;
-
-
-parameters
-    =
-    '(' [varargslist] ')'
-    ;
-
-
-varargslist
-    =
-        defparameter
-        {',' defparameter}
-        [','
-        [
-        '*' name [',' '**' name] | '**' name
-        ]
-        ]
-    |
-        '*' name [',' '**' name]
-    |
-        '**' name
-    ;
-
-
-defparameter
-    =
-    fpdef ['=' test]
-    ;
-
-
-fpdef
-    =
-    name | '(' fplist ')'
-    ;
-
-
-fplist
-    =
-    fpdef {',' fpdef} [',']
-    ;
-
-
-stmt
-    =
-    simple_stmt | compound_stmt
-    ;
-
-
-simple_stmt
-    =
-    small_stmt {';' small_stmt} [';'] newline
-    ;
-
-
-small_stmt
-    =
-    | expr_stmt
-    | print_stmt
-    | del_stmt
-    | pass_stmt
-    | flow_stmt
-    | import_stmt
-    | global_stmt
-    | exec_stmt
-    | assert_stmt
-    ;
-
-
-expr_stmt
-    =
-    testlist
-    [
-    augassign yield_expr | augassign testlist | assigns
-    ]
-    ;
-
-
-assigns
-    =
-    {assign_testlist}+ | {assign_yield}+
-    ;
-
-
-assign_testlist
-    =
-    '=' testlist
-    ;
-
-
-assign_yield
-    =
-    '=' yield_expr
-    ;
-
-
-augassign
-    =
-    | '+='
-    | '-='
-    | '*='
-    | '/='
-    | '%='
-    | '&='
-    | '|='
-    | '^='
-    | '<<='
-    | '>>='
-    | '**='
-    | '//='
-    ;
-
-
-print_stmt
-    =
-    'print'
-    [
-    printlist | '>>' printlist
-    ]
-    ;
-
-
-printlist
-    =
-    test {',' test} [',']
-    ;
-
-
-del_stmt
-    =
-    'del' exprlist
-    ;
-
-
-pass_stmt
-    =
-    'pass'
-    ;
-
-
-flow_stmt
-    =
-    break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
-    ;
-
-
-break_stmt
-    =
-    'break'
-    ;
-
-
-continue_stmt
-    =
-    'continue'
-    ;
-
-
-return_stmt
-    =
-    'return' [testlist]
-    ;
-
-
-yield_stmt
-    =
-    yield_expr
-    ;
-
-
-raise_stmt
-    =
-    'raise' [test [',' test [',' test]]]
-    ;
-
-
-import_stmt
-    =
-    import_name | import_from
-    ;
-
-
-import_name
-    =
-    'import' dotted_as_names
-    ;
-
-
-import_from
-    =
-    'from'
-    ({'.'} dotted_name | {'.'}+)
-    'import'
-    ('*' | import_as_names | '(' import_as_names ')')
-    ;
-
-
-import_as_names
-    =
-    import_as_name {',' import_as_name} [',']
-    ;
-
-
-import_as_name
-    =
-    name ['as' name]
-    ;
-
-
-dotted_as_name
-    =
-    dotted_name ['as' name]
-    ;
-
-
-dotted_as_names
-    =
-    dotted_as_name {',' dotted_as_name}
-    ;
-
-
-dotted_name
-    =
-    name {'.' name}
-    ;
-
-
-global_stmt
-    =
-    'global' name {',' name}
-    ;
-
-
-exec_stmt
-    =
-    'exec' expr ['in' test [',' test]]
-    ;
-
-
-assert_stmt
-    =
-    'assert' test [',' test]
-    ;
-
-
-compound_stmt
-    =
-    | if_stmt
-    | while_stmt
-    | for_stmt
-    | try_stmt
-    | with_stmt
-    | funcdef
-    | classdef
-    ;
-
-
-if_stmt
-    =
-    'if' test ':' suite {elif_clause} ['else' ':' suite]
-    ;
-
-
-elif_clause
-    =
-    'elif' test ':' suite
-    ;
-
-
-while_stmt
-    =
-    'while' test ':' suite ['else' ':' suite]
-    ;
-
-
-for_stmt
-    =
-    'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
-    ;
-
-
-try_stmt
-    =
-    'try'
-    ':'
-    suite
-    (
-        | {except_clause}+ ['else' ':' suite] ['finally' ':' suite]
-        | 'finally' ':' suite
-    )
-    ;
-
-
-with_stmt
-    =
-    'with' test [with_var] ':' suite
-    ;
-
-
-with_var
-    =
-    ('as' | name) expr
-    ;
-
-
-except_clause
-    =
-    'except' [test [',' test]] ':' suite
-    ;
-
-
-suite
-    =
-    simple_stmt | newline indent {stmt}+ dedent
-    ;
-
-
-indent
-    =
-    ()
-    ;
-
-
-dedent
-    =
-    ()
-    ;
-
-
-test
-    =
-    or_test ['if' or_test 'else' test] | lambdef
-    ;
-
-
-or_test
-    =
-    and_test {'or' and_test}
-    ;
-
-
-and_test
-    =
-    not_test {'and' not_test}
-    ;
-
-
-not_test
-    =
-    'not' not_test | comparison
-    ;
-
-
-comparison
-    =
-    expr {comp_op expr}
-    ;
-
-
-comp_op
-    =
-    | '<'
-    | '>'
-    | '=='
-    | '>='
-    | '<='
-    | '<>'
-    | '!='
-    | 'in'
-    | 'not' 'in'
-    | 'is'
-    | 'is' 'not'
-    ;
-
-
-expr
-    =
-    xor_expr {'|' xor_expr}
-    ;
-
-
-xor_expr
-    =
-    and_expr {'^' and_expr}
-    ;
-
-
-and_expr
-    =
-    shift_expr {'&' shift_expr}
-    ;
-
-
-shift_expr
-    =
-    arith_expr {('<<' | '>>') arith_expr}
-    ;
-
-
-arith_expr
-    =
-    term {('+' | '-') term}
-    ;
-
-
-term
-    =
-    factor {('*' | '/' | '%' | '//') factor}
-    ;
-
-
-factor
-    =
-    '+' factor | '-' factor | '~' factor | power
-    ;
-
-
-power
-    =
-    atom {trailer} ['**' factor]
-    ;
-
-
-atom
-    =
-        '('
-        [
-        yield_expr | testlist_gexp
-        ]
-
-        ')'
-    |
-        '[' [listmaker] ']'
-    |
-        '{' [dictmaker] '}'
-    |
-        '`' testlist '`'
-    |
-        name
-    |
-        int
-    |
-        LONGINT
-    |
-        float
-    |
-        complex
-    |
-        {STRING}+
-    ;
-
-
-listmaker
-    =
-    test (list_for | {',' test}) [',']
-    ;
-
-
-testlist_gexp
-    =
-    test ({',' test} [','] | gen_for)
-    ;
-
-
-lambdef
-    =
-    'lambda' [varargslist] ':' test
-    ;
-
-
-trailer
-    =
-    '(' [arglist] ')' | '[' subscriptlist ']' | '.' name
-    ;
-
-
-subscriptlist
-    =
-    subscript {',' subscript} [',']
-    ;
-
-
-subscript
-    =
-    '.' '.' '.' | test [':' [test] [sliceop]] | ':' [test] [sliceop]
-    ;
-
-
-sliceop
-    =
-    ':' [test]
-    ;
-
-
-exprlist
-    =
-    expr {',' expr} [',']
-    ;
-
-
-testlist
-    =
-    test {',' test} [',']
-    ;
-
-
-dictmaker
-    =
-    test ':' test {',' test ':' test} [',']
-    ;
-
-
-classdef
-    =
-    'class' name ['(' [testlist] ')'] ':' suite
-    ;
-
-
-arglist
-    =
-        argument
-        {',' argument}
-        [','
-        [
-        '*' test [',' '**' test] | '**' test
-        ]
-        ]
-    |
-        '*' test [',' '**' test]
-    |
-        '**' test
-    ;
-
-
-argument
-    =
-    test
-    [
-    ('=' test) | gen_for
-    ]
-    ;
-
-
-list_iter
-    =
-    list_for | list_if
-    ;
-
-
-list_for
-    =
-    'for' exprlist 'in' testlist [list_iter]
-    ;
-
-
-list_if
-    =
-    'if' test [list_iter]
-    ;
-
-
-gen_iter
-    =
-    gen_for | gen_if
-    ;
-
-
-gen_for
-    =
-    'for' exprlist 'in' or_test [gen_iter]
-    ;
-
-
-gen_if
-    =
-    'if' test [gen_iter]
-    ;
-
-
-yield_expr
-    =
-    'yield' [testlist]
-    ;
-
-
-float
-    =
-    | '.' DIGITS [EXPONENT]
-    | DIGITS '.' EXPONENT
-    | DIGITS ('.' [DIGITS [EXPONENT]] | EXPONENT)
-    ;
-
-
-LONGINT
-    =
-    int /[lL]/
-    ;
-
-
-EXPONENT
-    =
-    /[eE][+-]?/ DIGITS
-    ;
-
-
-int
-    =
-    '0' /[xX][0-9a-fA-F]+/ | '0' {DIGITS} | /[1-9]/ {DIGITS}
-    ;
-
-
-complex
-    =
-    int /[jJ]/ | float /[jJ]/
-    ;
-
-
-DIGITS
-    =
-    /[0-9]+/
-    ;
-
-
-name
-    =
-    /[a-zA-Z_][a-zA-Z_0-9]*/
-    ;
-
-
-STRING
-    =
-    [
-    /[ru]/ | 'ur'
-    ]
-
-    (
-        | "\\'\\'\\'" {TRIAPOS} "\\'\\'\\'"
-        | '"""' {TRIQUOTE} '"""'
-        | '"' {ESC | !/[\\\n"]/ /./} /["\']/ {ESC | !/[\\\n\']/ /./} "\\'"
-    )
-    ;
-
-
-TRIQUOTE
-    =
-    ['"'] ['"'] {ESC | !/[\\"]/ /./}+
-    ;
-
-
-TRIAPOS
-    =
-    ["\\'"] ["\\'"] {ESC | !/[\\\']/ /./}+
-    ;
-
-
-ESC
-    =
-    '\\\\' /\w+|\S+/
-    ;
-
-
-CONTINUED_LINE
-    =
-    '\\\\' ['\\r'] '\\n' /[ \t]*/ (nl:newline | ())
-    ;
-
-
-newline
-    =
-    {['\\u000C'] ['\\r'] '\\n'}+
-    ;
-
-
-ws
-    =
-    /[ \t\u000C]+/
-    ;
-
-
-leading_ws
-    =
-    (/[ \t]+/ | {' ' | '\\t'}+ {['\\r'] '\\n'})
-    ;
-
-
-comment
-    =
-    /[ \t]*/ '#' {!'\\n' /./} {'\\n'}+ | '#' {!'\\n' /./}
-    ;
-
-
-INDENT
-    =
-    !()
-    ;
-
-
-DEDENT
-    =
-    !()
-    ;
-
diff --git a/grammar/tatsu.ebnf b/grammar/tatsu.ebnf
index f017615..cc759e5 100644
--- a/grammar/tatsu.ebnf
+++ b/grammar/tatsu.ebnf
@@ -14,9 +14,17 @@ start
 grammar::Grammar
     =
     title:`TATSU`
-    directives:{directive}
-    keywords:keywords
-    rules:{rule}+
+    {
+        directives+:directive
+    |
+        keywords+:keyword
+    }
+    rules+:rule
+    {
+        rules+:rule
+    |
+        keywords+:keyword
+    }
     $
     ;
 
@@ -50,7 +58,13 @@ directive
 
 keywords
     =
-    {'@@keyword' ~ '::' ~ {@+:literal !(':'|'=')} }
+    { keywords }+
+    ;
+
+
+keyword
+    =
+    '@@keyword' ~ '::' ~ {@+:literal !(':'|'=')}
     ;
 
 
@@ -87,7 +101,7 @@ rule::Rule
 
 decorator
     =
-    '@' ~ @:('override'|'name')
+    '@' !'@' ~ @:('override'|'name')
     ;
 
 
diff --git a/tatsu/bootstrap.py b/tatsu/bootstrap.py
index 88103ff..87e6ebd 100644
--- a/tatsu/bootstrap.py
+++ b/tatsu/bootstrap.py
@@ -22,7 +22,9 @@ from tatsu.parsing import leftrec, nomemo  # noqa
 from tatsu.util import re, generic_main  # noqa
 
 
-KEYWORDS = {}  # type: ignore
+KEYWORDS = {
+    None,
+}  # type: ignore
 
 
 class EBNFBootstrapBuffer(Buffer):
@@ -89,21 +91,33 @@ class EBNFBootstrapParser(Parser):
         self._constant('TATSU')
         self.name_last_node('title')
 
-        def block2():
-            self._directive_()
-        self._closure(block2)
-        self.name_last_node('directives')
-        self._keywords_()
-        self.name_last_node('keywords')
-
-        def block5():
-            self._rule_()
-        self._positive_closure(block5)
-        self.name_last_node('rules')
+        def block1():
+            with self._choice():
+                with self._option():
+                    self._directive_()
+                    self.add_last_node_to_name('directives')
+                with self._option():
+                    self._keyword_()
+                    self.add_last_node_to_name('keywords')
+                self._error('no available options')
+        self._closure(block1)
+        self._rule_()
+        self.add_last_node_to_name('rules')
+
+        def block6():
+            with self._choice():
+                with self._option():
+                    self._rule_()
+                    self.add_last_node_to_name('rules')
+                with self._option():
+                    self._keyword_()
+                    self.add_last_node_to_name('keywords')
+                self._error('no available options')
+        self._closure(block6)
         self._check_eof()
         self.ast._define(
-            ['directives', 'keywords', 'rules', 'title'],
-            []
+            ['title'],
+            ['directives', 'keywords', 'rules']
         )
 
     @tatsumasu()
@@ -203,23 +217,27 @@ class EBNFBootstrapParser(Parser):
     def _keywords_(self):  # noqa
 
         def block0():
-            self._token('@@keyword')
-            self._cut()
-            self._token('::')
-            self._cut()
+            self._keywords_()
+        self._positive_closure(block0)
 
-            def block1():
-                self._literal_()
-                self.add_last_node_to_name('@')
-                with self._ifnot():
-                    with self._group():
-                        with self._choice():
-                            with self._option():
-                                self._token(':')
-                            with self._option():
-                                self._token('=')
-                            self._error('no available options')
-            self._closure(block1)
+    @tatsumasu()
+    def _keyword_(self):  # noqa
+        self._token('@@keyword')
+        self._cut()
+        self._token('::')
+        self._cut()
+
+        def block0():
+            self._literal_()
+            self.add_last_node_to_name('@')
+            with self._ifnot():
+                with self._group():
+                    with self._choice():
+                        with self._option():
+                            self._token(':')
+                        with self._option():
+                            self._token('=')
+                        self._error('no available options')
         self._closure(block0)
 
     @tatsumasu()
@@ -313,6 +331,8 @@ class EBNFBootstrapParser(Parser):
     @tatsumasu()
     def _decorator_(self):  # noqa
         self._token('@')
+        with self._ifnot():
+            self._token('@')
         self._cut()
         with self._group():
             with self._choice():
@@ -975,6 +995,9 @@ class EBNFBootstrapSemantics(object):
     def keywords(self, ast):  # noqa
         return ast
 
+    def keyword(self, ast):  # noqa
+        return ast
+
     def paramdef(self, ast):  # noqa
         return ast
 
diff --git a/tatsu/parser_semantics.py b/tatsu/parser_semantics.py
index f0a16ea..4e650c0 100644
--- a/tatsu/parser_semantics.py
+++ b/tatsu/parser_semantics.py
@@ -6,7 +6,7 @@ from collections import OrderedDict
 from tatsu import grammars
 from tatsu.exceptions import FailedSemantics
 from tatsu.semantics import ModelBuilderSemantics
-from tatsu.util import eval_escapes, re, warning
+from tatsu.util import eval_escapes, re, warning, flatten
 
 
 class EBNFGrammarSemantics(ModelBuilderSemantics):
@@ -119,8 +119,8 @@ class EBNFGrammarSemantics(ModelBuilderSemantics):
         return grammars.RuleInclude(rule)
 
     def grammar(self, ast, *args):
-        directives = OrderedDict((d.name, d.value) for d in ast.directives)
-        keywords = set(ast.keywords or [])
+        directives = OrderedDict((d.name, d.value) for d in flatten(ast.directives))
+        keywords = set(flatten(ast.keywords) or [])
 
         if directives.get('whitespace') in ('None', 'False'):
             directives['whitespace'] = ''
diff --git a/tatsu/util.py b/tatsu/util.py
index bd799c6..735ad42 100644
--- a/tatsu/util.py
+++ b/tatsu/util.py
@@ -130,6 +130,19 @@ def join_lists(lists):
     return sum(lists, [])
 
 
+def flatten(o):
+    def _flatten(x):
+        if not is_list(x):
+            yield x
+            return
+
+        for item in o:
+            for result in flatten(item):
+                yield result
+
+    return list(_flatten(o))
+
+
 def compress_seq(seq):
     seen = set()
     result = []


Test patch:
diff --git a/test/grammar/keyword_test.py b/test/grammar/keyword_test.py
index f964b9e..39d297b 100644
--- a/test/grammar/keyword_test.py
+++ b/test/grammar/keyword_test.py
@@ -103,3 +103,34 @@ class KeywordTests(unittest.TestCase):
         '''
         model = compile(grammar, 'test')
         model.parse("hello Øresund")
+
+    def test_sparse_keywords(self):
+        import parser
+
+        grammar = '''
+            @@keyword :: A
+
+            @@ignorecase :: False
+
+            start = {id}+ $ ;
+
+            @@keyword :: B
+
+            @name
+            id = /\w+/ ;
+        '''
+        model = compile(grammar, 'test', trace=True, colorize=True)
+        c = codegen(model)
+        parser.suite(c)
+
+        ast = model.parse('hello world')
+        self.assertEqual(['hello', 'world'], ast)
+
+        for k in ('A', 'B'):
+            try:
+                ast = model.parse("hello %s world" % k)
+                self.assertEqual(['hello', k, 'world'], ast)
+                self.fail('accepted keyword "%s" as name' % k)
+            except FailedParse as e:
+                self.assertTrue('"%s" is a reserved word' % k in str(e))
+                pass