Repository: googleapis/python-bigquery

Issue:
Simplify passing the arguments to OrderedDict in lexer
[Token definitions](https://github.com/googleapis/python-bigquery/blob/be4961257f077b96b595cfcd6553650bd4c618ad/google/cloud/bigquery/magics/line_arg_parser/lexer.py#L54) in the custom parser for bigquery cell magic are passed as a list of (key, value) pairs, which results in extra two levels of indentation after formatting and additional noise (parentheses, quotes...).

Since we don't support Python 3.5 and below anymore, we can define the tokens as kwargs to OrderedDict. The kwargs ordered in Python 3.6+ is preserved ([PEP 468](https://www.python.org/dev/peps/pep-0468/)).


Discussion:

Resolving patch:
diff --git a/google/cloud/bigquery/magics/line_arg_parser/lexer.py b/google/cloud/bigquery/magics/line_arg_parser/lexer.py
--- a/google/cloud/bigquery/magics/line_arg_parser/lexer.py
+++ b/google/cloud/bigquery/magics/line_arg_parser/lexer.py
@@ -49,90 +49,45 @@
 # the value of an option other than "--params", we do not really care about its
 # structure, and thus do not want to use any of the "Python tokens" for pattern matching.
 #
-# Since token definition order is important, an OrderedDict is needed with tightly
-# controlled member definitions (i.e. passed as a sequence, and *not* via kwargs).
+# Token definition order is important, thus an OrderedDict is used. In addition, PEP 468
+# guarantees us that the order of kwargs is preserved in Python 3.6+.
 token_types = OrderedDict(
-    [
-        (
-            "state_parse_pos_args",
-            OrderedDict(
-                [
-                    (
-                        "GOTO_PARSE_NON_PARAMS_OPTIONS",
-                        r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--))",  # double dash - starting the options list
-                    ),
-                    (
-                        "DEST_VAR",
-                        r"(?P<DEST_VAR>[^\d\W]\w*)",  # essentially a Python ID
-                    ),
-                ]
-            ),
-        ),
-        (
-            "state_parse_non_params_options",
-            OrderedDict(
-                [
-                    (
-                        "GOTO_PARSE_PARAMS_OPTION",
-                        r"(?P<GOTO_PARSE_PARAMS_OPTION>(?=--params(?:\s|=|--|$)))",  # the --params option
-                    ),
-                    ("OPTION_SPEC", r"(?P<OPTION_SPEC>--\w+)"),
-                    ("OPTION_EQ", r"(?P<OPTION_EQ>=)"),
-                    ("OPT_VAL", r"(?P<OPT_VAL>\S+?(?=\s|--|$))"),
-                ]
-            ),
-        ),
-        (
-            "state_parse_params_option",
-            OrderedDict(
-                [
-                    (
-                        "PY_STRING",
-                        r"(?P<PY_STRING>(?:{})|(?:{}))".format(
-                            r"'(?:[^'\\]|\.)*'",
-                            r'"(?:[^"\\]|\.)*"',  # single and double quoted strings
-                        ),
-                    ),
-                    ("PARAMS_OPT_SPEC", r"(?P<PARAMS_OPT_SPEC>--params(?=\s|=|--|$))"),
-                    ("PARAMS_OPT_EQ", r"(?P<PARAMS_OPT_EQ>=)"),
-                    (
-                        "GOTO_PARSE_NON_PARAMS_OPTIONS",
-                        r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--\w+))",  # found another option spec
-                    ),
-                    ("PY_BOOL", r"(?P<PY_BOOL>True|False)"),
-                    ("DOLLAR_PY_ID", r"(?P<DOLLAR_PY_ID>\$[^\d\W]\w*)"),
-                    (
-                        "PY_NUMBER",
-                        r"(?P<PY_NUMBER>-?[1-9]\d*(?:\.\d+)?(:?[e|E][+-]?\d+)?)",
-                    ),
-                    ("SQUOTE", r"(?P<SQUOTE>')"),
-                    ("DQUOTE", r'(?P<DQUOTE>")'),
-                    ("COLON", r"(?P<COLON>:)"),
-                    ("COMMA", r"(?P<COMMA>,)"),
-                    ("LCURL", r"(?P<LCURL>\{)"),
-                    ("RCURL", r"(?P<RCURL>})"),
-                    ("LSQUARE", r"(?P<LSQUARE>\[)"),
-                    ("RSQUARE", r"(?P<RSQUARE>])"),
-                    ("LPAREN", r"(?P<LPAREN>\()"),
-                    ("RPAREN", r"(?P<RPAREN>\))"),
-                ]
-            ),
-        ),
-        (
-            "common",
-            OrderedDict(
-                [
-                    ("WS", r"(?P<WS>\s+)"),
-                    ("EOL", r"(?P<EOL>$)"),
-                    (
-                        # anything not a whitespace or matched by something else
-                        "UNKNOWN",
-                        r"(?P<UNKNOWN>\S+)",
-                    ),
-                ]
-            ),
+    state_parse_pos_args=OrderedDict(
+        GOTO_PARSE_NON_PARAMS_OPTIONS=r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--))",  # double dash - starting the options list
+        DEST_VAR=r"(?P<DEST_VAR>[^\d\W]\w*)",  # essentially a Python ID
+    ),
+    state_parse_non_params_options=OrderedDict(
+        GOTO_PARSE_PARAMS_OPTION=r"(?P<GOTO_PARSE_PARAMS_OPTION>(?=--params(?:\s|=|--|$)))",  # the --params option
+        OPTION_SPEC=r"(?P<OPTION_SPEC>--\w+)",
+        OPTION_EQ=r"(?P<OPTION_EQ>=)",
+        OPT_VAL=r"(?P<OPT_VAL>\S+?(?=\s|--|$))",
+    ),
+    state_parse_params_option=OrderedDict(
+        PY_STRING=r"(?P<PY_STRING>(?:{})|(?:{}))".format(  # single and double quoted strings
+            r"'(?:[^'\\]|\.)*'", r'"(?:[^"\\]|\.)*"'
         ),
-    ]
+        PARAMS_OPT_SPEC=r"(?P<PARAMS_OPT_SPEC>--params(?=\s|=|--|$))",
+        PARAMS_OPT_EQ=r"(?P<PARAMS_OPT_EQ>=)",
+        GOTO_PARSE_NON_PARAMS_OPTIONS=r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--\w+))",  # found another option spec
+        PY_BOOL=r"(?P<PY_BOOL>True|False)",
+        DOLLAR_PY_ID=r"(?P<DOLLAR_PY_ID>\$[^\d\W]\w*)",
+        PY_NUMBER=r"(?P<PY_NUMBER>-?[1-9]\d*(?:\.\d+)?(:?[e|E][+-]?\d+)?)",
+        SQUOTE=r"(?P<SQUOTE>')",
+        DQUOTE=r'(?P<DQUOTE>")',
+        COLON=r"(?P<COLON>:)",
+        COMMA=r"(?P<COMMA>,)",
+        LCURL=r"(?P<LCURL>\{)",
+        RCURL=r"(?P<RCURL>})",
+        LSQUARE=r"(?P<LSQUARE>\[)",
+        RSQUARE=r"(?P<RSQUARE>])",
+        LPAREN=r"(?P<LPAREN>\()",
+        RPAREN=r"(?P<RPAREN>\))",
+    ),
+    common=OrderedDict(
+        WS=r"(?P<WS>\s+)",
+        EOL=r"(?P<EOL>$)",
+        UNKNOWN=r"(?P<UNKNOWN>\S+)",  # anything not a whitespace or matched by something else
+    ),
 )
 
 


Test patch: