Repository: facebookresearch/hydra

Issue:
command line is incorrectly removing whitespace in primitives
```commandline
$ python foo.py "+x=a b c"
```
results in:
```yaml
x: abc
```

because the grammar is ignoring whitespace outside of quoted strings and there is a concatenation phase for primitives.


Discussion:
cc @odelalleau 
I'll take it

Resolving patch:
diff --git a/build_helpers/build_helpers.py b/build_helpers/build_helpers.py
--- a/build_helpers/build_helpers.py
+++ b/build_helpers/build_helpers.py
@@ -178,7 +178,8 @@ def run(self) -> None:
         root_dir = abspath(dirname(__file__))
         project_root = abspath(dirname(basename(__file__)))
         for grammar in [
-            "hydra/grammar/Override.g4",
+            "hydra/grammar/OverrideLexer.g4",
+            "hydra/grammar/OverrideParser.g4",
         ]:
             command = [
                 "java",
diff --git a/hydra/core/override_parser/overrides_parser.py b/hydra/core/override_parser/overrides_parser.py
--- a/hydra/core/override_parser/overrides_parser.py
+++ b/hydra/core/override_parser/overrides_parser.py
@@ -28,6 +28,9 @@
     )
     sys.exit(1)
 
+# The set of parser rules that require the lexer to be in lexical mode `KEY`.
+KEY_RULES = {"key", "override", "package", "packageOrGroup"}
+
 
 class OverridesParser:
     functions: Functions
@@ -49,6 +52,11 @@ def parse_rule(self, s: str, rule_name: str) -> Any:
         lexer = OverrideLexer(istream)
         lexer.removeErrorListeners()
         lexer.addErrorListener(error_listener)
+
+        # Set the lexer in the correct mode to parse the desired rule.
+        lexer_mode = "KEY" if rule_name in KEY_RULES else "VALUE"
+        lexer.mode(getattr(OverrideLexer, lexer_mode))
+
         stream = CommonTokenStream(lexer)
         parser = OverrideParser(stream)
         parser.removeErrorListeners()
diff --git a/hydra/core/override_parser/overrides_visitor.py b/hydra/core/override_parser/overrides_visitor.py
--- a/hydra/core/override_parser/overrides_visitor.py
+++ b/hydra/core/override_parser/overrides_visitor.py
@@ -26,7 +26,7 @@
 try:
     from hydra.grammar.gen.OverrideLexer import OverrideLexer
     from hydra.grammar.gen.OverrideParser import OverrideParser
-    from hydra.grammar.gen.OverrideVisitor import OverrideVisitor
+    from hydra.grammar.gen.OverrideParserVisitor import OverrideParserVisitor
 
 except ModuleNotFoundError:
     print(
@@ -35,7 +35,7 @@
     sys.exit(1)
 
 
-class HydraOverrideVisitor(OverrideVisitor):  # type: ignore
+class HydraOverrideVisitor(OverrideParserVisitor):  # type: ignore
     def __init__(self, functions: Functions):
         self.functions = functions
 
@@ -83,6 +83,11 @@ def visitPrimitive(
         last_idx = ctx.getChildCount()
         # skip first if whitespace
         if self.is_ws(ctx.getChild(0)):
+            if last_idx == 1:
+                # Only whitespaces => this is not allowed.
+                raise HydraException(
+                    "Trying to parse a primitive that is all whitespaces"
+                )
             first_idx = 1
         if self.is_ws(ctx.getChild(-1)):
             last_idx = last_idx - 1
@@ -144,18 +149,18 @@ def visitDictValue(
     ) -> Dict[str, ParsedElementType]:
         ret = {}
         children = ctx.getChildren()
-        assert self.is_matching_terminal(next(children), "{")
+        assert self.is_matching_terminal(next(children), OverrideLexer.BRACE_OPEN)
         first = True
         while True:
             item = next(children)
-            if self.is_matching_terminal(item, "}"):
+            if self.is_matching_terminal(item, OverrideLexer.BRACE_CLOSE):
                 break
-            if not first and self.is_matching_terminal(item, ","):
+            if not first and self.is_matching_terminal(item, OverrideLexer.COMMA):
                 continue
 
             pkey = item.getText()
 
-            assert self.is_matching_terminal(next(children), ":")
+            assert self.is_matching_terminal(next(children), OverrideLexer.COLON)
 
             value = next(children)
             if isinstance(value, OverrideParser.ElementContext):
@@ -182,8 +187,6 @@ def visitElement(self, ctx: OverrideParser.ElementContext) -> ParsedElementType:
     def visitValue(
         self, ctx: OverrideParser.ValueContext
     ) -> Union[ChoiceSweep, RangeSweep, IntervalSweep, ParsedElementType]:
-        if self.is_matching_terminal(ctx, "<EOF>"):
-            return ""
         if ctx.element():
             return self.visitElement(ctx.element())
         elif ctx.simpleChoiceSweep() is not None:
@@ -217,7 +220,7 @@ def visitOverride(self, ctx: OverrideParser.OverrideContext) -> Override:
             value = None
             value_type = None
         else:
-            assert self.is_matching_terminal(eq_node, "=")
+            assert self.is_matching_terminal(eq_node, OverrideLexer.EQUAL)
             if ctx.value() is None:
                 value = ""
                 value_type = ValueType.ELEMENT
@@ -246,15 +249,15 @@ def visitOverride(self, ctx: OverrideParser.OverrideContext) -> Override:
             pkg2=key.pkg2,
         )
 
-    def is_matching_terminal(self, node: Any, text: str) -> bool:
-        return isinstance(node, TerminalNodeImpl) and node.getText() == text
+    def is_matching_terminal(self, node: Any, symbol_type: int) -> bool:
+        return isinstance(node, TerminalNodeImpl) and node.symbol.type == symbol_type
 
     def visitSimpleChoiceSweep(
         self, ctx: OverrideParser.SimpleChoiceSweepContext
     ) -> ChoiceSweep:
         ret = []
         for child in ctx.getChildren(
-            predicate=lambda x: not self.is_matching_terminal(x, ",")
+            predicate=lambda x: not self.is_matching_terminal(x, OverrideLexer.COMMA)
         ):
             ret.append(self.visitElement(child))
         return ChoiceSweep(simple_form=True, list=ret)
@@ -264,11 +267,11 @@ def visitFunction(self, ctx: OverrideParser.FunctionContext) -> Any:
         kwargs = {}
         children = ctx.getChildren()
         func_name = next(children).getText()
-        assert self.is_matching_terminal(next(children), "(")
+        assert self.is_matching_terminal(next(children), OverrideLexer.POPEN)
         in_kwargs = False
         while True:
             cur = next(children)
-            if self.is_matching_terminal(cur, ")"):
+            if self.is_matching_terminal(cur, OverrideLexer.PCLOSE):
                 break
 
             if isinstance(cur, OverrideParser.ArgNameContext):
@@ -278,7 +281,7 @@ def visitFunction(self, ctx: OverrideParser.FunctionContext) -> Any:
                 value = self.visitElement(cur)
                 kwargs[name] = value
             else:
-                if self.is_matching_terminal(cur, ","):
+                if self.is_matching_terminal(cur, OverrideLexer.COMMA):
                     continue
                 if in_kwargs:
                     raise HydraException("positional argument follows keyword argument")


Test patch:
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -1013,7 +1013,9 @@ def test_complex_defaults(overrides: Any, expected: Any) -> None:
             ["db"],
             pytest.raises(
                 HydraException,
-                match=re.escape("Error parsing override 'db'\nmissing '=' at '<EOF>'"),
+                match=re.escape(
+                    "Error parsing override 'db'\nmissing EQUAL at '<EOF>'"
+                ),
             ),
             id="syntax_error",
         ),
diff --git a/tests/test_overrides_parser.py b/tests/test_overrides_parser.py
--- a/tests/test_overrides_parser.py
+++ b/tests/test_overrides_parser.py
@@ -49,6 +49,9 @@ def eq(item1: Any, item2: Any) -> bool:
         pytest.param("abc", "abc", id="value:id"),
         pytest.param("abc123", "abc123", id="value:idint"),
         pytest.param("abc-123", "abc-123", id="value:id-int"),
+        pytest.param("a b c\t-\t1 2 3", "a b c\t-\t1 2 3", id="value:str-ws-in"),
+        pytest.param(" abc-123 ", "abc-123", id="value:str-ws-out"),
+        pytest.param("123abc", "123abc", id="value:str-int-id"),
         pytest.param("xyz_${a.b.c}", "xyz_${a.b.c}", id="value:str_interpolation"),
         pytest.param(
             "${env:USER,root}", "${env:USER,root}", id="value:custom_interpolation"
@@ -367,7 +370,8 @@ def test_interval_sweep(value: str, expected: Any) -> None:
             "primitive",
             " ",
             pytest.raises(
-                HydraException, match=re.escape("mismatched input '<EOF>' expecting"),
+                HydraException,
+                match=re.escape("Trying to parse a primitive that is all whitespaces"),
             ),
             id="error:value:whitespace",
         ),
@@ -1490,19 +1494,19 @@ def error(msg: str) -> Any:
             "interval(1.0, 2.0)",
             CastResults(
                 int=CastResults.error(
-                    "ValueError while evaluating 'int(interval(1.0,2.0))':"
+                    "ValueError while evaluating 'int(interval(1.0, 2.0))':"
                     " Intervals are always interpreted as floating-point intervals and cannot be cast"
                 ),
                 float=CastResults.error(
-                    "ValueError while evaluating 'float(interval(1.0,2.0))':"
+                    "ValueError while evaluating 'float(interval(1.0, 2.0))':"
                     " Intervals are always interpreted as floating-point intervals and cannot be cast"
                 ),
                 str=CastResults.error(
-                    "ValueError while evaluating 'str(interval(1.0,2.0))':"
+                    "ValueError while evaluating 'str(interval(1.0, 2.0))':"
                     " Intervals are always interpreted as floating-point intervals and cannot be cast"
                 ),
                 bool=CastResults.error(
-                    "ValueError while evaluating 'bool(interval(1.0,2.0))':"
+                    "ValueError while evaluating 'bool(interval(1.0, 2.0))':"
                     " Intervals are always interpreted as floating-point intervals and cannot be cast"
                 ),
             ),
@@ -1812,3 +1816,81 @@ def test_glob_filter(
 ) -> None:
     strings = ["the", "quick", "brown", "fox", "jumped", "under", "the", "lazy", "dog"]
     assert Glob(include=include, exclude=exclude).filter(strings) == expected
+
+
+@pytest.mark.parametrize(  # type: ignore
+    "value,expected_key,expected_value,expected_value_type",
+    [
+        pytest.param(
+            "key= \tvalue \t", "key", "value", ValueType.ELEMENT, id="leading+trailing",
+        ),
+        pytest.param(
+            "key=value \t 123",
+            "key",
+            "value \t 123",
+            ValueType.ELEMENT,
+            id="inside_primitive",
+        ),
+        pytest.param(
+            "key='  foo,bar\t'",
+            "key",
+            QuotedString(text="  foo,bar\t", quote=Quote.single),
+            ValueType.ELEMENT,
+            id="inside_quoted_value_outer",
+        ),
+        pytest.param(
+            "key='foo , \t\t bar'",
+            "key",
+            QuotedString(text="foo , \t\t bar", quote=Quote.single),
+            ValueType.ELEMENT,
+            id="inside_quoted_value_inter",
+        ),
+        pytest.param(
+            "key=1 ,  2,\t\t3",
+            "key",
+            ChoiceSweep(list=[1, 2, 3], simple_form=True),
+            ValueType.SIMPLE_CHOICE_SWEEP,
+            id="around_commas",
+        ),
+        pytest.param(
+            "key=choice\t(  1\t\t )",
+            "key",
+            ChoiceSweep(list=[1]),
+            ValueType.CHOICE_SWEEP,
+            id="function_one_arg",
+        ),
+        pytest.param(
+            "key=choice(\t\t1,   2, \t 3 )",
+            "key",
+            ChoiceSweep(list=[1, 2, 3]),
+            ValueType.CHOICE_SWEEP,
+            id="function_many_args",
+        ),
+        pytest.param(
+            "key=[1  ,\t2\t],[\t3\t   ,4  ]",
+            "key",
+            ChoiceSweep(list=[[1, 2], [3, 4]], simple_form=True),
+            ValueType.SIMPLE_CHOICE_SWEEP,
+            id="in_lists",
+        ),
+        pytest.param(
+            "key=str(\t  [  1,\t2  ]\t   )",
+            "key",
+            ["1", "2"],
+            ValueType.ELEMENT,
+            id="mixed",
+        ),
+    ],
+)
+def test_whitespaces(
+    value: str, expected_key: str, expected_value: Any, expected_value_type: ValueType,
+) -> None:
+    ret = parse_rule(value, "override")
+    expected = Override(
+        input_line=value,
+        type=OverrideType.CHANGE,
+        key_or_group=expected_key,
+        _value=expected_value,
+        value_type=expected_value_type,
+    )
+    assert ret == expected