Repository: facebookresearch/hydra

Issue:
[Bug] Cannot use overrides containing '%' character
# 🐛 Bug
## Description
I am unable to override a value that contains the '%' character without doing special quoting around the override

## To reproduce

```
import hydra

@hydra.main()
def foo(*args, **kargs):
    pass

foo()
```

** Minimal Code/Config snippet to reproduce **
```
python test.py +bar=/path/to/pattern_%r.bin
```

** Stack trace/error message **
```
LexerNoViableAltException: +bar=/path/to/pattern_%r.bin
                                                 ^
See https://hydra.cc/docs/next/advanced/override_grammar/basic for details

Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
```

## Expected Behavior
bar's value should be set to "/path/to/pattern_%r.bin".

## System information
- **Hydra Version** :  1.0.0
- **Python version** : 3.7.7
- **Virtual environment type and version** : 
- **Operating system** : 

## Additional context
Similar error to: Issue #798.



Discussion:

Resolving patch:
diff --git a/hydra/core/override_parser/overrides_visitor.py b/hydra/core/override_parser/overrides_visitor.py
--- a/hydra/core/override_parser/overrides_visitor.py
+++ b/hydra/core/override_parser/overrides_visitor.py
@@ -1,7 +1,7 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import sys
 import warnings
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from antlr4 import TerminalNode, Token
 from antlr4.error.ErrorListener import ErrorListener
@@ -93,7 +93,20 @@ def visitPrimitive(
             last_idx = last_idx - 1
         num = last_idx - first_idx
         if num > 1:
-            ret = ctx.getText().strip()
+            # Concatenate, while un-escaping as needed.
+            tokens = []
+            for i, n in enumerate(ctx.getChildren()):
+                if n.symbol.type == OverrideLexer.WS and (
+                    i < first_idx or i >= last_idx
+                ):
+                    # Skip leading / trailing whitespaces.
+                    continue
+                tokens.append(
+                    n.symbol.text[1::2]  # un-escape by skipping every other char
+                    if n.symbol.type == OverrideLexer.ESC
+                    else n.symbol.text
+                )
+            ret = "".join(tokens)
         else:
             node = ctx.getChild(first_idx)
             if node.symbol.type == OverrideLexer.QUOTED_VALUE:
@@ -125,6 +138,8 @@ def visitPrimitive(
                     ret = False
                 else:
                     assert False
+            elif node.symbol.type == OverrideLexer.ESC:
+                ret = node.symbol.text[1::2]
             else:
                 return node.getText()  # type: ignore
         return ret
@@ -147,29 +162,23 @@ def visitListValue(
     def visitDictValue(
         self, ctx: OverrideParser.DictValueContext
     ) -> Dict[str, ParsedElementType]:
-        ret = {}
-        children = ctx.getChildren()
-        assert self.is_matching_terminal(next(children), OverrideLexer.BRACE_OPEN)
-        first = True
-        while True:
-            item = next(children)
-            if self.is_matching_terminal(item, OverrideLexer.BRACE_CLOSE):
-                break
-            if not first and self.is_matching_terminal(item, OverrideLexer.COMMA):
-                continue
-
-            pkey = item.getText()
-
-            assert self.is_matching_terminal(next(children), OverrideLexer.COLON)
-
-            value = next(children)
-            if isinstance(value, OverrideParser.ElementContext):
-                ret[pkey] = self.visitElement(value)
-            else:
-                assert False
-            first = False
+        assert self.is_matching_terminal(ctx.getChild(0), OverrideLexer.BRACE_OPEN)
+        return dict(
+            self.visitDictKeyValuePair(ctx.getChild(i))
+            for i in range(1, ctx.getChildCount() - 1, 2)
+        )
 
-        return ret
+    def visitDictKeyValuePair(
+        self, ctx: OverrideParser.DictKeyValuePairContext
+    ) -> Tuple[str, ParsedElementType]:
+        children = ctx.getChildren()
+        item = next(children)
+        assert self.is_matching_terminal(item, OverrideLexer.ID)
+        pkey = item.getText()
+        assert self.is_matching_terminal(next(children), OverrideLexer.COLON)
+        value = next(children)
+        assert isinstance(value, OverrideParser.ElementContext)
+        return pkey, self.visitElement(value)
 
     def visitElement(self, ctx: OverrideParser.ElementContext) -> ParsedElementType:
         assert isinstance(ctx, OverrideParser.ElementContext)


Test patch:
diff --git a/tests/test_overrides_parser.py b/tests/test_overrides_parser.py
--- a/tests/test_overrides_parser.py
+++ b/tests/test_overrides_parser.py
@@ -53,6 +53,17 @@ def eq(item1: Any, item2: Any) -> bool:
         pytest.param("a b c\t-\t1 2 3", "a b c\t-\t1 2 3", id="value:str-ws-in"),
         pytest.param(" abc-123 ", "abc-123", id="value:str-ws-out"),
         pytest.param("123abc", "123abc", id="value:str-int-id"),
+        pytest.param(r"a\,b", "a,b", id="value:str-esc-comma"),
+        pytest.param(r"a\:b", "a:b", id="value:str-esc-colon"),
+        pytest.param(r"a\=b", "a=b", id="value:str-esc-equal"),
+        pytest.param(r"\ ab", " ab", id="value:str-esc-space"),
+        pytest.param("ab\\\t", "ab\t", id="value:str-esc-tab"),
+        pytest.param("ab\\\\", "ab\\", id="value:str-esc-backslash"),
+        pytest.param(r"\,", ",", id="value:str-esc-comma-alone"),
+        pytest.param(r"f\(a\, b\)", "f(a, b)", id="value:str-esc-parentheses"),
+        pytest.param(r"\[a\, b\]", "[a, b]", id="value:str-esc-brackets"),
+        pytest.param(r"\{a\: b\}", "{a: b}", id="value:str-esc-braces"),
+        pytest.param(r"$\{foo.bar\}", "${foo.bar}", id="value:str-esc-braces"),
         pytest.param("xyz_${a.b.c}", "xyz_${a.b.c}", id="value:str_interpolation"),
         pytest.param(
             "${env:USER,root}", "${env:USER,root}", id="value:custom_interpolation"
@@ -62,9 +73,12 @@ def eq(item1: Any, item2: Any) -> bool:
         pytest.param("null", None, id="value:null"),
         # int
         pytest.param("1", 1, id="value:int:pos"),
+        pytest.param("+1", 1, id="value:int:explicit_pos"),
+        pytest.param("1___0___", "1___0___", id="value:int:not_an_int"),
         # float
         pytest.param("0.51", 0.51, id="value:float:positive"),
         pytest.param("10e0", 10.0, id="value:float:exp"),
+        pytest.param("+inf", math.inf, id="value:float:plus_inf"),
         # bool
         pytest.param("true", True, id="value:bool"),
         pytest.param(".", ".", id="value:dot"),
@@ -260,9 +274,9 @@ def test_shuffle_sequence(value: str, expected: Any) -> None:
         pytest.param("{a:10}", {"a": 10}, id="dict"),
         pytest.param("{a:[a,10]}", {"a": ["a", 10]}, id="dict"),
         pytest.param("{a:[true,10]}", {"a": [True, 10]}, id="dict"),
-        pytest.param("{a:10,b:20}}", {"a": 10, "b": 20}, id="dict"),
-        pytest.param("{a:10,b:{}}}", {"a": 10, "b": {}}, id="dict"),
-        pytest.param("{a:10,b:{c:[1,2]}}}", {"a": 10, "b": {"c": [1, 2]}}, id="dict"),
+        pytest.param("{a:10,b:20}", {"a": 10, "b": 20}, id="dict"),
+        pytest.param("{a:10,b:{}}", {"a": 10, "b": {}}, id="dict"),
+        pytest.param("{a:10,b:{c:[1,2]}}", {"a": 10, "b": {"c": [1, 2]}}, id="dict"),
     ],
 )
 def test_dict_value(value: str, expected: Any) -> None:
@@ -409,7 +423,7 @@ def test_interval_sweep(value: str, expected: Any) -> None:
             "key",
             "foo@",
             pytest.raises(
-                HydraException, match=re.escape("missing {ID, DOT_PATH} at '<EOF>'")
+                HydraException, match=re.escape("missing {DOT_PATH, ID} at '<EOF>'")
             ),
             id="error:left_overs",
         ),
@@ -417,7 +431,7 @@ def test_interval_sweep(value: str, expected: Any) -> None:
             "key",
             "foo@abc:",
             pytest.raises(
-                HydraException, match=re.escape("missing {ID, DOT_PATH} at '<EOF>'")
+                HydraException, match=re.escape("missing {DOT_PATH, ID} at '<EOF>'")
             ),
             id="error:left_overs",
         ),