Repository: xonsh/xonsh

Issue:
Crash on parsing list nested in dict
When parsing a line containing a dict that has a list (or tuple) as a value, xonsh crashes.

```
bollmann@underberg ~/repositories/xonsh master >>> {'foo': ['a', 'b']}
Traceback (most recent call last):
  File "/usr/local/bin/xonsh", line 12, in <module>
    exec(compile(open(__file__).read(), __file__, 'exec'))
  File "/home/bollmann/repositories/xonsh/scripts/xonsh", line 3, in <module>
    main()
  File "/home/bollmann/repositories/xonsh/xonsh/main.py", line 25, in main
    shell.cmdloop()
  File "/home/bollmann/repositories/xonsh/xonsh/shell.py", line 152, in cmdloop
    super(Shell, self).cmdloop(intro=intro)
  File "/usr/lib/python3.4/cmd.py", line 138, in cmdloop
    stop = self.onecmd(line)
  File "/usr/lib/python3.4/cmd.py", line 211, in onecmd
    return self.default(line)
  File "/home/bollmann/repositories/xonsh/xonsh/shell.py", line 105, in default
    code = self.push(line)
  File "/home/bollmann/repositories/xonsh/xonsh/shell.py", line 126, in push
    locs=self.ctx)
  File "/home/bollmann/repositories/xonsh/xonsh/execer.py", line 87, in compile
    code = compile(tree, self.filename, mode)
ValueError: Dict doesn't have the same number of keys as values
bollmann@underberg ~/repositories/xonsh (master) $ 
```

Other nested structures seem to work fine. Putting the list in a variable first and then constructing the dict also works fine.

```
bollmann@underberg ~/repositories/xonsh master >>> x = ['a', 'b']
bollmann@underberg ~/repositories/xonsh master >>> {'foo': x}
{'foo': ['a', 'b']}
bollmann@underberg ~/repositories/xonsh master >>> {'a': {'b': 'c', 'd': 'e'}}
{'a': {'b': 'c', 'd': 'e'}}
bollmann@underberg ~/repositories/xonsh master >>> [{'a': 'b', 'c': 'd'}]
[{'a': 'b', 'c': 'd'}]
bollmann@underberg ~/repositories/xonsh master >>> [{'a': 'b'}, {'c': 'd'}]
[{'a': 'b'}, {'c': 'd'}]
bollmann@underberg ~/repositories/xonsh master >>> {'a': {'b': 'c', 'd': 'e'}}
{'a': {'b': 'c', 'd': 'e'}}
```



Discussion:
Thanks for reporting @mbollmann!


Resolving patch:
diff --git a/xonsh/parser.py b/xonsh/parser.py
--- a/xonsh/parser.py
+++ b/xonsh/parser.py
@@ -1536,6 +1536,7 @@ def p_atom(self, p):
             # filled, possible group container tuple atoms
             if isinstance(p2, ast.AST):
                 p0 = p2
+                p0._real_tuple = True
             elif len(p2) == 1 and isinstance(p2[0], ast.AST):
                 p0 = p2[0]
             else:
@@ -1545,8 +1546,14 @@ def p_atom(self, p):
                 p0 = ast.ListComp(elt=p2.elt, generators=p2.generators, 
                                   lineno=p2.lineno, col_offset=p2.col_offset)
             else:
-                p2 = ensure_has_elts(p2)
-                p0 = ast.List(elts=p2.elts, ctx=ast.Load(), lineno=self.lineno, 
+                if isinstance(p2, ast.Tuple):
+                    if hasattr(p2, '_real_tuple') and p2._real_tuple:
+                        elts = [p2]
+                    else:
+                        elts = p2.elts
+                else:
+                    elts = [p2]
+                p0 = ast.List(elts=elts, ctx=ast.Load(), lineno=self.lineno,
                               col_offset=self.col)
         elif p1 == '{':
             p0 = p2
@@ -1589,19 +1596,21 @@ def p_testlist_comp(self, p):
                          | test_or_star_expr comma_test_or_star_expr_list comma_opt
         """
         p1, p2 = p[1], p[2]
-        p0 = ensure_has_elts(p1, lineno=self.lineno, col_offset=self.col)
         if len(p) == 3:
             if p2 is None:
                 # split out grouping parentheses.
-                p0 = p0.elts[0]
+                p0 = p1
             elif p2 == ',':
-                pass
+                p0 = ast.Tuple(elts=[p1], ctx=ast.Load(), lineno=self.lineno,
+                               col_offset=self.col)
             elif 'comps' in p2:
-                p0 = ast.GeneratorExp(elt=p0.elts[0], generators=p2['comps'], 
+                p0 = ast.GeneratorExp(elt=p1, generators=p2['comps'],
                                       lineno=self.lineno, col_offset=self.col)
             else:
                 assert False
         elif len(p) == 4:
+            p0 = ast.Tuple(elts=[p1], ctx=ast.Load(), lineno=self.lineno,
+                           col_offset=self.col)
             if p2 is not None:
                 p0.elts.extend(p2) 
             else:
@@ -1694,7 +1703,12 @@ def p_testlist(self, p):
         lenp = len(p)
         p1 = p[1]
         if lenp > 2:
-            p1 = ensure_has_elts(p1, lineno=self.lineno, col_offset=self.col)
+            if isinstance(p1, ast.Tuple) and \
+                    (hasattr(p1, '_real_tuple') and p1._real_tuple):
+                p1 = ast.Tuple(elts=[p1], ctx=ast.Load(), lineno=self.lineno,
+                               col_offset=self.col)
+            else:
+                p1 = ensure_has_elts(p1, lineno=self.lineno, col_offset=self.col)
             p2 = p[2] if lenp > 2 else []
             p2 = [] if p2 == ',' else p2
             p1.elts += p2
@@ -1714,16 +1728,23 @@ def p_dictorsetmaker(self, p):
         p1 = p[1]
         lenp = len(p)
         if lenp == 2:
-            p1 = ensure_has_elts(p1)
-            p0 = ast.Set(elts=p1.elts, ctx=ast.Load(), lineno=self.lineno, 
+            elts = [p1]
+            if isinstance(p1, ast.Tuple) and \
+                    not (hasattr(p1, '_real_tuple') and p1._real_tuple):
+                elts = p1.elts
+            p0 = ast.Set(elts=elts, ctx=ast.Load(), lineno=self.lineno,
                          col_offset=self.col)
         elif lenp == 3:
             comps = p[2].get('comps', [])
             p0 = ast.SetComp(elt=p1, generators=comps, lineno=self.lineno, 
                              col_offset=self.col)
         elif lenp == 4:
-            p3 = ensure_has_elts(p[3])
-            p0 = ast.Dict(keys=[p1], values=p3.elts, ctx=ast.Load(),
+            p3 = p[3]
+            vals = [p3]
+            if isinstance(p3, ast.Tuple) and \
+                    not (hasattr(p3, '_real_tuple') and p3._real_tuple):
+                vals = p3.elts
+            p0 = ast.Dict(keys=[p1], values=vals, ctx=ast.Load(),
                           lineno=self.lineno, col_offset=self.col)
         elif lenp == 5:
             comps = p[4].get('comps', [])


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -309,9 +309,48 @@ def test_list_two():
 def test_list_three():
     yield check_ast, '[1, 42, 65]'
 
-def test_list_three():
+def test_list_three_comma():
     yield check_ast, '[1, 42, 65,]'
 
+def test_list_one_nested():
+    yield check_ast, '[[1]]'
+
+def test_list_list_four_nested():
+    yield check_ast, '[[1], [2], [3], [4]]'
+
+def test_list_tuple_three_nested():
+    yield check_ast, '[(1,), (2,), (3,)]'
+
+def test_list_set_tuple_three_nested():
+    yield check_ast, '[{(1,)}, {(2,)}, {(3,)}]'
+
+def test_list_tuple_one_nested():
+    yield check_ast, '[(1,)]'
+
+def test_tuple_tuple_one_nested():
+    yield check_ast, '((1,),)'
+
+def test_dict_list_one_nested():
+    yield check_ast, '{1: [2]}'
+
+def test_dict_list_one_nested_comma():
+    yield check_ast, '{1: [2],}'
+
+def test_dict_tuple_one_nested():
+    yield check_ast, '{1: (2,)}'
+
+def test_dict_tuple_one_nested_comma():
+    yield check_ast, '{1: (2,),}'
+
+def test_dict_list_two_nested():
+    yield check_ast, '{1: [2], 3: [4]}'
+
+def test_set_tuple_one_nested():
+    yield check_ast, '{(1,)}'
+
+def test_set_tuple_two_nested():
+    yield check_ast, '{(1,), (2,)}'
+
 def test_tuple_empty():
     yield check_ast, '()'