Repository: xonsh/xonsh

Issue:
Generator expressions sometimes not properly recognized
This code snippet executes successfully using Python 3.5.2...

``` python
import os

HOME_BIN = os.path.expanduser('~/.bin')


def get_all_paths(path):
    paths = [path]
    for root, dirs, files in os.walk(path):
        paths.extend(os.path.join(root, d) for d in dirs)
        paths.extend(os.path.join(root, f) for f in files)

    return paths


print(len(get_all_paths(HOME_BIN)))
```

 ...but fails when using xonsh/0.4.6.dev43:

```
cryzed@arch ~ $ python test.py
8
cryzed@arch ~ $ xonsh test.py
Traceback (most recent call last):
  File "/usr/lib/python3.5/site-packages/xonsh/__amalgam__.py", line 16319, in run_subproc
    **subproc_kwargs)
  File "/usr/lib/python3.5/subprocess.py", line 947, in __init__
    restore_signals, start_new_session)
  File "/usr/lib/python3.5/subprocess.py", line 1551, in _execute_child
    raise child_exception_type(errno_num, err_msg)
FileNotFoundError: [Errno 2] No such file or directory: 'for'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/bin/xonsh", line 3, in <module>
    main()
  File "/usr/lib/python3.5/site-packages/xonsh/__amalgam__.py", line 17534, in main
    loc=None, mode='exec')
  File "/usr/lib/python3.5/site-packages/xonsh/__amalgam__.py", line 2041, in run_script_with_cache
    run_compiled_code(ccode, glb, loc, mode)
  File "/usr/lib/python3.5/site-packages/xonsh/__amalgam__.py", line 1961, in run_compiled_code
    func(code, glb, loc)
  File "test.py", line 15, in <module>
    print(len(get_all_paths(HOME_BIN)))
  File "test.py", line 9, in get_all_paths
    paths.extend(os.path.join(root, d) for d in dirs)
  File "/usr/lib/python3.5/site-packages/xonsh/__amalgam__.py", line 16456, in subproc_captured_hiddenobject
    return run_subproc(cmds, captured='hiddenobject')
  File "/usr/lib/python3.5/site-packages/xonsh/__amalgam__.py", line 16329, in run_subproc
    raise XonshError(e)
xonsh.__amalgam__.XonshError: xonsh: subprocess mode: command not found: for
Did you mean one of the following?
    pr:   Command (/usr/bin/pr)
    EOF:  Alias
    tr:   Command (/usr/bin/tr)
    od:   Command (/usr/bin/od)
    fg:   Alias
```

I didn't manage to simplify the code snippet much more, because this works:

``` python
import os

a = []
a.extend(os.path.join('/', str(b)) for b in range(10))
print(repr(a))
```

So I'm not exactly sure what's going on. The issue can be worked around by adding additional parenthesis to the `paths.extend` argument:

``` python
paths.extend((os.path.join(root, d) for d in dirs))
paths.extend((os.path.join(root, f) for f in files))
```

I mentioned this in IRC and Gitter, but it seems to have been missed completely.



Discussion:
Thanks for reporting @cryzed, and sorry about missing the earlier reports! We'll try to figure this out shortly.

Sounds great, and no worries!

It seems that it went wrong because `dirs` is a valid command in $PATH. So this may be related to #1604 

That is strange.  I doubt it is that, because PATH is never checked when transforming the AST.  If anything it is that the multiple targets in the for-loop are not being added to the context.


Resolving patch:
diff --git a/xonsh/ast.py b/xonsh/ast.py
--- a/xonsh/ast.py
+++ b/xonsh/ast.py
@@ -108,6 +108,30 @@ def gather_names(node):
     return rtn
 
 
+def get_id_ctx(node):
+    """Gets the id and attribute of a node, or returns a default."""
+    nid = getattr(node, 'id', None)
+    if nid is None:
+        return (None, None)
+    return (nid, node.ctx)
+
+
+def gather_load_store_names(node):
+    """Returns the names present in the node's tree in a set of load nodes and
+    a set of store nodes.
+    """
+    load = set()
+    store = set()
+    for nid, ctx in map(get_id_ctx, walk(node)):
+        if nid is None:
+            continue
+        elif isinstance(ctx, Load):
+            load.add(nid)
+        else:
+            store.add(nid)
+    return (load, store)
+
+
 def has_elts(x):
     """Tests if x is an AST node with elements."""
     return isinstance(x, AST) and hasattr(x, 'elts')
@@ -226,7 +250,8 @@ def try_subproc_toks(self, node, strip_expr=False):
 
     def is_in_scope(self, node):
         """Determines whether or not the current node is in scope."""
-        names = gather_names(node)
+        names, store = gather_load_store_names(node)
+        names -= store
         if not names:
             return True
         inscope = False
@@ -421,10 +446,7 @@ def visit_With(self, node):
     def visit_For(self, node):
         """Handle visiting a for statement."""
         targ = node.target
-        if isinstance(targ, (Tuple, List)):
-            self.ctxupdate(leftmostname(elt) for elt in targ.elts)
-        else:
-            self.ctxadd(leftmostname(targ))
+        self.ctxupdate(gather_names(targ))
         self.generic_visit(node)
         return node
 


Test patch:
diff --git a/tests/test_ast.py b/tests/test_ast.py
--- a/tests/test_ast.py
+++ b/tests/test_ast.py
@@ -2,7 +2,7 @@
 import ast as pyast
 
 from xonsh import ast
-from xonsh.ast import Tuple, Name, Store, min_line, Call, BinOp
+from xonsh.ast import Tuple, Name, Store, min_line, Call, BinOp, pdump
 
 import pytest
 
@@ -28,6 +28,16 @@ def test_gather_names_tuple():
     assert exp == obs
 
 
+def test_gather_load_store_names_tuple():
+    node = Tuple(elts=[Name(id='y', ctx=Store()),
+                       Name(id='z', ctx=Store())])
+    lexp = set()
+    sexp = {'y', 'z'}
+    lobs, sobs = ast.gather_load_store_names(node)
+    assert lexp == lobs
+    assert sexp == sobs
+
+
 @pytest.mark.parametrize('line1', [
     # this second line wil be transformed into a subprocess call
     'x = 1',
@@ -83,9 +93,17 @@ def test_multilline_no_transform():
     if not kw and b:
         pass
 """,
+"""import os
+path = '/path/to/wakka'
+paths = []
+for root, dirs, files in os.walk(path):
+    paths.extend(os.path.join(root, d) for d in dirs)
+    paths.extend(os.path.join(root, f) for f in files)
+""",
 ])
-def test_args_in_scope(inp):
+def test_unmodified(inp):
     # Context sensitive parsing should not modify AST
     exp = pyast.parse(inp)
     obs = check_parse(inp)
+
     assert nodes_equal(exp, obs)