Repository: ipython/ipython

Issue:
Error in error reporting.
The program to reproduce the error is the following:

``` python
# file: ~/tmp/ipython-except.py
def foo(u):
    raise ValueError

# This shows the correct traceback:
# sum(sum(foo(u) for u in [0]) for _ in [0])
sum(sum(foo(u) for _ in [0]) for u in [0])
```

The following error is produced:

```
16:28:05 [5]: %run -i ~/tmp/ipython_except.py
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-5-21c031f5aefb> in <module>()
----> 1 get_ipython().magic(u'run -i ~/tmp/ipython_except.py')

/Users/musically_ut/prog/rare/ipython/IPython/core/interactiveshell.py in magic(self, arg_s)
   2306         magic_name, _, magic_arg_s = arg_s.partition(' ')
   2307         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
-> 2308         return self.run_line_magic(magic_name, magic_arg_s)
   2309
   2310     #-------------------------------------------------------------------------

/Users/musically_ut/prog/rare/ipython/IPython/core/interactiveshell.py in run_line_magic(self, magic_name, line)
   2227                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
   2228             with self.builtin_trap:
-> 2229                 result = fn(*args,**kwargs)
   2230             return result
   2231

/Users/musically_ut/prog/rare/ipython/IPython/core/magics/execution.py in run(self, parameter_s, runner, file_finder)

/Users/musically_ut/prog/rare/ipython/IPython/core/magic.py in <lambda>(f, *a, **k)
    191     # but it's overkill for just that one bit of state.
    192     def magic_deco(arg):
--> 193         call = lambda f, *a, **k: f(*a, **k)
    194
    195         if callable(arg):

/Users/musically_ut/prog/rare/ipython/IPython/core/magics/execution.py in run(self, parameter_s, runner, file_finder)
    736                         else:
    737                             # regular execution
--> 738                             run()
    739
    740                 if 'i' in opts:

/Users/musically_ut/prog/rare/ipython/IPython/core/magics/execution.py in run()
    722                             def run():
    723                                 runner(filename, prog_ns, prog_ns,
--> 724                                        exit_ignore=exit_ignore)
    725
    726                         if 't' in opts:

/Users/musically_ut/prog/rare/ipython/IPython/core/interactiveshell.py in safe_execfile(self, fname, *where, **kw)
   2655                     raise
   2656                 # tb offset is 2 because we wrap execfile
-> 2657                 self.showtraceback(tb_offset=2)
   2658
   2659     def safe_execfile_ipy(self, fname, shell_futures=False):

/Users/musically_ut/prog/rare/ipython/IPython/core/interactiveshell.py in showtraceback(self, exc_tuple, filename, tb_offset, exception_only)
   1850                     except Exception:
   1851                         stb = self.InteractiveTB.structured_traceback(etype,
-> 1852                                             value, tb, tb_offset=tb_offset)
   1853
   1854                     self._showtraceback(etype, value, stb)

/Users/musically_ut/prog/rare/ipython/IPython/core/ultratb.pyc in structured_traceback(self, etype, value, tb, tb_offset, number_of_lines_of_context)
   1238         self.tb = tb
   1239         return FormattedTB.structured_traceback(
-> 1240             self, etype, value, tb, tb_offset, number_of_lines_of_context)
   1241
   1242

/Users/musically_ut/prog/rare/ipython/IPython/core/ultratb.pyc in structured_traceback(self, etype, value, tb, tb_offset, number_of_lines_of_context)
   1146             # Verbose modes need a full traceback
   1147             return VerboseTB.structured_traceback(
-> 1148                 self, etype, value, tb, tb_offset, number_of_lines_of_context
   1149             )
   1150         else:

/Users/musically_ut/prog/rare/ipython/IPython/core/ultratb.pyc in structured_traceback(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)
    998
    999         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,
-> 1000                                                                tb_offset)
   1001
   1002         colors = self.Colors  # just a shorthand + quicker name lookup

/Users/musically_ut/prog/rare/ipython/IPython/core/ultratb.pyc in format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset)
    949         records = self.get_records(etb, number_of_lines_of_context, tb_offset)
    950
--> 951         frames = self.format_records(records)
    952         if records is None:
    953             return ""

/Users/musically_ut/prog/rare/ipython/IPython/core/ultratb.pyc in format_records(self, records)
    745             file = py3compat.cast_unicode(file, util_path.fs_encoding)
    746             link = tpl_link % file
--> 747             args, varargs, varkw, locals = inspect.getargvalues(frame)
    748
    749             if func == '?':

/usr/local/Cellar/python/2.7.9/Frameworks/Python.framework/Versions/2.7/lib/python2.7/inspect.pyc in getargvalues(frame)
    827     'varargs' and 'varkw' are the names of the * and ** arguments or None.
    828     'locals' is the locals dictionary of the given frame."""
--> 829     args, varargs, varkw = getargs(frame.f_code)
    830     return ArgInfo(args, varargs, varkw, frame.f_locals)
    831

/usr/local/Cellar/python/2.7.9/Frameworks/Python.framework/Versions/2.7/lib/python2.7/inspect.pyc in getargs(co)
    789                                 remain[-1] = remain[-1] - 1
    790                             if not remain: break
--> 791             args[i] = stack[0]
    792
    793     varargs = None

IndexError: list index out of range
```

Expected output is something along the lines of the following:

```
16:31:07 [11]: %run -i ~/tmp/ipython_except.py
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/Users/musically_ut/tmp/ipython_except.py in <module>()
      3
      4 # This shows the correct traceback:
----> 5 sum(sum(foo(u) for u in [0]) for _ in [0])
      6 # sum(sum(foo(u) for _ in [0]) for u in [0])

/Users/musically_ut/tmp/ipython_except.py in <genexpr>((_,))
      3
      4 # This shows the correct traceback:
----> 5 sum(sum(foo(u) for u in [0]) for _ in [0])
      6 # sum(sum(foo(u) for _ in [0]) for u in [0])

/Users/musically_ut/tmp/ipython_except.py in <genexpr>((u,))
      3
      4 # This shows the correct traceback:
----> 5 sum(sum(foo(u) for u in [0]) for _ in [0])
      6 # sum(sum(foo(u) for _ in [0]) for u in [0])

/Users/musically_ut/tmp/ipython_except.py in foo(u)
      1 def foo(u):
----> 2     raise ValueError
      3
      4 # This shows the correct traceback:
      5 sum(sum(foo(u) for u in [0]) for _ in [0])

ValueError:
```

Tested against IPython `4.0.0-dev` (current `master`) and Python `2.7.9`:

```
$ python -m IPython
Python 2.7.9 (default, Jan  7 2015, 11:49:12)
Type "copyright", "credits" or "license" for more information.

IPython 4.0.0-dev -- An enhanced Interactive Python.
```



Discussion:
Update, this problem doesn't occur with Python-3, i.e. the correct traceback is shown for both the statements:

```
sum(sum(foo(u) for u in [0]) for _ in [0])
sum(sum(foo(u) for _ in [0]) for u in [0])
```

Might be a Python 2.7 bug. the frame does: 

```
LOAD_FAST
FOR_ITER
STORE_DEREF
LOAD_GLOBAL
LOAD_CLOSURE
BUILD_TUPLE
LOAD_CONST
MAKE_CLOSURE
LOAD_CONST
BUILD_LIST
GET_ITER
CALL_FUNCTION
CALL_FUNCTION
YIELD_VALUE
POP_TOP
JUMP_ABSOLUTE
LOAD_CONST
RETURN_VALUE
```

For which obviously getargs will fail. ...

Then should it be escalated to the Python2.7 bug tracker?  
I think it would be best if you (@Carreau) write the report. :)

Yeah, trying to make a smaller example. 
I don't use getframe a lot so have to re-read the doc, but will probably do. 

Thanks!
Also, please post the link to the Python bug here when you file it. :+1: 

Sure, that might not be today though. 

Any updates on this? 

No not particularly. 
I trying to up my installation to work on both 3.5 and 2.7 head of Python, but it's a rabbit hole.

![](http://i.imgur.com/t0XHtgJ.gif)


Resolving patch:
diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py
--- a/IPython/core/ultratb.py
+++ b/IPython/core/ultratb.py
@@ -83,6 +83,7 @@
 from __future__ import unicode_literals
 from __future__ import print_function
 
+import dis
 import inspect
 import keyword
 import linecache
@@ -222,21 +223,98 @@ def findsource(object):
     raise IOError('could not find code object')
 
 
+# This is a patched version of inspect.getargs that applies the (unmerged)
+# patch for http://bugs.python.org/issue14611 by Stefano Taschini.  This fixes
+# https://github.com/ipython/ipython/issues/8205 and
+# https://github.com/ipython/ipython/issues/8293
+def getargs(co):
+    """Get information about the arguments accepted by a code object.
+
+    Three things are returned: (args, varargs, varkw), where 'args' is
+    a list of argument names (possibly containing nested lists), and
+    'varargs' and 'varkw' are the names of the * and ** arguments or None."""
+    if not iscode(co):
+        raise TypeError('{!r} is not a code object'.format(co))
+
+    nargs = co.co_argcount
+    names = co.co_varnames
+    args = list(names[:nargs])
+    step = 0
+
+    # The following acrobatics are for anonymous (tuple) arguments.
+    for i in range(nargs):
+        if args[i][:1] in ('', '.'):
+            stack, remain, count = [], [], []
+            while step < len(co.co_code):
+                op = ord(co.co_code[step])
+                step = step + 1
+                if op >= dis.HAVE_ARGUMENT:
+                    opname = dis.opname[op]
+                    value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
+                    step = step + 2
+                    if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
+                        remain.append(value)
+                        count.append(value)
+                    elif opname in ('STORE_FAST', 'STORE_DEREF'):
+                        if op in dis.haslocal:
+                            stack.append(co.co_varnames[value])
+                        elif op in dis.hasfree:
+                            stack.append((co.co_cellvars + co.co_freevars)[value])
+                        # Special case for sublists of length 1: def foo((bar))
+                        # doesn't generate the UNPACK_TUPLE bytecode, so if
+                        # `remain` is empty here, we have such a sublist.
+                        if not remain:
+                            stack[0] = [stack[0]]
+                            break
+                        else:
+                            remain[-1] = remain[-1] - 1
+                            while remain[-1] == 0:
+                                remain.pop()
+                                size = count.pop()
+                                stack[-size:] = [stack[-size:]]
+                                if not remain: break
+                                remain[-1] = remain[-1] - 1
+                            if not remain: break
+            args[i] = stack[0]
+
+    varargs = None
+    if co.co_flags & inspect.CO_VARARGS:
+        varargs = co.co_varnames[nargs]
+        nargs = nargs + 1
+    varkw = None
+    if co.co_flags & inspect.CO_VARKEYWORDS:
+        varkw = co.co_varnames[nargs]
+    return inspect.Arguments(args, varargs, varkw)
+
+
 # Monkeypatch inspect to apply our bugfix.
 def with_patch_inspect(f):
     """decorator for monkeypatching inspect.findsource"""
 
     def wrapped(*args, **kwargs):
         save_findsource = inspect.findsource
+        save_getargs = inspect.getargs
         inspect.findsource = findsource
+        inspect.getargs = getargs
         try:
             return f(*args, **kwargs)
         finally:
             inspect.findsource = save_findsource
+            inspect.getargs = save_getargs
 
     return wrapped
 
 
+if py3compat.PY3:
+    fixed_getargvalues = inspect.getargvalues
+else:
+    # Fixes for https://github.com/ipython/ipython/issues/8293
+    #       and https://github.com/ipython/ipython/issues/8205.
+    # The relevant bug is caused by failure to correctly handle anonymous tuple
+    # unpacking, which only exists in Python 2.
+    fixed_getargvalues = with_patch_inspect(inspect.getargvalues)
+
+
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
@@ -744,7 +822,7 @@ def format_records(self, records):
 
             file = py3compat.cast_unicode(file, util_path.fs_encoding)
             link = tpl_link % file
-            args, varargs, varkw, locals = inspect.getargvalues(frame)
+            args, varargs, varkw, locals = fixed_getargvalues(frame)
 
             if func == '?':
                 call = ''


Test patch:
diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py
--- a/IPython/core/tests/test_ultratb.py
+++ b/IPython/core/tests/test_ultratb.py
@@ -3,8 +3,10 @@
 """
 import io
 import os.path
+from textwrap import dedent
 import unittest
 
+
 from IPython.testing import tools as tt
 from IPython.testing.decorators import onlyif_unicode_paths
 from IPython.utils.syspathcontext import prepended_to_syspath
@@ -89,6 +91,29 @@ def test_iso8859_5(self):
                 with tt.AssertPrints(u'дбИЖ', suppress=False):
                     ip.run_cell('fail()')
 
+
+class NestedGenExprTestCase(unittest.TestCase):
+    """
+    Regression test for the following issues:
+    https://github.com/ipython/ipython/issues/8293
+    https://github.com/ipython/ipython/issues/8205
+    """
+    def test_nested_genexpr(self):
+        code = dedent(
+            """\
+            class SpecificException(Exception):
+                pass
+
+            def foo(x):
+                raise SpecificException("Success!")
+
+            sum(sum(foo(x) for _ in [0]) for x in [0])
+            """
+        )
+        with tt.AssertPrints('SpecificException: Success!', suppress=False):
+            ip.run_cell(code)
+
+
 indentationerror_file = """if True:
 zoon()
 """