Repository: ipython/ipython

Issue:
Test failures with Python 3 and enabled warnings
When warnings are enabled (using e.g. PYTHONWARNINGS="d" environmental variable), then some tests fail with Python 3.3 due to DeprecationWarnings / ResourceWarnings triggered by tests:

```
**********************************************************************
IPython test group: IPython.core
...
======================================================================
ERROR: Test we're not loading modules on startup that we shouldn't.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_interactiveshell.py", line 449, in test_extraneous_loads
    tt.ipexec_validate(self.fname, out)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmpxt9yqe.py' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

======================================================================
ERROR: test_startup_ipy (IPython.core.tests.test_profile.ProfileStartupTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/decorators.py", line 233, in skipper_func
    return f(*args, **kwargs)
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_profile.py", line 117, in test_startup_ipy
    self.validate('test')
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_profile.py", line 106, in validate
    tt.ipexec_validate(self.fname, output, '', options=self.options)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmpe7ivqu/test.py' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

======================================================================
ERROR: test_startup_py (IPython.core.tests.test_profile.ProfileStartupTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/decorators.py", line 233, in skipper_func
    return f(*args, **kwargs)
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_profile.py", line 112, in test_startup_py
    self.validate('123')
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_profile.py", line 106, in validate
    tt.ipexec_validate(self.fname, output, '', options=self.options)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmpe7ivqu/test.py' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

======================================================================
ERROR: IPython.core.tests.test_run.TestMagicRunSimple.test_obj_del
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib64/python3.3/site-packages/nose/case.py", line 198, in runTest
    self.test(*self.arg)
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_run.py", line 245, in test_obj_del
    tt.ipexec_validate(self.fname, 'object A deleted', err)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmpzvlc22.py' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

======================================================================
ERROR: IPython.core.tests.test_run.TestMagicRunSimple.test_tclass
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib64/python3.3/site-packages/nose/case.py", line 198, in runTest
    self.test(*self.arg)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/decorators.py", line 233, in skipper_func
    return f(*args, **kwargs)
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_run.py", line 303, in test_tclass
    tt.ipexec_validate(self.fname, out, err)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmpai9_b6.ipy' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

======================================================================
ERROR: Test that `__file__` is set when running `ipython file.ipy`
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_shellapp.py", line 46, in test_ipy_script_file_attribute
    tt.ipexec_validate(self.fname, self.fname, err)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmpqgc37s.ipy' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

======================================================================
ERROR: Test that `__file__` is set when running `ipython file.py`
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/core/tests/test_shellapp.py", line 35, in test_py_script_file_attribute
    tt.ipexec_validate(self.fname, self.fname, err)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmp7q_73o.py' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

----------------------------------------------------------------------
Ran 384 tests in 22.403s

FAILED (SKIP=2, errors=7)
...
**********************************************************************
IPython test group: IPython.lib
...
======================================================================
FAIL: Verify that plot is not available when pylab_import_all = False
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/decorators.py", line 233, in skipper_func
    return f(*args, **kwargs)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/decorators.py", line 233, in skipper_func
    return f(*args, **kwargs)
  File "/tmp/ipython-1.0.0/build/lib/IPython/lib/tests/test_irunner_pylab_magic.py", line 119, in test_pylab_import_all_disabled
    self._test_runner(runner,source,output)
  File "/tmp/ipython-1.0.0/build/lib/IPython/lib/tests/test_irunner_pylab_magic.py", line 54, in _test_runner
    self.fail(message)
nose.proxy.AssertionError: Mismatch in number of lines

Expected:
~~~~~~~~~ 
In \[1\]: from IPython\.config\.application import Application
In \[2\]: app = Application\.instance\(\)
In \[3\]: app\.pylab_import_all = False
In \[4\]: pylab
^Using matplotlib backend:
Populating the interactive namespace from numpy and matplotlib
In \[5\]: ip=get_ipython\(\)
In \[6\]: 'plot' in ip\.user_ns
Out\[6\]: False

Got:
~~~~~~~~~ 
In [1]: from IPython.config.application import Application
In [2]: app = Application.instance()
In [3]: app.pylab_import_all = False
In [4]: pylab
/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:108: ResourceWarning: unclosed file <_io.BufferedReader name='/usr/lib64/python3.3/site-packages/PyQt4/QtCore.so'>
  imp.find_module('QtCore', mod.__path__)
/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:109: ResourceWarning: unclosed file <_io.BufferedReader name='/usr/lib64/python3.3/site-packages/PyQt4/QtGui.so'>
  imp.find_module('QtGui', mod.__path__)
/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:110: ResourceWarning: unclosed file <_io.BufferedReader name='/usr/lib64/python3.3/site-packages/PyQt4/QtSvg.so'>
  imp.find_module('QtSvg', mod.__path__)
Fontconfig warning: "/etc/fonts/conf.d/50-user.conf", line 14: reading configurations from ~/.fonts.conf is deprecated.
Using matplotlib backend: Qt4Agg
Populating the interactive namespace from numpy and matplotlib
In [5]: ip=get_ipython()
In [6]: 'plot' in ip.user_ns
Out[6]: False
    """Fail immediately, with the given message."""
>>  raise self.failureException('Mismatch in number of lines\n\nExpected:\n~~~~~~~~~\nIn \\[1\\]: from IPython\\.config\\.application import Application\nIn \\[2\\]: app = Application\\.instance\\(\\)\nIn \\[3\\]: app\\.pylab_import_all = False\nIn \\[4\\]: pylab\n^Using matplotlib backend:\nPopulating the interactive namespace from numpy and matplotlib\nIn \\[5\\]: ip=get_ipython\\(\\)\nIn \\[6\\]: \'plot\' in ip\\.user_ns\nOut\\[6\\]: False\n\nGot:\n~~~~~~~~~\nIn [1]: from IPython.config.application import Application\nIn [2]: app = Application.instance()\nIn [3]: app.pylab_import_all = False\nIn [4]: pylab\n/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:108: ResourceWarning: unclosed file <_io.BufferedReader name=\'/usr/lib64/python3.3/site-packages/PyQt4/QtCore.so\'>\n  imp.find_module(\'QtCore\', mod.__path__)\n/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:109: ResourceWarning: unclosed file <_io.BufferedReader name=\'/usr/lib64/python3.3/site-packages/PyQt4/QtGui.so\'>\n  imp.find_module(\'QtGui\', mod.__path__)\n/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:110: ResourceWarning: unclosed file <_io.BufferedReader name=\'/usr/lib64/python3.3/site-packages/PyQt4/QtSvg.so\'>\n  imp.find_module(\'QtSvg\', mod.__path__)\nFontconfig warning: "/etc/fonts/conf.d/50-user.conf", line 14: reading configurations from ~/.fonts.conf is deprecated.\nUsing matplotlib backend: Qt4Agg\nPopulating the interactive namespace from numpy and matplotlib\nIn [5]: ip=get_ipython()\nIn [6]: \'plot\' in ip.user_ns\nOut[6]: False')


======================================================================
FAIL: Verify that plot is available when pylab_import_all = True
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/decorators.py", line 233, in skipper_func
    return f(*args, **kwargs)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/decorators.py", line 233, in skipper_func
    return f(*args, **kwargs)
  File "/tmp/ipython-1.0.0/build/lib/IPython/lib/tests/test_irunner_pylab_magic.py", line 93, in test_pylab_import_all_enabled
    self._test_runner(runner,source,output)
  File "/tmp/ipython-1.0.0/build/lib/IPython/lib/tests/test_irunner_pylab_magic.py", line 54, in _test_runner
    self.fail(message)
nose.proxy.AssertionError: Mismatch in number of lines

Expected:
~~~~~~~~~ 
In \[1\]: from IPython\.config\.application import Application
In \[2\]: app = Application\.instance\(\)
In \[3\]: app\.pylab_import_all = True
In \[4\]: pylab
^Using matplotlib backend:
Populating the interactive namespace from numpy and matplotlib
In \[5\]: ip=get_ipython\(\)
In \[6\]: 'plot' in ip\.user_ns
Out\[6\]: True

Got:
~~~~~~~~~ 
In [1]: from IPython.config.application import Application
In [2]: app = Application.instance()
In [3]: app.pylab_import_all = True
In [4]: pylab
/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:108: ResourceWarning: unclosed file <_io.BufferedReader name='/usr/lib64/python3.3/site-packages/PyQt4/QtCore.so'>
  imp.find_module('QtCore', mod.__path__)
/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:109: ResourceWarning: unclosed file <_io.BufferedReader name='/usr/lib64/python3.3/site-packages/PyQt4/QtGui.so'>
  imp.find_module('QtGui', mod.__path__)
/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:110: ResourceWarning: unclosed file <_io.BufferedReader name='/usr/lib64/python3.3/site-packages/PyQt4/QtSvg.so'>
  imp.find_module('QtSvg', mod.__path__)
Fontconfig warning: "/etc/fonts/conf.d/50-user.conf", line 14: reading configurations from ~/.fonts.conf is deprecated.
Using matplotlib backend: Qt4Agg
Populating the interactive namespace from numpy and matplotlib
In [5]: ip=get_ipython()
In [6]: 'plot' in ip.user_ns
Out[6]: True
    """Fail immediately, with the given message."""
>>  raise self.failureException('Mismatch in number of lines\n\nExpected:\n~~~~~~~~~\nIn \\[1\\]: from IPython\\.config\\.application import Application\nIn \\[2\\]: app = Application\\.instance\\(\\)\nIn \\[3\\]: app\\.pylab_import_all = True\nIn \\[4\\]: pylab\n^Using matplotlib backend:\nPopulating the interactive namespace from numpy and matplotlib\nIn \\[5\\]: ip=get_ipython\\(\\)\nIn \\[6\\]: \'plot\' in ip\\.user_ns\nOut\\[6\\]: True\n\nGot:\n~~~~~~~~~\nIn [1]: from IPython.config.application import Application\nIn [2]: app = Application.instance()\nIn [3]: app.pylab_import_all = True\nIn [4]: pylab\n/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:108: ResourceWarning: unclosed file <_io.BufferedReader name=\'/usr/lib64/python3.3/site-packages/PyQt4/QtCore.so\'>\n  imp.find_module(\'QtCore\', mod.__path__)\n/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:109: ResourceWarning: unclosed file <_io.BufferedReader name=\'/usr/lib64/python3.3/site-packages/PyQt4/QtGui.so\'>\n  imp.find_module(\'QtGui\', mod.__path__)\n/tmp/ipython-1.0.0/build/lib/IPython/external/qt_loaders.py:110: ResourceWarning: unclosed file <_io.BufferedReader name=\'/usr/lib64/python3.3/site-packages/PyQt4/QtSvg.so\'>\n  imp.find_module(\'QtSvg\', mod.__path__)\nFontconfig warning: "/etc/fonts/conf.d/50-user.conf", line 14: reading configurations from ~/.fonts.conf is deprecated.\nUsing matplotlib backend: Qt4Agg\nPopulating the interactive namespace from numpy and matplotlib\nIn [5]: ip=get_ipython()\nIn [6]: \'plot\' in ip.user_ns\nOut[6]: True')


----------------------------------------------------------------------
Ran 59 tests in 17.220s

FAILED (failures=2)
...
**********************************************************************
IPython test group: IPython.testing
...
======================================================================
ERROR: Test with only stdout results.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tests/test_tools.py", line 100, in test_main_path
    tt.ipexec_validate(self.fname, out)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmprdjxb9.py' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

======================================================================
ERROR: Test with only stdout results, expecting windows line endings.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tests/test_tools.py", line 109, in test_main_path2
    tt.ipexec_validate(self.fname, out)
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 264, in ipexec_validate
    (fname, err))
ValueError: Running file '/tmp/tmp7hwr2o.py' produced error: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n  r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n"

======================================================================
FAIL: Test exception path in exception_validate.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tests/test_tools.py", line 122, in test_exception_path
    tt.ipexec_validate(self.fname, expected_out=out, expected_err="C\nD")
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 261, in ipexec_validate
    nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
nose.proxy.AssertionError: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarnin [truncated]... != 'C\nD'
- /tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead
-   r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))
  C
  D
    """Fail immediately, with the given message."""
>>  raise self.failureException('"/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarnin [truncated]... != \'C\\nD\'\n- /tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n-   r\'|\'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n  C\n  D')


======================================================================
FAIL: Test exception path in exception_validate, expecting windows line endings.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tests/test_tools.py", line 135, in test_exception_path2
    tt.ipexec_validate(self.fname, expected_out=out, expected_err="C\r\nD")
  File "/tmp/ipython-1.0.0/build/lib/IPython/testing/tools.py", line 261, in ipexec_validate
    nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
nose.proxy.AssertionError: "/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarnin [truncated]... != 'C\nD'
- /tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead
-   r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))
  C
  D
    """Fail immediately, with the given message."""
>>  raise self.failureException('"/tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarnin [truncated]... != \'C\\nD\'\n- /tmp/ipython-1.0.0/build/lib/IPython/core/completerlib.py:53: DeprecationWarning: imp.get_suffixes() is deprecated; use the constants defined on importlib.machinery instead\n-   r\'|\'.join(re.escape(s[0]) for s in imp.get_suffixes()))\n  C\n  D')


----------------------------------------------------------------------
Ran 57 tests in 6.920s

FAILED (SKIP=3, errors=2, failures=2)
```



Discussion:

Resolving patch:
diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py
--- a/IPython/core/completerlib.py
+++ b/IPython/core/completerlib.py
@@ -17,12 +17,19 @@
 
 # Stdlib imports
 import glob
-import imp
 import inspect
 import os
 import re
 import sys
 
+try:
+    # Python 3
+    from importlib.machinery import all_suffixes
+    _suffixes = all_suffixes()
+except ImportError:
+    from imp import get_suffixes
+    _suffixes = [ s[0] for s in get_suffixes() ]
+
 # Third-party imports
 from time import time
 from zipimport import zipimporter
@@ -51,7 +58,7 @@
 import_re = re.compile(r'(?P<name>[a-zA-Z_][a-zA-Z0-9_]*?)'
                        r'(?P<package>[/\\]__init__)?'
                        r'(?P<suffix>%s)$' %
-                       r'|'.join(re.escape(s[0]) for s in imp.get_suffixes()))
+                       r'|'.join(re.escape(s) for s in _suffixes))
 
 # RE for the ipython %run command (python + ipython scripts)
 magic_run_re = re.compile(r'.*(\.ipy|\.ipynb|\.py[w]?)$')


Test patch: