Repository: ipython/ipython

Issue:
Deep reload throws a NonImplementedError
I tried to deep reload module `types`, but got the following error. This kind of error does not occur in Python 3.4.2, but occurs in Python 3.5.2 and Python 3.6.0.

<details>
<summary>Long and winding traceback...</summary>

```
Python 3.5.2 (default, Oct 31 2016, 21:20:07) 
Type "copyright", "credits" or "license" for more information.

IPython 5.2.1 -- An enhanced Interactive Python.
?         -> Introduction and overview of IPython's features.
%quickref -> Quick reference.
help      -> Python's own help system.
object?   -> Details about 'object', use 'object??' for extra details.

In [1]: import types
In [2]: from IPython.lib.deepreload import reload
In [3]: reload(types)
Reloading functools
Reloading _functools
Reloading abc
Reloading _weakrefset
Reloading _weakref
Reloading collections
Reloading _collections_abc
Reloading operator
Reloading _operator
Reloading keyword
Reloading heapq
Reloading _heapq
Reloading importlib
Reloading _imp
Reloading _frozen_importlib
Reloading _frozen_importlib_external
Reloading types
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
<ipython-input-3-2c2c5ada3170> in <module>()
----> 1 reload(types)

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in reload(module, exclude)
    340     try:
    341         with replace_import_hook(deep_import_hook):
--> 342             return deep_reload_hook(module)
    343     finally:
    344         found_now = {}

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/contextlib.py in __exit__(self, type, value, traceback)
     75                 value = type()
     76             try:
---> 77                 self.gen.throw(type, value, traceback)
     78                 raise RuntimeError("generator didn't stop after throw()")
     79             except StopIteration as exc:

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in replace_import_hook(new_import)
     50     builtin_mod.__import__ = new_import
     51     try:
---> 52         yield
     53     finally:
     54         builtin_mod.__import__ = saved_import

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in reload(module, exclude)
    340     try:
    341         with replace_import_hook(deep_import_hook):
--> 342             return deep_reload_hook(module)
    343     finally:
    344         found_now = {}

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in deep_reload_hook(m)
    309 
    310     try:
--> 311         newm = imp.load_module(name, fp, filename, stuff)
    312     except:
    313          # load_module probably removed name from modules because of

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_module(name, file, filename, details)
    232         raise ValueError(msg)
    233     elif type_ == PY_SOURCE:
--> 234         return load_source(name, filename, file)
    235     elif type_ == PY_COMPILED:
    236         return load_compiled(name, filename, file)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_source(name, pathname, file)
    168     spec = util.spec_from_file_location(name, pathname, loader=loader)
    169     if name in sys.modules:
--> 170         module = _exec(spec, sys.modules[name])
    171     else:
    172         module = _load(spec)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _exec(spec, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap_external.py in exec_module(self, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/types.py in <module>()
    164 
    165 
--> 166 import functools as _functools
    167 import collections.abc as _collections_abc
    168 

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in deep_import_hook(name, globals, locals, fromlist, level)
    250     parent, buf = get_parent(globals, level)
    251 
--> 252     head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
    253 
    254     tail = head

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in load_next(mod, altmod, name, buf)
    154     buf += subname
    155 
--> 156     result = import_submodule(mod, subname, buf)
    157     if result is None and mod != altmod:
    158         result = import_submodule(altmod, subname, subname)

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in import_submodule(mod, subname, fullname)
    199 
    200         try:
--> 201             m = imp.load_module(fullname, fp, filename, stuff)
    202         except:
    203             # load_module probably removed name from modules because of

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_module(name, file, filename, details)
    232         raise ValueError(msg)
    233     elif type_ == PY_SOURCE:
--> 234         return load_source(name, filename, file)
    235     elif type_ == PY_COMPILED:
    236         return load_compiled(name, filename, file)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_source(name, pathname, file)
    168     spec = util.spec_from_file_location(name, pathname, loader=loader)
    169     if name in sys.modules:
--> 170         module = _exec(spec, sys.modules[name])
    171     else:
    172         module = _load(spec)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _exec(spec, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap_external.py in exec_module(self, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/functools.py in <module>()
     19     pass
     20 from abc import get_cache_token
---> 21 from collections import namedtuple
     22 from types import MappingProxyType
     23 from weakref import WeakKeyDictionary

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in deep_import_hook(name, globals, locals, fromlist, level)
    250     parent, buf = get_parent(globals, level)
    251 
--> 252     head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
    253 
    254     tail = head

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in load_next(mod, altmod, name, buf)
    154     buf += subname
    155 
--> 156     result = import_submodule(mod, subname, buf)
    157     if result is None and mod != altmod:
    158         result = import_submodule(altmod, subname, subname)

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in import_submodule(mod, subname, fullname)
    199 
    200         try:
--> 201             m = imp.load_module(fullname, fp, filename, stuff)
    202         except:
    203             # load_module probably removed name from modules because of

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_module(name, file, filename, details)
    242             return load_dynamic(name, filename, file)
    243     elif type_ == PKG_DIRECTORY:
--> 244         return load_package(name, filename)
    245     elif type_ == C_BUILTIN:
    246         return init_builtin(name)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_package(name, path)
    212                                         submodule_search_locations=[])
    213     if name in sys.modules:
--> 214         return _exec(spec, sys.modules[name])
    215     else:
    216         return _load(spec)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _exec(spec, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap_external.py in exec_module(self, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/collections/__init__.py in <module>()
     27 from keyword import iskeyword as _iskeyword
     28 import sys as _sys
---> 29 import heapq as _heapq
     30 from _weakref import proxy as _proxy
     31 from itertools import repeat as _repeat, chain as _chain, starmap as _starmap

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in deep_import_hook(name, globals, locals, fromlist, level)
    250     parent, buf = get_parent(globals, level)
    251 
--> 252     head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
    253 
    254     tail = head

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in load_next(mod, altmod, name, buf)
    154     buf += subname
    155 
--> 156     result = import_submodule(mod, subname, buf)
    157     if result is None and mod != altmod:
    158         result = import_submodule(altmod, subname, subname)

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in import_submodule(mod, subname, fullname)
    199 
    200         try:
--> 201             m = imp.load_module(fullname, fp, filename, stuff)
    202         except:
    203             # load_module probably removed name from modules because of

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_module(name, file, filename, details)
    232         raise ValueError(msg)
    233     elif type_ == PY_SOURCE:
--> 234         return load_source(name, filename, file)
    235     elif type_ == PY_COMPILED:
    236         return load_compiled(name, filename, file)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_source(name, pathname, file)
    168     spec = util.spec_from_file_location(name, pathname, loader=loader)
    169     if name in sys.modules:
--> 170         module = _exec(spec, sys.modules[name])
    171     else:
    172         module = _load(spec)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _exec(spec, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap_external.py in exec_module(self, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/heapq.py in <module>()
    585 # If available, use C implementation
    586 try:
--> 587     from _heapq import *
    588 except ImportError:
    589     pass

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in deep_import_hook(name, globals, locals, fromlist, level)
    250     parent, buf = get_parent(globals, level)
    251 
--> 252     head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
    253 
    254     tail = head

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in load_next(mod, altmod, name, buf)
    154     buf += subname
    155 
--> 156     result = import_submodule(mod, subname, buf)
    157     if result is None and mod != altmod:
    158         result = import_submodule(altmod, subname, subname)

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in import_submodule(mod, subname, fullname)
    199 
    200         try:
--> 201             m = imp.load_module(fullname, fp, filename, stuff)
    202         except:
    203             # load_module probably removed name from modules because of

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_module(name, file, filename, details)
    240                 return load_dynamic(name, filename, opened_file)
    241         else:
--> 242             return load_dynamic(name, filename, file)
    243     elif type_ == PKG_DIRECTORY:
    244         return load_package(name, filename)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_dynamic(name, path, file)
    333         Load an extension module.
    334         """
--> 335         import importlib.machinery
    336         loader = importlib.machinery.ExtensionFileLoader(name, path)
    337 

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in deep_import_hook(name, globals, locals, fromlist, level)
    250     parent, buf = get_parent(globals, level)
    251 
--> 252     head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
    253 
    254     tail = head

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in load_next(mod, altmod, name, buf)
    154     buf += subname
    155 
--> 156     result = import_submodule(mod, subname, buf)
    157     if result is None and mod != altmod:
    158         result = import_submodule(altmod, subname, subname)

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in import_submodule(mod, subname, fullname)
    199 
    200         try:
--> 201             m = imp.load_module(fullname, fp, filename, stuff)
    202         except:
    203             # load_module probably removed name from modules because of

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_module(name, file, filename, details)
    242             return load_dynamic(name, filename, file)
    243     elif type_ == PKG_DIRECTORY:
--> 244         return load_package(name, filename)
    245     elif type_ == C_BUILTIN:
    246         return init_builtin(name)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_package(name, path)
    212                                         submodule_search_locations=[])
    213     if name in sys.modules:
--> 214         return _exec(spec, sys.modules[name])
    215     else:
    216         return _load(spec)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _exec(spec, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap_external.py in exec_module(self, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/__init__.py in <module>()
     55 # dependencies and startup overhead minimisation permitting :)
     56 
---> 57 import types
     58 import warnings
     59 

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in deep_import_hook(name, globals, locals, fromlist, level)
    250     parent, buf = get_parent(globals, level)
    251 
--> 252     head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
    253 
    254     tail = head

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in load_next(mod, altmod, name, buf)
    154     buf += subname
    155 
--> 156     result = import_submodule(mod, subname, buf)
    157     if result is None and mod != altmod:
    158         result = import_submodule(altmod, subname, subname)

/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython/lib/deepreload.py in import_submodule(mod, subname, fullname)
    199 
    200         try:
--> 201             m = imp.load_module(fullname, fp, filename, stuff)
    202         except:
    203             # load_module probably removed name from modules because of

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_module(name, file, filename, details)
    232         raise ValueError(msg)
    233     elif type_ == PY_SOURCE:
--> 234         return load_source(name, filename, file)
    235     elif type_ == PY_COMPILED:
    236         return load_compiled(name, filename, file)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/imp.py in load_source(name, pathname, file)
    168     spec = util.spec_from_file_location(name, pathname, loader=loader)
    169     if name in sys.modules:
--> 170         module = _exec(spec, sys.modules[name])
    171     else:
    172         module = _load(spec)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _exec(spec, module)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in _init_module_attrs(spec, module, override)

/home/sviatoslav/.pythonz/pythons/CPython-3.5.2/lib/python3.5/importlib/_bootstrap.py in cached(self)

NotImplementedError: 
```

System information:
```
$ python -c "import IPython; print(IPython.sys_info())"
{'commit_hash': 'be4827b',
 'commit_source': 'installation',
 'default_encoding': 'UTF-8',
 'ipython_path': '/home/sviatoslav/workspace/py35/lib/python3.5/site-packages/IPython',
 'ipython_version': '5.2.1',
 'os_name': 'posix',
 'platform': 'Linux-3.16.0-4-amd64-x86_64-with-debian-8.7',
 'sys_executable': '/home/sviatoslav/workspace/py35/bin/python',
 'sys_platform': 'linux',
 'sys_version': '3.5.2 (default, Oct 31 2016, 21:20:07) \n[GCC 4.9.2]'}
```

</details>


Discussion:
Hum, `types` and deeprelaod are pretty dark magic. We likely need to blacklist some modules.

Do you want to attempt a PR ?
> Do you want to attempt a PR ?

Sorry, I have no idea what's going on there.
> Sorry, I have no idea what's going on there.

No worries. We'll have a look ! Thanks. 

Resolving patch:
diff --git a/IPython/lib/deepreload.py b/IPython/lib/deepreload.py
--- a/IPython/lib/deepreload.py
+++ b/IPython/lib/deepreload.py
@@ -39,6 +39,7 @@
 
 from types import ModuleType
 from warnings import warn
+import types
 
 original_import = builtin_mod.__import__
 
@@ -162,6 +163,7 @@ def load_next(mod, altmod, name, buf):
 
     return result, next, buf
 
+
 # Need to keep track of what we've already reloaded to prevent cyclic evil
 found_now = {}
 
@@ -269,6 +271,12 @@ def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
 
 def deep_reload_hook(m):
     """Replacement for reload()."""
+    # Hardcode this one  as it would raise a NotImplemeentedError from the
+    # bowels of Python and screw up the import machinery after.
+    # unlike other imports the `exclude` list aleady in place is not enough.
+
+    if m is types:
+        return m
     if not isinstance(m, ModuleType):
         raise TypeError("reload() argument must be module")
 
@@ -319,7 +327,7 @@ def deep_reload_hook(m):
     return newm
 
 # Save the original hooks
-original_reload = imp.reload    # Python 3
+original_reload = imp.reload
 
 # Replacement for reload()
 def reload(module, exclude=('sys', 'os.path', 'builtins', '__main__',


Test patch: