Repository: ipython/ipython

Issue:
Ipcontroller broken in current HEAD on windows
I get the below error when running ipcontroller on windows 64 bit, using the latest HEAD. This error does not occur on the 1.1.0 stable release. This may be related to  #4402.

```
PS Z:\> ipcontroller
2013-10-28 09:50:05.732 [IPControllerApp] Using existing profile dir: u'C:\\Users\\t-aadefa\\.ipython\\profile_default'
2013-10-28 09:50:05.759 [IPControllerApp] Hub listening on tcp://127.0.0.1:48991 for registration.
2013-10-28 09:50:05.763 [IPControllerApp] Hub using DB backend: 'NoDB'
2013-10-28 09:50:06.017 [IPControllerApp] hub::created hub
2013-10-28 09:50:06.019 [IPControllerApp] WARNING | Could not identify this machine's IP, assuming 127.0.0.1. You may need to specify '--location=<external_ip_a
ddress>' to help IPython decide when to connect via loopback.
2013-10-28 09:50:06.020 [IPControllerApp] writing connection info to C:\Users\t-aadefa\.ipython\profile_default\security\ipcontroller-client.json
2013-10-28 09:50:06.026 [IPControllerApp] WARNING | Could not identify this machine's IP, assuming 127.0.0.1. You may need to specify '--location=<external_ip_a
ddress>' to help IPython decide when to connect via loopback.
2013-10-28 09:50:06.026 [IPControllerApp] writing connection info to C:\Users\t-aadefa\.ipython\profile_default\security\ipcontroller-engine.json
2013-10-28 09:50:06.032 [IPControllerApp] task::using Python leastload Task scheduler
2013-10-28 09:50:06.032 [IPControllerApp] Heartmonitor started
Traceback (most recent call last):
  File "C:\Python27\Scripts\ipcontroller-script.py", line 9, in <module>
    load_entry_point('ipython==2.0.0-dev', 'console_scripts', 'ipcontroller')()
  File "C:\Python27\lib\site-packages\ipython-2.0.0_dev-py2.7.egg\IPython\parallel\apps\ipcontrollerapp.py", line 543, in launch_new_instance
    return IPControllerApp.launch_instance(*args, **kwargs)
  File "C:\Python27\lib\site-packages\ipython-2.0.0_dev-py2.7.egg\IPython\config\application.py", line 543, in launch_instance
    app.start()
  File "C:\Python27\lib\site-packages\ipython-2.0.0_dev-py2.7.egg\IPython\parallel\apps\ipcontrollerapp.py", line 514, in start
    child.start()
  File "C:\Python27\lib\multiprocessing\process.py", line 130, in start
    self._popen = Popen(self)
  File "C:\Python27\lib\multiprocessing\forking.py", line 277, in __init__
    dump(process_obj, to_child, HIGHEST_PROTOCOL)
  File "C:\Python27\lib\multiprocessing\forking.py", line 199, in dump
    ForkingPickler(file, protocol).dump(obj)
  File "C:\Python27\lib\pickle.py", line 224, in dump
    self.save(obj)
  File "C:\Python27\lib\pickle.py", line 331, in save
    self.save_reduce(obj=obj, *rv)
  File "C:\Python27\lib\pickle.py", line 419, in save_reduce
    save(state)
  File "C:\Python27\lib\pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "C:\Python27\lib\pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "C:\Python27\lib\pickle.py", line 681, in _batch_setitems
    save(v)
  File "C:\Python27\lib\pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "C:\Python27\lib\pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "C:\Python27\lib\pickle.py", line 681, in _batch_setitems
    save(v)
  File "C:\Python27\lib\pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "C:\Python27\lib\pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "C:\Python27\lib\pickle.py", line 681, in _batch_setitems
    save(v)
  File "C:\Python27\lib\pickle.py", line 306, in save
    rv = reduce(self.proto)
TypeError: 'LazyConfigValue' object is not callable

If you suspect this is an IPython bug, please report it at:
    https://github.com/ipython/ipython/issues
or send an email to the mailing list at ipython-dev@scipy.org

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
    c.Application.verbose_crash=True

PS Z:\> Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Python27\lib\multiprocessing\forking.py", line 381, in main
    self = load(from_parent)
  File "C:\Python27\lib\pickle.py", line 1378, in load
    return Unpickler(file).load()
  File "C:\Python27\lib\pickle.py", line 858, in load
    dispatch[key](self)
  File "C:\Python27\lib\pickle.py", line 880, in load_eof
    raise EOFError
EOFError
```



Discussion:
Probably due to #4276 .

@adefazio what is your config?

I have not done any configuration of ipython, as far as I can remember. The error occurs with the default cluster profile.

Okay, I'll give it a look.


Resolving patch:
diff --git a/IPython/config/loader.py b/IPython/config/loader.py
--- a/IPython/config/loader.py
+++ b/IPython/config/loader.py
@@ -32,7 +32,7 @@
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils import py3compat, warn
 from IPython.utils.encoding import DEFAULT_ENCODING
-from IPython.utils.py3compat import builtin_mod, unicode_type, iteritems
+from IPython.utils.py3compat import unicode_type, iteritems
 from IPython.utils.traitlets import HasTraits, List, Any, TraitError
 
 #-----------------------------------------------------------------------------
@@ -163,14 +163,19 @@ def to_dict(self):
         return d
 
 
+def _is_section_key(key):
+    """Is a Config key a section name (does it start with a capital)?"""
+    if key and key[0].upper()==key[0] and not key.startswith('_'):
+        return True
+    else:
+        return False
+
+
 class Config(dict):
     """An attribute based dict that can do smart merges."""
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
-        # This sets self.__dict__ = self, but it has to be done this way
-        # because we are also overriding __setattr__.
-        dict.__setattr__(self, '__dict__', self)
         self._ensure_subconfig()
     
     def _ensure_subconfig(self):
@@ -181,10 +186,10 @@ def _ensure_subconfig(self):
         """
         for key in self:
             obj = self[key]
-            if self._is_section_key(key) \
+            if _is_section_key(key) \
                     and isinstance(obj, dict) \
                     and not isinstance(obj, Config):
-                dict.__setattr__(self, key, Config(obj))
+                setattr(self, key, Config(obj))
     
     def _merge(self, other):
         """deprecated alias, use Config.merge()"""
@@ -195,23 +200,17 @@ def merge(self, other):
         to_update = {}
         for k, v in iteritems(other):
             if k not in self:
-                to_update[k] = v
+                to_update[k] = copy.deepcopy(v)
             else: # I have this key
                 if isinstance(v, Config) and isinstance(self[k], Config):
                     # Recursively merge common sub Configs
                     self[k].merge(v)
                 else:
                     # Plain updates for non-Configs
-                    to_update[k] = v
+                    to_update[k] = copy.deepcopy(v)
 
         self.update(to_update)
 
-    def _is_section_key(self, key):
-        if key[0].upper()==key[0] and not key.startswith('_'):
-            return True
-        else:
-            return False
-
     def __contains__(self, key):
         # allow nested contains of the form `"Section.key" in config`
         if '.' in key:
@@ -220,20 +219,14 @@ def __contains__(self, key):
                 return False
             return remainder in self[first]
         
-        # we always have Sections
-        if self._is_section_key(key):
-            return True
-        else:
-            return super(Config, self).__contains__(key)
+        return super(Config, self).__contains__(key)
+    
     # .has_key is deprecated for dictionaries.
     has_key = __contains__
-
+    
     def _has_section(self, key):
-        if self._is_section_key(key):
-            if super(Config, self).__contains__(key):
-                return True
-        return False
-
+        return _is_section_key(key) and key in self
+    
     def copy(self):
         return type(self)(dict.copy(self))
 
@@ -245,57 +238,45 @@ def __deepcopy__(self, memo):
         return type(self)(copy.deepcopy(list(self.items())))
     
     def __getitem__(self, key):
-        # We cannot use directly self._is_section_key, because it triggers
-        # infinite recursion on top of PyPy. Instead, we manually fish the
-        # bound method.
-        is_section_key = self.__class__._is_section_key.__get__(self)
-
-        # Because we use this for an exec namespace, we need to delegate
-        # the lookup of names in __builtin__ to itself.  This means
-        # that you can't have section or attribute names that are
-        # builtins.
         try:
-            return getattr(builtin_mod, key)
-        except AttributeError:
-            pass
-        if is_section_key(key):
-            try:
-                return dict.__getitem__(self, key)
-            except KeyError:
+            return dict.__getitem__(self, key)
+        except KeyError:
+            if _is_section_key(key):
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
-        else:
-            try:
-                return dict.__getitem__(self, key)
-            except KeyError:
-                # undefined
+            else:
+                # undefined, create lazy value, used for container methods
                 v = LazyConfigValue()
                 dict.__setitem__(self, key, v)
                 return v
-            
 
     def __setitem__(self, key, value):
-        if self._is_section_key(key):
+        if _is_section_key(key):
             if not isinstance(value, Config):
                 raise ValueError('values whose keys begin with an uppercase '
                                  'char must be Config instances: %r, %r' % (key, value))
-        else:
-            dict.__setitem__(self, key, value)
+        dict.__setitem__(self, key, value)
 
     def __getattr__(self, key):
+        if key.startswith('__'):
+            return dict.__getattr__(self, key)
         try:
             return self.__getitem__(key)
         except KeyError as e:
             raise AttributeError(e)
 
     def __setattr__(self, key, value):
+        if key.startswith('__'):
+            return dict.__setattr__(self, key, value)
         try:
             self.__setitem__(key, value)
         except KeyError as e:
             raise AttributeError(e)
 
     def __delattr__(self, key):
+        if key.startswith('__'):
+            return dict.__delattr__(self, key)
         try:
             dict.__delitem__(self, key)
         except KeyError as e:
diff --git a/IPython/nbconvert/exporters/exporter.py b/IPython/nbconvert/exporters/exporter.py
--- a/IPython/nbconvert/exporters/exporter.py
+++ b/IPython/nbconvert/exporters/exporter.py
@@ -83,12 +83,12 @@ def __init__(self, config=None, **kw):
         config : config
             User configuration instance.
         """
-        if not config:
-            config = self.default_config
+        with_default_config = self.default_config
+        if config:
+            with_default_config.merge(config)
+        
+        super(Exporter, self).__init__(config=with_default_config, **kw)
 
-        super(Exporter, self).__init__(config=config, **kw)
-
-        #Init
         self._init_preprocessors()
 
 
@@ -96,15 +96,6 @@ def __init__(self, config=None, **kw):
     def default_config(self):
         return Config()
 
-    def _config_changed(self, name, old, new):
-        """When setting config, make sure to start with our default_config"""
-        c = self.default_config
-        if new:
-            c.merge(new)
-        if c != old:
-            self.config = c
-        super(Exporter, self)._config_changed(name, old, c)
-
 
     def from_notebook_node(self, nb, resources=None, **kw):
         """
diff --git a/IPython/nbconvert/exporters/templateexporter.py b/IPython/nbconvert/exporters/templateexporter.py
--- a/IPython/nbconvert/exporters/templateexporter.py
+++ b/IPython/nbconvert/exporters/templateexporter.py
@@ -140,10 +140,7 @@ def __init__(self, config=None, extra_loaders=None, **kw):
         template : str (optional, kw arg)
             Template to use when exporting.
         """
-        if not config:
-            config = self.default_config
-        
-        super(Exporter, self).__init__(config=config, **kw)
+        super(TemplateExporter, self).__init__(config=config, **kw)
 
         #Init
         self._init_template()


Test patch:
diff --git a/IPython/config/tests/test_configurable.py b/IPython/config/tests/test_configurable.py
--- a/IPython/config/tests/test_configurable.py
+++ b/IPython/config/tests/test_configurable.py
@@ -357,4 +357,19 @@ def test_dict_update(self):
         obj = Containers(config=c)
         self.assertEqual(obj.d, {'a':'b', 'c':'d', 'e':'f'})
     
+    def test_update_twice(self):
+        c = Config()
+        c.MyConfigurable.a = 5
+        m = MyConfigurable(config=c)
+        self.assertEqual(m.a, 5)
+        
+        c2 = Config()
+        c2.MyConfigurable.a = 10
+        m.update_config(c2)
+        self.assertEqual(m.a, 10)
+        
+        c2.MyConfigurable.a = 15
+        m.update_config(c2)
+        self.assertEqual(m.a, 15)
+    
 
diff --git a/IPython/config/tests/test_loader.py b/IPython/config/tests/test_loader.py
--- a/IPython/config/tests/test_loader.py
+++ b/IPython/config/tests/test_loader.py
@@ -20,6 +20,7 @@
 #-----------------------------------------------------------------------------
 
 import os
+import pickle
 import sys
 from tempfile import mkstemp
 from unittest import TestCase
@@ -30,11 +31,12 @@
 
 from IPython.config.loader import (
     Config,
+    LazyConfigValue,
     PyFileConfigLoader,
     KeyValueConfigLoader,
     ArgParseConfigLoader,
     KVArgParseConfigLoader,
-    ConfigError
+    ConfigError,
 )
 
 #-----------------------------------------------------------------------------
@@ -206,14 +208,15 @@ def test_setget(self):
 
     def test_auto_section(self):
         c = Config()
-        self.assertEqual('A' in c, True)
-        self.assertEqual(c._has_section('A'), False)
+        self.assertNotIn('A', c)
+        assert not c._has_section('A')
         A = c.A
         A.foo = 'hi there'
-        self.assertEqual(c._has_section('A'), True)
+        self.assertIn('A', c)
+        assert c._has_section('A')
         self.assertEqual(c.A.foo, 'hi there')
         del c.A
-        self.assertEqual(len(c.A.keys()),0)
+        self.assertEqual(c.A, Config())
 
     def test_merge_doesnt_exist(self):
         c1 = Config()
@@ -256,8 +259,6 @@ def test_deepcopy(self):
 
     def test_builtin(self):
         c1 = Config()
-        exec('foo = True', c1)
-        self.assertEqual(c1.foo, True)
         c1.format = "json"
     
     def test_fromdict(self):
@@ -289,4 +290,48 @@ def test_contains(self):
         self.assertIn('Foo.bar', c2)
         self.assertNotIn('Foo.bar', c1)
     
+    def test_pickle_config(self):
+        cfg = Config()
+        cfg.Foo.bar = 1
+        pcfg = pickle.dumps(cfg)
+        cfg2 = pickle.loads(pcfg)
+        self.assertEqual(cfg2, cfg)
+    
+    def test_getattr_section(self):
+        cfg = Config()
+        self.assertNotIn('Foo', cfg)
+        Foo = cfg.Foo
+        assert isinstance(Foo, Config)
+        self.assertIn('Foo', cfg)
+
+    def test_getitem_section(self):
+        cfg = Config()
+        self.assertNotIn('Foo', cfg)
+        Foo = cfg['Foo']
+        assert isinstance(Foo, Config)
+        self.assertIn('Foo', cfg)
+
+    def test_getattr_not_section(self):
+        cfg = Config()
+        self.assertNotIn('foo', cfg)
+        foo = cfg.foo
+        assert isinstance(foo, LazyConfigValue)
+        self.assertIn('foo', cfg)
+
+    def test_getitem_not_section(self):
+        cfg = Config()
+        self.assertNotIn('foo', cfg)
+        foo = cfg['foo']
+        assert isinstance(foo, LazyConfigValue)
+        self.assertIn('foo', cfg)
+    
+    def test_merge_copies(self):
+        c = Config()
+        c2 = Config()
+        c2.Foo.trait = []
+        c.merge(c2)
+        c2.Foo.trait.append(1)
+        self.assertIsNot(c.Foo, c2.Foo)
+        self.assertEqual(c.Foo.trait, [])
+        self.assertEqual(c2.Foo.trait, [1])