Repository: Pyomo/pyomo

Issue:
Merge PySPConfigBlock back into pyutilib.misc.config
In an effort to simplify / standardize configuration / options in Pyomo, we should ensure that we only use a *single* configuration management system.  The customizations / overrides in `PySPConfigBlock` and `PySPConfigValue` should be reviewed and, if appropriate, merged into `pyutilib.misc.config`.  PySP should then directly use the `pyutilib.misc.config` system.
Merge PySPConfigBlock back into pyutilib.misc.config
In an effort to simplify / standardize configuration / options in Pyomo, we should ensure that we only use a *single* configuration management system.  The customizations / overrides in `PySPConfigBlock` and `PySPConfigValue` should be reviewed and, if appropriate, merged into `pyutilib.misc.config`.  PySP should then directly use the `pyutilib.misc.config` system.
Merge PySPConfigBlock back into pyutilib.misc.config
In an effort to simplify / standardize configuration / options in Pyomo, we should ensure that we only use a *single* configuration management system.  The customizations / overrides in `PySPConfigBlock` and `PySPConfigValue` should be reviewed and, if appropriate, merged into `pyutilib.misc.config`.  PySP should then directly use the `pyutilib.misc.config` system.


Discussion:
Plus:
- the zipfile for supporting offline installations
- the zipfile containing the examples

A workaround has been implemented and these files are being published again.

Plus:
- the zipfile for supporting offline installations
- the zipfile containing the examples

A workaround has been implemented and these files are being published again.

Plus:
- the zipfile for supporting offline installations
- the zipfile containing the examples

A workaround has been implemented and these files are being published again.


Resolving patch:
diff --git a/pyomo/common/config.py b/pyomo/common/config.py
--- a/pyomo/common/config.py
+++ b/pyomo/common/config.py
@@ -29,6 +29,7 @@
 import builtins
 
 from pyomo.common.deprecation import deprecated
+from pyomo.common.modeling import NoArgumentGiven
 
 logger = logging.getLogger('pyomo.common.config')
 
@@ -164,7 +165,8 @@ def NonNegativeFloat(val):
 
 
 class In(object):
-    """Domain validation function admitting a Container of possible values
+    """In(domain, cast=None)
+    Domain validation class admitting a Container of possible values
 
     This will admit any value that is in the `domain` Container (i.e.,
     Container.__contains__() returns True).  Most common domains are
@@ -172,7 +174,26 @@ class In(object):
     first passed to `cast()` to convert them to the appropriate type
     before looking them up in `domain`.
 
+    Parameters
+    ----------
+    domain: Container
+        The container that specifies the allowable values.  Incoming
+        values are passed to ``domain.__contains__()``, and if ``True``
+        is returned, the value is accepted and returned.
+
+    cast: callable, optional
+        A callable object.  If specified, incoming values are first
+        passed to `cast`, and the resulting object is checked for
+        membership in `domain`
+
+    Note
+    ----
+    For backwards compatibility, `In` accepts `enum.Enum` classes as
+    `domain` Containers.  If the domain is an Enum, then the constructor
+    returns an instance of `InEnum`.
+
     """
+
     def __new__(cls, domain=None, cast=None):
         # Convenience: enum.Enum supported __contains__ through Python
         # 3.7.  If the domain is an Enum and cast is not specified,
@@ -186,6 +207,7 @@ def __init__(self, domain, cast=None):
         self._domain = domain
         self._cast = cast
 
+
     def __call__(self, value):
         if self._cast is not None:
             v = self._cast(value)
@@ -197,47 +219,70 @@ def __call__(self, value):
 
 
 class InEnum(object):
-    """Domain validation function admitting an enum value/name.
+    """Domain validation class admitting an enum value/name.
 
     This will admit any value that is in the specified Enum, including
     Enum members, values, and string names.  The incoming value will be
     automatically cast to an Enum member.
 
+    Parameters
+    ----------
+    domain: enum.Enum
+        The enum that incoming values should be mapped to
+
     """
     def __init__(self, domain):
         self._domain = domain
 
     def __call__(self, value):
-        v = self._cast_to_enum_domain(value)
-        if v in self._domain:
-            return v
-        raise ValueError("value %s not in domain %s" % (value, self._domain))
-
-    def _cast_to_enum_domain(self, arg):
         try:
-            # First check if the arg is a valid enum value
-            return self._domain(arg)
+            # First check if this is a valid enum value
+            return self._domain(value)
         except ValueError:
             # Assume this is a string and look it up
             try:
-                return self._domain[arg]
+                return self._domain[value]
             except KeyError:
                 pass
         raise ValueError("%r is not a valid %s" % (
-            arg, self._domain.__name__))
+            value, self._domain.__name__))
 
 
 class Path(object):
+    """Domain validator for path-like options.
+
+    This will admit any object and convert it to a string.  It will then
+    expand any environment variables and leading usernames (e.g.,
+    "~myuser" or "~/") appearing in either the value or the base path
+    before concatenating the base path and value, expanding the path to
+    an absolute path, and normalizing the path.
+
+    Parameters
+    ----------
+    basePath: None, str, ConfigValue
+        The base path that will be prepended to any non-absolute path
+        values provided.  If None, defaults to :py:attr:`Path.BasePath`.
+
+    expandPath: bool
+        If True, then the value will be expanded and normalized.  If
+        False, the string representation of the value will be returned
+        unchanged.  If None, expandPath will defer to the (negated)
+        value of :py:attr:`Path.SuppressPathExpansion`
+
+    """
     BasePath = None
     SuppressPathExpansion = False
 
-    def __init__(self, basePath=None):
+    def __init__(self, basePath=None, expandPath=None):
         self.basePath = basePath
+        self.expandPath = expandPath
 
     def __call__(self, path):
-        #print "normalizing path '%s' " % (path,),
         path = str(path)
-        if path is None or Path.SuppressPathExpansion:
+        _expand = self.expandPath
+        if _expand is None:
+            _expand = not Path.SuppressPathExpansion
+        if not _expand:
             return path
 
         if self.basePath:
@@ -259,13 +304,33 @@ def __call__(self, path):
             path = os.getcwd() + path[6:]
 
         ans = os.path.normpath(os.path.abspath(os.path.join(
-            os.path.expandvars(os.path.expanduser(base)),
-            os.path.expandvars(os.path.expanduser(path)))))
-        #print "to '%s'" % (ans,)
+            os.path.expanduser(os.path.expandvars(base)),
+            os.path.expanduser(os.path.expandvars(path)))))
         return ans
 
 
 class PathList(Path):
+    """Domain validator for a list of path-like objects.
+
+    This will admit any iterable or object convertable to a string.
+    Iterable objects (other than strings) will have each member
+    normalized using :py:class:`Path`.  Other types will be passed to
+    :py:class:`Path`, returning a list with the single resulting path.
+
+    Parameters
+    ----------
+    basePath: Union[None, str, ConfigValue]
+        The base path that will be prepended to any non-absolute path
+        values provided.  If None, defaults to :py:attr:`Path.BasePath`.
+
+    expandPath: bool
+        If True, then the value will be expanded and normalized.  If
+        False, the string representation of the value will be returned
+        unchanged.  If None, expandPath will defer to the (negated)
+        value of :py:attr:`Path.SuppressPathExpansion`
+
+    """
+
     def __call__(self, data):
         if hasattr(data, "__iter__") and not isinstance(data, str):
             return [ super(PathList, self).__call__(i) for i in data ]
@@ -317,7 +382,8 @@ def from_enum_or_string(cls, arg):
             return cls(arg)
 
 
-"""=================================
+__doc__ = """
+=================================
 The Pyomo Configuration System
 =================================
 
@@ -335,15 +401,10 @@ def from_enum_or_string(cls, arg):
 values for those entries, and retrieve the current values:
 
 .. doctest::
-    :hide:
 
-    >>> import argparse
     >>> from pyomo.common.config import (
     ...     ConfigDict, ConfigList, ConfigValue, In,
     ... )
-
-.. doctest::
-
     >>> config = ConfigDict()
     >>> config.declare('filename', ConfigValue(
     ...     default=None,
@@ -385,6 +446,9 @@ def from_enum_or_string(cls, arg):
     >>> print(config.iteration_limit)
     20
 
+Domain validation
+=================
+
 All Config objects support a ``domain`` keyword that accepts a callable
 object (type, function, or callable instance).  The domain callable
 should take data and map it onto the desired domain, optionally
@@ -401,6 +465,26 @@ def from_enum_or_string(cls, arg):
     >>> print(type(config.iteration_limit).__name__)
     int
 
+In addition to common types (like ``int``, ``float``, ``bool``, and
+``str``), the config system profides a number of custom domain
+validators for common use cases:
+
+.. autosummary::
+
+   PositiveInt
+   NegativeInt
+   NonNegativeInt
+   NonPositiveInt
+   PositiveFloat
+   NegativeFloat
+   NonPositiveFloat
+   NonNegativeFloat
+   In
+   InEnum
+   Path
+   PathList
+
+
 Configuring class hierarchies
 =============================
 
@@ -486,12 +570,14 @@ class will still create ``c`` instances that only have the single
 
 In addition to basic storage and retrieval, the Config system provides
 hooks to the argparse command-line argument parsing system.  Individual
-Config entries can be declared as argparse arguments.  To make
-declaration simpler, the :py:meth:`declare` method returns the declared Config
+Config entries can be declared as argparse arguments using the
+:py:meth:`~ConfigBase.declare_as_argument` method.  To make declaration
+simpler, the :py:meth:`declare` method returns the declared Config
 object so that the argument declaration can be done inline:
 
 .. doctest::
 
+    >>> import argparse
     >>> config = ConfigDict()
     >>> config.declare('iterlim', ConfigValue(
     ...     domain=int,
@@ -569,8 +655,8 @@ class will still create ``c`` instances that only have the single
 ===============================
 
 It is frequently useful to know which values a user explicitly set, and
-which values a user explicitly set, but have never been retrieved.  The
-configuration system provides two gemerator methods to return the items
+which values a user explicitly set but have never been retrieved.  The
+configuration system provides two generator methods to return the items
 that a user explicitly set (:py:meth:`user_values`) and the items that
 were set but never retrieved (:py:meth:`unused_user_values`):
 
@@ -675,23 +761,23 @@ class will still create ``c`` instances that only have the single
 
     >>> print(config.generate_documentation())
     \\begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em]
-      \\item[{output}]\hfill
+      \\item[{output}]\\hfill
         \\\\output results filename
-      \\item[{verbose}]\hfill
+      \\item[{verbose}]\\hfill
         \\\\This sets the system verbosity.  The default (0) only logs warnings and
         errors.  Larger integer values will produce additional log messages.
-      \\item[{solvers}]\hfill
+      \\item[{solvers}]\\hfill
         \\\\list of solvers to apply
       \\begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em]
-        \\item[{iterlim}]\hfill
+        \\item[{iterlim}]\\hfill
           \\\\iteration limit
-        \\item[{lbfgs}]\hfill
+        \\item[{lbfgs}]\\hfill
           \\\\use limited memory BFGS update
-        \\item[{linesearch}]\hfill
+        \\item[{linesearch}]\\hfill
           \\\\use line search
-        \\item[{relative tolerance}]\hfill
+        \\item[{relative tolerance}]\\hfill
           \\\\relative convergence tolerance
-        \\item[{absolute tolerance}]\hfill
+        \\item[{absolute tolerance}]\\hfill
           \\\\absolute convergence tolerance
       \\end{description}
     \\end{description}
@@ -721,7 +807,7 @@ def _munge_name(name, space_to_dash=True):
     return re.sub(r'[^a-zA-Z0-9-_]', '_', name)
 
 
-_leadingSpace = re.compile('^([ \n\t]*)')
+_leadingSpace = re.compile('^([ \t]*)')
 
 def _strip_indentation(doc):
     if not doc:
@@ -777,7 +863,7 @@ def __init__(self, obj):
 
     def __call__(self, arg):
         logging.error(
-"""%s %s was pickled with an unpicklable domain.
+"""%s '%s' was pickled with an unpicklable domain.
     The domain was stripped and lost during the pickle process.  Setting
     new values on the restored object cannot be mapped into the correct
     domain.
@@ -875,9 +961,10 @@ def __setstate__(self, state):
             # of setting self.__dict__[key] = val.
             object.__setattr__(self, key, val)
 
-    def __call__(self, value=NoArgument, default=NoArgument, domain=NoArgument,
-                 description=NoArgument, doc=NoArgument, visibility=NoArgument,
-                 implicit=NoArgument, implicit_domain=NoArgument,
+    def __call__(self, value=NoArgumentGiven, default=NoArgumentGiven,
+                 domain=NoArgumentGiven,  description=NoArgumentGiven,
+                 doc=NoArgumentGiven, visibility=NoArgumentGiven,
+                 implicit=NoArgumentGiven, implicit_domain=NoArgumentGiven,
                  preserve_implicit=False):
         # We will pass through overriding arguments to the constructor.
         # This way if the constructor does special processing of any of
@@ -886,47 +973,35 @@ def __call__(self, value=NoArgument, default=NoArgument, domain=NoArgument,
         # of logic to be sure we only pass through appropriate
         # arguments.
         kwds = {}
-        kwds['description'] = ( self._description
-                                if description is ConfigBase.NoArgument else
-                                description )
-        kwds['doc'] = ( self._doc
-                        if doc is ConfigBase.NoArgument else
-                        doc )
-        kwds['visibility'] = ( self._visibility
-                               if visibility is ConfigBase.NoArgument else
-                               visibility )
+        fields = ('description', 'doc', 'visibility')
         if isinstance(self, ConfigDict):
-            kwds['implicit'] = ( self._implicit_declaration
-                                 if implicit is ConfigBase.NoArgument else
-                                 implicit )
-            kwds['implicit_domain'] = (
-                self._implicit_domain
-                if implicit_domain is ConfigBase.NoArgument else
-                implicit_domain )
-            if domain is not ConfigBase.NoArgument:
-                logger.warn("domain ignored by __call__(): "
-                            "class is a ConfigDict" % (type(self),))
-            if default is not ConfigBase.NoArgument:
-                logger.warn("default ignored by __call__(): "
-                            "class is a ConfigDict" % (type(self),))
+            fields += (('implicit', '_implicit_declaration'), 'implicit_domain')
+            assert domain is NoArgumentGiven
+            assert default is NoArgumentGiven
         else:
-            kwds['default'] = ( self.value()
-                                if default is ConfigBase.NoArgument else
-                                default )
-            kwds['domain'] = ( self._domain
-                               if domain is ConfigBase.NoArgument else
-                               domain )
-            if implicit is not ConfigBase.NoArgument:
-                logger.warn("implicit ignored by __call__(): "
-                            "class %s is not a ConfigDict" % (type(self),))
-            if implicit_domain is not ConfigBase.NoArgument:
-                logger.warn("implicit_domain ignored by __call__(): "
-                            "class %s is not a ConfigDict" % (type(self),))
-
-        # Copy over any other object-specific information (mostly Dict
-        # definitions)
+            fields += ('domain',)
+            kwds['default'] = (
+                self.value() if default is NoArgumentGiven else default
+            )
+            assert implicit is NoArgumentGiven
+            assert implicit_domain is NoArgumentGiven
+        for field in fields:
+            if type(field) is tuple:
+                field, attr = field
+            else:
+                attr = '_'+field
+            if locals()[field] is NoArgumentGiven:
+                kwds[field] = getattr(self, attr, NoArgumentGiven)
+            else:
+                kwds[field] = locals()[field]
+
+        # Initialize the new config object
         ans = self.__class__(**kwds)
-        if isinstance(self, ConfigDict):
+
+        if not isinstance(self, ConfigDict):
+            ans.reset()
+        else:
+            # Copy over any Dict definitions
             for k in self._decl_order:
                 if preserve_implicit or k in self._declared:
                     v = self._data[k]
@@ -936,10 +1011,9 @@ def __call__(self, value=NoArgument, default=NoArgument, domain=NoArgument,
                         ans._declared.add(k)
                     _tmp._parent = ans
                     _tmp._name = v._name
-        else:
-            ans.reset()
+
         # ... and set the value, if appropriate
-        if value is not ConfigBase.NoArgument:
+        if value is not NoArgumentGiven:
             ans.set_value(value)
         return ans
 
@@ -969,7 +1043,7 @@ def _cast(self, value):
             return value
         if self._domain is not None:
             try:
-                if value is not ConfigBase.NoArgument:
+                if value is not NoArgumentGiven:
                     return self._domain(value)
                 else:
                     return self._domain()
@@ -1005,10 +1079,9 @@ def declare_as_argument(self, *args, **kwds):
 
         Valid arguments include all valid arguments to argparse's
         ArgumentParser.add_argument() with the exception of 'default'.
-        In addition, you may provide a group keyword argument can be
-        used to either pass in a pre-defined option group or subparser,
-        or else pass in the title of a group, subparser, or (subparser,
-        group).
+        In addition, you may provide a group keyword argument to either
+        pass in a pre-defined option group or subparser, or else pass in
+        the string name of a group, subparser, or (subparser, group).
 
         """
 
@@ -1123,18 +1196,15 @@ def display(self, content_filter=None, indent_spacing=2, ostream=None,
         if content_filter not in ConfigDict.content_filters:
             raise ValueError("unknown content filter '%s'; valid values are %s"
                              % (content_filter, ConfigDict.content_filters))
-
         _blocks = []
         if ostream is None:
             ostream=sys.stdout
 
         for lvl, prefix, value, obj in self._data_collector(0, "", visibility):
-            if content_filter == 'userdata' and not obj._userSet:
-                continue
-
             _str = _value2string(prefix, value, obj)
             _blocks[lvl:] = [' ' * indent_spacing * lvl + _str + "\n",]
-
+            if content_filter == 'userdata' and not obj._userSet:
+                continue
             for i, v in enumerate(_blocks):
                 if v is not None:
                     ostream.write(v)
@@ -1210,23 +1280,29 @@ def generate_yaml_template(self, indent_spacing=2, width=78, visibility=0):
             os.write('\n')
         return os.getvalue()
 
-    def generate_documentation\
-            ( self,
-              block_start= "\\begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em]\n",
-              block_end=   "\\end{description}\n",
-              item_start=  "\\item[{%s}]\\hfill\n",
-              item_body=   "\\\\%s",
-              item_end=    "",
-              indent_spacing=2,
-              width=78,
-              visibility=0
-              ):
+
+    def generate_documentation(
+            self, block_start=None, block_end=None,
+            item_start=None, item_body=None, item_end=None,
+            indent_spacing=2, width=78, visibility=0,
+            format='latex'):
+        _formats = ConfigBase.generate_documentation.formats
+        if block_start is None:
+            block_start = _formats.get(format, {}).get('block_start','')
+        if block_end is None:
+            block_end = _formats.get(format, {}).get('block_end','')
+        if item_start is None:
+            item_start = _formats.get(format, {}).get('item_start','')
+        if item_body is None:
+            item_body = _formats.get(format, {}).get('item_body','')
+        if item_end is None:
+            item_end = _formats.get(format, {}).get('item_end','')
+
         os = io.StringIO()
         level = []
         lastObj = self
         indent = ''
         for lvl, pre, val, obj in self._data_collector(1, '', visibility, True):
-            #print len(level), lvl, val, obj
             if len(level) < lvl:
                 while len(level) < lvl - 1:
                     level.append(None)
@@ -1297,6 +1373,16 @@ def unused_user_values(self):
             if obj._userSet and not obj._userAccessed:
                 yield obj
 
+ConfigBase.generate_documentation.formats = {
+    'latex': {
+        'block_start': "\\begin{description}["
+            "topsep=0pt,parsep=0.5em,itemsep=-0.4em]\n",
+        'block_end': "\\end{description}\n",
+        'item_start': "\\item[{%s}]\\hfill\n",
+        'item_body': "\\\\%s",
+        'item_end': "",
+    }
+}
 
 class ConfigValue(ConfigBase):
     """Store and manipulate a single configuration value.
@@ -1344,6 +1430,9 @@ def value(self, accessValue=True):
         return self._data
 
     def set_value(self, value):
+        # Trap self-assignment (useful for providing editor completion)
+        if value is self:
+            return
         self._data = self._cast(value)
         self._userSet = True
 
@@ -1354,22 +1443,17 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False):
 
 
 class ImmutableConfigValue(ConfigValue):
+    def __new__(self, *args, **kwds):
+        # ImmutableConfigValue objects are never directly created, and
+        # any attempt to copy one will generate a mutable ConfigValue
+        # object
+        return ConfigValue(*args, **kwds)
+
     def set_value(self, value):
         if self._cast(value) != self._data:
             raise RuntimeError(str(self) + ' is currently immutable')
         super(ImmutableConfigValue, self).set_value(value)
 
-    def reset(self):
-        try:
-            super(ImmutableConfigValue, self).set_value(self._default)
-        except:
-            if hasattr(self._default, '__call__'):
-                super(ImmutableConfigValue, self).set_value(self._default())
-            else:
-                raise
-        self._userAccessed = False
-        self._userSet = False
-
 
 class MarkImmutable(object):
     """
@@ -1393,13 +1477,18 @@ class MarkImmutable(object):
     >>> locker.release_lock()
     """
     def __init__(self, *args):
-        self._locked = list()
+        self._targets = args
+        self._locked = []
+        self.lock()
+
+    def lock(self):
         try:
-            for arg in args:
-                if type(arg) is not ConfigValue:
-                    raise ValueError('Only ConfigValue instances can be marked immutable.')
-                arg.__class__ = ImmutableConfigValue
-                self._locked.append(arg)
+            for cfg in self._targets:
+                if type(cfg) is not ConfigValue:
+                    raise ValueError(
+                        'Only ConfigValue instances can be marked immutable.')
+                cfg.__class__ = ImmutableConfigValue
+                self._locked.append(cfg)
         except:
             self.release_lock()
             raise
@@ -1407,7 +1496,15 @@ def __init__(self, *args):
     def release_lock(self):
         for arg in self._locked:
             arg.__class__ = ConfigValue
-        self._locked = list()
+        self._locked = []
+
+    def __enter__(self):
+        if not self._locked:
+            self.lock()
+        return self
+
+    def __exit__(self, t, v, tb):
+        self.release_lock()
 
 
 class ConfigList(ConfigBase):
@@ -1473,7 +1570,7 @@ def __getitem__(self, key):
         else:
             return val
 
-    def get(self, key, default=ConfigBase.NoArgument):
+    def get(self, key, default=NoArgumentGiven):
         # Note: get() is borrowed from ConfigDict for cases where we
         # want the raw stored object (and to aviod the implicit
         # conversion of ConfigValue members to their stored data).
@@ -1481,14 +1578,11 @@ def get(self, key, default=ConfigBase.NoArgument):
             val = self._data[key]
             self._userAccessed = True
             return val
-        except:
-            pass
-        if default is ConfigBase.NoArgument:
-            return None
-        if self._domain is not None:
-            return self._domain(default)
-        else:
-            return ConfigValue(default)
+        except IndexError:
+            if default is NoArgumentGiven:
+                raise
+        # Note: self._domain is ALWAYS derived from ConfigBase
+        return self._domain(default)
 
     def __setitem__(self, key, val):
         # Note: this will fail if the element doesn't exist in _data.
@@ -1539,20 +1633,23 @@ def reset(self):
         for val in self.user_values():
             val._userSet = False
 
-    def append(self, value=ConfigBase.NoArgument):
+    def append(self, value=NoArgumentGiven):
         val = self._cast(value)
         if val is None:
             return
         self._data.append(val)
-        #print self._data[-1], type(self._data[-1])
         self._data[-1]._parent = self
         self._data[-1]._name = '[%s]' % (len(self._data) - 1,)
         self._data[-1]._userSet = True
-        self._userSet = True
+        # Adding something to the container should not change the
+        # userSet on the container (see Pyomo/pyomo#352; now
+        # Pyomo/pysp#8 for justification)
+        #self._userSet = True
 
     @deprecated("ConfigList.add() has been deprecated.  Use append()",
                 version='5.7.2')
-    def add(self, value=ConfigBase.NoArgument):
+    def add(self, value=NoArgumentGiven):
+        "Append the specified value to the list, casting as necessary."
         return self.append(value)
 
     def _data_collector(self, level, prefix, visibility=None, docMode=False):
@@ -1617,11 +1714,11 @@ class ConfigDict(ConfigBase):
 
     """
 
-    content_filters = (None, 'all', 'userdata')
+    content_filters = {None, 'all', 'userdata'}
 
     __slots__ = ('_decl_order', '_declared', '_implicit_declaration',
                  '_implicit_domain')
-    _all_slots = __slots__ + ConfigBase.__slots__
+    _all_slots = set(__slots__ + ConfigBase.__slots__)
 
     def __init__(self,
                  description=None,
@@ -1650,61 +1747,66 @@ def __setstate__(self, state):
         for x in self._data.values():
             x._parent = self
 
+    def __dir__(self):
+        # Note that dir() returns the *normalized* names (i.e., no spaces)
+        return sorted(super(ConfigDict, self).__dir__() + list(self._data))
+
     def __getitem__(self, key):
         self._userAccessed = True
-        key = str(key)
-        if isinstance(self._data[key], ConfigValue):
-            return self._data[key].value()
+        _key = str(key).replace(' ','_')
+        if isinstance(self._data[_key], ConfigValue):
+            return self._data[_key].value()
         else:
-            return self._data[key]
+            return self._data[_key]
 
-    def get(self, key, default=ConfigBase.NoArgument):
+    def get(self, key, default=NoArgumentGiven):
         self._userAccessed = True
-        key = str(key)
-        if key in self._data:
-            return self._data[key]
-        if default is ConfigBase.NoArgument:
+        _key = str(key).replace(' ','_')
+        if _key in self._data:
+            return self._data[_key]
+        if default is NoArgumentGiven:
             return None
         if self._implicit_domain is not None:
             return self._implicit_domain(default)
         else:
             return ConfigValue(default)
 
-    def setdefault(self, key, default=ConfigBase.NoArgument):
+    def setdefault(self, key, default=NoArgumentGiven):
         self._userAccessed = True
-        key = str(key)
-        if key in self._data:
-            return self._data[key]
-        if default is ConfigBase.NoArgument:
+        _key = str(key).replace(' ','_')
+        if _key in self._data:
+            return self._data[_key]
+        if default is NoArgumentGiven:
             return self.add(key, None)
         else:
             return self.add(key, default)
 
     def __setitem__(self, key, val):
-        key = str(key)
-        if key not in self._data:
+        _key = str(key).replace(' ','_')
+        if _key not in self._data:
             self.add(key, val)
         else:
-            self._data[key].set_value(val)
+            self._data[_key].set_value(val)
         #self._userAccessed = True
 
     def __delitem__(self, key):
         # Note that this will produce a KeyError if the key is not valid
         # for this ConfigDict.
-        del self._data[key]
+        _key = str(key).replace(' ','_')
+        del self._data[_key]
         # Clean up the other data structures
-        self._decl_order.remove(key)
-        self._declared.discard(key)
+        self._decl_order.remove(_key)
+        self._declared.discard(_key)
 
     def __contains__(self, key):
-        key = str(key)
-        return key in self._data
+        _key = str(key).replace(' ','_')
+        return _key in self._data
 
     def __len__(self):
         return self._decl_order.__len__()
 
     def __iter__(self):
-        return self._decl_order.__iter__()
+        return (self._data[key]._name for key in self._decl_order)
 
     def __getattr__(self, name):
         # Note: __getattr__ is only called after all "usual" attribute
@@ -1712,23 +1814,30 @@ def __getattr__(self, name):
         # know that key is not a __slot__ or a method, etc...
         #if name in ConfigDict._all_slots:
         #    return super(ConfigDict,self).__getattribute__(name)
-        if name not in self._data:
-            _name = name.replace('_', ' ')
-            if _name not in self._data:
-                raise AttributeError("Unknown attribute '%s'" % name)
-            name = _name
-        return ConfigDict.__getitem__(self, name)
+        _name = name.replace(' ', '_')
+        if _name not in self._data:
+            raise AttributeError("Unknown attribute '%s'" % name)
+        return ConfigDict.__getitem__(self, _name)
 
     def __setattr__(self, name, value):
         if name in ConfigDict._all_slots:
             super(ConfigDict, self).__setattr__(name, value)
         else:
-            if name not in self._data:
-                name = name.replace('_', ' ')
             ConfigDict.__setitem__(self, name, value)
 
+    def __delattr__(self, name):
+        _key = str(name).replace(' ','_')
+        if _key in self._data:
+            del self[_key]
+        elif _key in dir(self):
+            raise AttributeError("'%s' object attribute '%s' is read-only" %
+                                 (type(self).__name__, name))
+        else:
+            raise AttributeError("'%s' object has no attribute '%s'" %
+                                 (type(self).__name__, name))
+
     def keys(self):
-        return self._decl_order.__iter__()
+        return iter(self)
 
     def values(self):
         self._userAccessed = True
@@ -1738,7 +1847,7 @@ def values(self):
     def items(self):
         self._userAccessed = True
         for key in self._decl_order:
-            yield (key, self[key])
+            yield (self._data[key]._name, self[key])
 
     @deprecated('The iterkeys method is deprecated. Use dict.keys().',
                 version='TBD')
@@ -1757,12 +1866,13 @@ def iteritems(self):
 
     def _add(self, name, config):
         name = str(name)
+        _name = name.replace(' ', '_')
         if config._parent is not None:
             raise ValueError(
                 "config '%s' is already assigned to ConfigDict '%s'; "
                 "cannot reassign to '%s'" %
                 (name, config._parent.name(True), self.name(True)))
-        if name in self._data:
+        if _name in self._data:
             raise ValueError(
                 "duplicate config '%s' defined for ConfigDict '%s'" %
                 (name, self.name(True)))
@@ -1770,15 +1880,16 @@ def _add(self, name, config):
             raise ValueError(
                 "Illegal character in config '%s' for ConfigDict '%s': "
                 "'.[]' are not allowed." % (name, self.name(True)))
-        self._data[name] = config
-        self._decl_order.append(name)
+        self._data[_name] = config
+        self._decl_order.append(_name)
         config._parent = self
         config._name = name
         return config
 
     def declare(self, name, config):
+        _name = str(name).replace(' ', '_')
         ans = self._add(name, config)
-        self._declared.add(name)
+        self._declared.add(_name)
         return ans
 
     def declare_from(self, other, skip=None):
@@ -1792,8 +1903,8 @@ def declare_from(self, other, skip=None):
                 continue
             if key in self:
                 raise ValueError("ConfigDict.declare_from passed a block "
-                                 "with a duplicate field, %s" % (key,))
-            self.declare(key, other._data[key]())
+                                 "with a duplicate field, '%s'" % (key,))
+            self.declare(key, other.get(key)())
 
     def add(self, name, config):
         if not self._implicit_declaration:
@@ -1808,14 +1919,18 @@ def add(self, name, config):
                 ans = self._add(name, ConfigValue(config))
         else:
             ans = self._add(name, self._implicit_domain(config))
-        self._userSet = True
+        ans._userSet = True
+        # Adding something to the container should not change the
+        # userSet on the container (see Pyomo/pyomo#352; now
+        # Pyomo/pysp#8 for justification)
+        #self._userSet = True
         return ans
 
     def value(self, accessValue=True):
         if accessValue:
             self._userAccessed = True
-        return dict((name, config.value(accessValue))
-                    for name, config in self._data.items())
+        return { cfg._name: cfg.value(accessValue)
+                 for cfg in map(self._data.__getitem__, self._decl_order) }
 
     def set_value(self, value, skip_implicit=False):
         if value is None:
@@ -1829,26 +1944,22 @@ def set_value(self, value, skip_implicit=False):
         _implicit = []
         _decl_map = {}
         for key in value:
-            _key = str(key)
+            _key = str(key).replace(' ', '_')
             if _key in self._data:
                 # str(key) may not be key... store the mapping so that
                 # when we later iterate over the _decl_order, we can map
                 # the local keys back to the incoming value keys.
                 _decl_map[_key] = key
             else:
-                _key = _key.replace('_', ' ')
-                if _key in self._data:
-                    _decl_map[str(_key)] = key
+                if skip_implicit:
+                    pass
+                elif self._implicit_declaration:
+                    _implicit.append(key)
                 else:
-                    if skip_implicit:
-                        pass
-                    elif self._implicit_declaration:
-                        _implicit.append(key)
-                    else:
-                        raise ValueError(
-                            "key '%s' not defined for ConfigDict '%s' and "
-                            "implicit (undefined) keys are not allowed" %
-                            (key, self.name(True)))
+                    raise ValueError(
+                        "key '%s' not defined for ConfigDict '%s' and "
+                        "implicit (undefined) keys are not allowed" %
+                        (key, self.name(True)))
 
         # If the set_value fails part-way through the new values, we
         # want to restore a deterministic state.  That is, either
@@ -1860,7 +1971,6 @@ def set_value(self, value, skip_implicit=False):
             # on the order)
             for key in self._decl_order:
                 if key in _decl_map:
-                    #print "Setting", key, " = ", value
                     self[key] = value[_decl_map[key]]
             # implicit data is declared at the end (in sorted order)
             for key in sorted(_implicit):
@@ -1895,8 +2005,9 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False):
             if level is not None:
                 level += 1
         for key in self._decl_order:
-            for v in self._data[key]._data_collector(level, key + ': ',
-                                                     visibility, docMode):
+            cfg = self._data[key]
+            for v in cfg._data_collector(
+                    level, cfg._name + ': ', visibility, docMode):
                 yield v
 
 # Backwards compatibility: ConfigDict was originally named ConfigBlock.


Test patch:
diff --git a/pyomo/common/tests/test_config.py b/pyomo/common/tests/test_config.py
--- a/pyomo/common/tests/test_config.py
+++ b/pyomo/common/tests/test_config.py
@@ -26,9 +26,10 @@
 import argparse
 import enum
 import os
-import sys
 import os.path
+import pickle
 import re
+import sys
 import pyomo.common.unittest as unittest
 
 from io import StringIO
@@ -39,10 +40,11 @@ def yaml_load(arg):
 
 from pyomo.common.config import (
     ConfigDict, ConfigValue,
-    ConfigList, MarkImmutable,
+    ConfigList, MarkImmutable, ImmutableConfigValue,
     PositiveInt, NegativeInt, NonPositiveInt, NonNegativeInt,
     PositiveFloat, NegativeFloat, NonPositiveFloat, NonNegativeFloat,
-    In, Path, PathList, ConfigEnum
+    In, Path, PathList, ConfigEnum,
+    _UnpickleableDomain,
 )
 from pyomo.common.log import LoggingIntercept
 
@@ -384,6 +386,8 @@ def norm(x):
         self.assertEqual(len(c.a), 1)
         self.assertTrue(os.path.sep in c.a[0])
         self.assertEqual(c.a[0], norm('/a/b/c'))
+        c.a = None
+        self.assertIsNone(c.a)
 
         c.a = ["a/b/c", "/a/b/c", "${CWD}/a/b/c"]
         self.assertEqual(len(c.a), 3)
@@ -429,6 +433,7 @@ class TestEnum(ConfigEnum):
         with self.assertRaisesRegex(ValueError, '.*invalid value'):
             cfg.enum ='ITEM_THREE'
 
+
 class TestImmutableConfigValue(unittest.TestCase):
     def test_immutable_config_value(self):
         config = ConfigDict()
@@ -439,9 +444,9 @@ def test_immutable_config_value(self):
         self.assertEqual(config.a, 2)
         self.assertEqual(config.b, 3)
         locker = MarkImmutable(config.get('a'), config.get('b'))
-        with self.assertRaises(Exception):
+        with self.assertRaisesRegex(RuntimeError, 'is currently immutable'):
             config.a = 4
-        with self.assertRaises(Exception):
+        with self.assertRaisesRegex(RuntimeError, 'is currently immutable'):
             config.b = 5
         config.a = 2
         config.b = 3
@@ -452,7 +457,9 @@ def test_immutable_config_value(self):
         config.b = 5
         self.assertEqual(config.a, 4)
         self.assertEqual(config.b, 5)
-        with self.assertRaises(ValueError):
+        with self.assertRaisesRegex(
+                ValueError,
+                'Only ConfigValue instances can be marked immutable'):
             locker = MarkImmutable(config.get('a'), config.b)
         self.assertEqual(type(config.get('a')), ConfigValue)
         config.a = 6
@@ -460,15 +467,58 @@ def test_immutable_config_value(self):
 
         config.declare('c', ConfigValue(default=-1, domain=int))
         locker = MarkImmutable(config.get('a'), config.get('b'))
+        # Making a copy of an immutable config value results in a
+        # *mutable* config value
         config2 = config({'c': -2})
         self.assertEqual(config2.a, 6)
         self.assertEqual(config2.b, 5)
         self.assertEqual(config2.c, -2)
-        with self.assertRaises(Exception):
-            config3 = config({'a': 1})
-        locker.release_lock()
+        self.assertIs(type(config2.get('a')), ConfigValue)
+        self.assertIs(type(config2.get('b')), ConfigValue)
+        self.assertIs(type(config2.get('c')), ConfigValue)
+        # you can even update the original, as long as you don't change
+        # the immutable value:
+        config.set_value(config2)
+        self.assertEqual(config.a, 6)
+        self.assertEqual(config.b, 5)
+        self.assertEqual(config.c, -2)
+        self.assertIs(type(config.get('a')), ImmutableConfigValue)
+        self.assertIs(type(config.get('b')), ImmutableConfigValue)
+        self.assertIs(type(config.get('c')), ConfigValue)
+
+        # Making a copy of an immutable config value results in a
+        # *mutable* config value, even if you change the value of
+        # something that is currently immutable
         config3 = config({'a': 1})
         self.assertEqual(config3.a, 1)
+        self.assertEqual(config3.b, 5)
+        self.assertEqual(config3.c, -2)
+        self.assertIs(type(config3.get('a')), ConfigValue)
+        self.assertIs(type(config3.get('b')), ConfigValue)
+        self.assertIs(type(config3.get('c')), ConfigValue)
+        # but attempting to update the original will generate an
+        # exception
+        with self.assertRaisesRegex(RuntimeError, ' is currently immutable'):
+            config.set_value(config3)
+        locker.release_lock()
+
+        # test reset
+        config.reset()
+        self.assertEqual(config.a, 1)
+        self.assertEqual(config.b, 1)
+        with locker:
+            # Reset is OK as long as the values are all currently at
+            # their defaults
+            config.reset()
+            self.assertEqual(config.a, 1)
+            self.assertEqual(config.b, 1)
+
+        config.a = 2
+        with locker:
+            # But if reset would change an immutable value you will get
+            # an exception
+            with self.assertRaisesRegex(RuntimeError, 'is currently immutable'):
+                config.reset()
 
 
 class TestConfig(unittest.TestCase):
@@ -824,20 +874,40 @@ def test_display_userdata_list_nonDefault(self):
     detection: []
 """)
 
-    def test_display_userdata_block(self):
+    def test_display_userdata_add_block(self):
         self.config.add("foo", ConfigValue(0, int, None, None))
         self.config.add("bar", ConfigDict())
         test = _display(self.config, 'userdata')
         sys.stdout.write(test)
-        self.assertEqual(test, "")
+        self.assertEqual(test, """foo: 0
+bar:
+""")
 
-    def test_display_userdata_block_nonDefault(self):
+    def test_display_userdata_add_block_nonDefault(self):
         self.config.add("foo", ConfigValue(0, int, None, None))
         self.config.add("bar", ConfigDict(implicit=True)) \
                    .add("baz", ConfigDict())
         test = _display(self.config, 'userdata')
         sys.stdout.write(test)
-        self.assertEqual(test, "bar:\n")
+        self.assertEqual(test, """foo: 0
+bar:
+  baz:
+""")
+
+    def test_display_userdata_declare_block(self):
+        self.config.declare("foo", ConfigValue(0, int, None, None))
+        self.config.declare("bar", ConfigDict())
+        test = _display(self.config, 'userdata')
+        sys.stdout.write(test)
+        self.assertEqual(test, "")
+
+    def test_display_userdata_declare_block_nonDefault(self):
+        self.config.declare("foo", ConfigValue(0, int, None, None))
+        self.config.declare("bar", ConfigDict(implicit=True)) \
+                   .add("baz", ConfigDict())
+        test = _display(self.config, 'userdata')
+        sys.stdout.write(test)
+        self.assertEqual(test, "bar:\n  baz:\n")
 
     def test_unusedUserValues_default(self):
         test = '\n'.join(x.name(True) for x in self.config.unused_user_values())
@@ -854,16 +924,14 @@ def test_unusedUserValues_list(self):
         self.config['scenarios'].append()
         test = '\n'.join(x.name(True) for x in self.config.unused_user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, """scenarios
-scenarios[0]""")
+        self.assertEqual(test, """scenarios[0]""")
 
     def test_unusedUserValues_list_nonDefault(self):
         self.config['scenarios'].append()
         self.config['scenarios'].append({'merlion': True, 'detection': []})
         test = '\n'.join(x.name(True) for x in self.config.unused_user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, """scenarios
-scenarios[0]
+        self.assertEqual(test, """scenarios[0]
 scenarios[1]
 scenarios[1].merlion
 scenarios[1].detection""")
@@ -889,18 +957,33 @@ def test_unusedUserValues_list_nonDefault_itemAccessed(self):
         self.assertEqual(test, """scenarios[0]
 scenarios[1].detection""")
 
-    def test_unusedUserValues_topBlock(self):
+    def test_unusedUserValues_add_topBlock(self):
         self.config.add('foo', ConfigDict())
         test = '\n'.join(x.name(True) for x in self.config.unused_user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, "")
+        self.assertEqual(test, "foo")
+        test = '\n'.join(x.name(True) for x in
+                         self.config.foo.unused_user_values())
+        sys.stdout.write(test)
+        self.assertEqual(test, "foo")
 
-    def test_unusedUserValues_subBlock(self):
+    def test_unusedUserValues_add_subBlock(self):
         self.config['scenario'].add('foo', ConfigDict())
         test = '\n'.join(x.name(True) for x in self.config.unused_user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, """scenario
-scenario.foo""")
+        self.assertEqual(test, """scenario.foo""")
+
+    def test_unusedUserValues_declare_topBlock(self):
+        self.config.declare('foo', ConfigDict())
+        test = '\n'.join(x.name(True) for x in self.config.unused_user_values())
+        sys.stdout.write(test)
+        self.assertEqual(test, "")
+
+    def test_unusedUserValues_declare_subBlock(self):
+        self.config['scenario'].declare('foo', ConfigDict())
+        test = '\n'.join(x.name(True) for x in self.config.unused_user_values())
+        sys.stdout.write(test)
+        self.assertEqual(test, "")
 
     def test_UserValues_default(self):
         test = '\n'.join(x.name(True) for x in self.config.user_values())
@@ -917,16 +1000,14 @@ def test_UserValues_list(self):
         self.config['scenarios'].append()
         test = '\n'.join(x.name(True) for x in self.config.user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, """scenarios
-scenarios[0]""")
+        self.assertEqual(test, """scenarios[0]""")
 
     def test_UserValues_list_nonDefault(self):
         self.config['scenarios'].append()
         self.config['scenarios'].append({'merlion': True, 'detection': []})
         test = '\n'.join(x.name(True) for x in self.config.user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, """scenarios
-scenarios[0]
+        self.assertEqual(test, """scenarios[0]
 scenarios[1]
 scenarios[1].merlion
 scenarios[1].detection""")
@@ -938,8 +1019,7 @@ def test_UserValues_list_nonDefault_listAccessed(self):
             pass
         test = '\n'.join(x.name(True) for x in self.config.user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, """scenarios
-scenarios[0]
+        self.assertEqual(test, """scenarios[0]
 scenarios[1]
 scenarios[1].merlion
 scenarios[1].detection""")
@@ -950,24 +1030,37 @@ def test_UserValues_list_nonDefault_itemAccessed(self):
         self.config['scenarios'][1]['merlion']
         test = '\n'.join(x.name(True) for x in self.config.user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, """scenarios
-scenarios[0]
+        self.assertEqual(test, """scenarios[0]
 scenarios[1]
 scenarios[1].merlion
 scenarios[1].detection""")
 
-    def test_UserValues_topBlock(self):
+    def test_UserValues_add_topBlock(self):
         self.config.add('foo', ConfigDict())
         test = '\n'.join(x.name(True) for x in self.config.user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, "")
+        self.assertEqual(test, "foo")
+        test = '\n'.join(x.name(True) for x in self.config.foo.user_values())
+        sys.stdout.write(test)
+        self.assertEqual(test, "foo")
 
-    def test_UserValues_subBlock(self):
+    def test_UserValues_add_subBlock(self):
         self.config['scenario'].add('foo', ConfigDict())
         test = '\n'.join(x.name(True) for x in self.config.user_values())
         sys.stdout.write(test)
-        self.assertEqual(test, """scenario
-scenario.foo""")
+        self.assertEqual(test, """scenario.foo""")
+
+    def test_UserValues_declare_topBlock(self):
+        self.config.declare('foo', ConfigDict())
+        test = '\n'.join(x.name(True) for x in self.config.user_values())
+        sys.stdout.write(test)
+        self.assertEqual(test, "")
+
+    def test_UserValues_declare_subBlock(self):
+        self.config['scenario'].declare('foo', ConfigDict())
+        test = '\n'.join(x.name(True) for x in self.config.user_values())
+        sys.stdout.write(test)
+        self.assertEqual(test, "")
 
     @unittest.skipIf(not yaml_available, "Test requires PyYAML")
     def test_parseDisplayAndValue_default(self):
@@ -1008,21 +1101,38 @@ def test_parseDisplay_userdata_list_nonDefault(self):
                                           'detection': []}]})
 
     @unittest.skipIf(not yaml_available, "Test requires PyYAML")
-    def test_parseDisplay_userdata_block(self):
+    def test_parseDisplay_userdata_add_block(self):
         self.config.add("foo", ConfigValue(0, int, None, None))
         self.config.add("bar", ConfigDict())
         test = _display(self.config, 'userdata')
         sys.stdout.write(test)
-        self.assertEqual(yaml_load(test), None)
+        self.assertEqual(yaml_load(test), {'foo': 0, 'bar': None})
 
     @unittest.skipIf(not yaml_available, "Test requires PyYAML")
-    def test_parseDisplay_userdata_block_nonDefault(self):
+    def test_parseDisplay_userdata_add_block_nonDefault(self):
         self.config.add("foo", ConfigValue(0, int, None, None))
         self.config.add("bar", ConfigDict(implicit=True)) \
                    .add("baz", ConfigDict())
         test = _display(self.config, 'userdata')
         sys.stdout.write(test)
-        self.assertEqual(yaml_load(test), {'bar': None})
+        self.assertEqual(yaml_load(test), {'bar': {'baz': None}, foo: 0})
+
+    @unittest.skipIf(not yaml_available, "Test requires PyYAML")
+    def test_parseDisplay_userdata_add_block(self):
+        self.config.declare("foo", ConfigValue(0, int, None, None))
+        self.config.declare("bar", ConfigDict())
+        test = _display(self.config, 'userdata')
+        sys.stdout.write(test)
+        self.assertEqual(yaml_load(test), None)
+
+    @unittest.skipIf(not yaml_available, "Test requires PyYAML")
+    def test_parseDisplay_userdata_add_block_nonDefault(self):
+        self.config.declare("foo", ConfigValue(0, int, None, None))
+        self.config.declare("bar", ConfigDict(implicit=True)) \
+                   .add("baz", ConfigDict())
+        test = _display(self.config, 'userdata')
+        sys.stdout.write(test)
+        self.assertEqual(yaml_load(test), {'bar': {'baz': None}})
 
     def test_value_ConfigValue(self):
         val = self.config['flushing']['flush nodes']['rate']
@@ -1236,6 +1346,18 @@ def test_default_function(self):
                 ValueError, 'invalid value for configuration'):
             c = ConfigValue('a', domain=int)
 
+    def test_set_default(self):
+        c = ConfigValue()
+        self.assertIsNone(c.value())
+        c.set_default_value(10.5)
+        self.assertIsNone(c.value())
+        c.reset()
+        self.assertIs(type(c.value()), float)
+        self.assertEqual(c.value(), 10.5)
+        c.set_domain(int)
+        self.assertIs(type(c.value()), int)
+        self.assertEqual(c.value(), 10)
+
     def test_getItem_setItem(self):
         # a freshly-initialized object should not be accessed
         self.assertFalse(self.config._userAccessed)
@@ -1273,11 +1395,41 @@ def test_delitem(self):
         self.assertEqual(sorted(config.keys()), ['bar'])
         config.foo = 5
         self.assertEqual(sorted(config.keys()), ['bar', 'foo'])
+        self.assertEqual(sorted(config._declared), ['bar'])
         del config['foo']
         self.assertEqual(sorted(config.keys()), ['bar'])
+        self.assertEqual(sorted(config._declared), ['bar'])
         del config['bar']
         self.assertEqual(sorted(config.keys()), [])
+        self.assertEqual(sorted(config._declared), [])
+
+        with self.assertRaisesRegex(KeyError, "'get'"):
+            del config['get']
+        with self.assertRaisesRegex(KeyError, "'foo'"):
+            del config['foo']
+
+    def test_delattr(self):
+        config = ConfigDict(implicit=True)
+        config.declare('bar', ConfigValue())
+        self.assertEqual(sorted(config.keys()), ['bar'])
+        config.foo = 5
+        self.assertEqual(sorted(config.keys()), ['bar', 'foo'])
+        self.assertEqual(sorted(config._declared), ['bar'])
+        del config.foo
+        self.assertEqual(sorted(config._declared), ['bar'])
+        self.assertEqual(sorted(config.keys()), ['bar'])
+        del config.bar
+        self.assertEqual(sorted(config.keys()), [])
+        self.assertEqual(sorted(config._declared), [])
 
+        with self.assertRaisesRegex(
+                AttributeError,
+                "'ConfigDict' object attribute 'get' is read-only"):
+            del config.get
+        with self.assertRaisesRegex(
+                AttributeError,
+                "'ConfigDict' object has no attribute 'foo'"):
+            del config.foo
 
     def test_generate_custom_documentation(self):
         reference = \
@@ -1407,6 +1559,44 @@ def test_generate_custom_documentation(self):
         #print(test)
         self.assertEqual(test, stripped_reference)
 
+
+    def test_generate_latex_documentation(self):
+        cfg = ConfigDict()
+        cfg.declare('int', ConfigValue(
+            domain=int, default=10,
+            doc="This is an integer parameter",
+        ))
+        cfg.declare('in', ConfigValue(
+            domain=In([1,3,5]), default=1,
+            description="This parameter must be in {1,3,5}",
+        ))
+        cfg.declare('lambda', ConfigValue(
+            domain=lambda x: int(x), default=1,
+            description="This is a float",
+            doc="This parameter is actually a float, but for testing "
+            "purposes we will use a lambda function for validation"
+        ))
+        cfg.declare('list', ConfigList(
+            domain=str,
+            description="A simple list of strings",
+        ))
+        self.assertEqual(
+            cfg.generate_documentation(format='latex').strip(),
+            """
+\\begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em]
+  \\item[{int}]\\hfill
+    \\\\This is an integer parameter
+  \\item[{in}]\\hfill
+    \\\\This parameter must be in {1,3,5}
+  \\item[{lambda}]\\hfill
+    \\\\This parameter is actually a float, but for testing purposes we will use
+    a lambda function for validation
+  \\item[{list}]\\hfill
+    \\\\A simple list of strings
+\\end{description}
+            """.strip())
+
+
     def test_block_get(self):
         self.assertTrue('scenario' in self.config)
         self.assertNotEquals(
@@ -1475,61 +1665,70 @@ def test_setdefault(self):
     def test_block_keys(self):
         ref = ['scenario file', 'merlion', 'detection']
 
-        # list of keys
+        # keys iterator
         keys = self.config['scenario'].keys()
         # lists are independent
-        self.assertFalse(keys is self.config['scenario'].keys())
+        self.assertIsNot(keys, self.config['scenario'].keys())
         self.assertIsNot(type(keys), list)
         self.assertEqual(list(keys), ref)
 
-        # keys iterator
-        keyiter = self.config['scenario'].iterkeys()
+        # (deprecated) python 2 iterator
+        out = StringIO()
+        with LoggingIntercept(out):
+            keyiter = self.config['scenario'].iterkeys()
+            # iterators are independent
+            self.assertIsNot(keyiter, self.config['scenario'].iterkeys())
+        self.assertIn("The iterkeys method is deprecated", out.getvalue())
         self.assertIsNot(type(keyiter), list)
         self.assertEqual(list(keyiter), ref)
-        # iterators are independent
-        self.assertFalse(keyiter is self.config['scenario'].iterkeys())
 
         # default iterator
         keyiter = self.config['scenario'].__iter__()
         self.assertIsNot(type(keyiter), list)
         self.assertEqual(list(keyiter), ref)
         # iterators are independent
-        self.assertFalse(keyiter is self.config['scenario'].__iter__())
+        self.assertIsNot(keyiter, self.config['scenario'].__iter__())
 
     def test_block_values(self):
         ref = ['Net3.tsg', False, [1, 2, 3]]
 
-        # list of values
+        # values iterator
         values = self.config['scenario'].values()
         self.assertIsNot(type(values), list)
         self.assertEqual(list(values), ref)
         # lists are independent
-        self.assertFalse(values is self.config['scenario'].values())
+        self.assertIsNot(values, self.config['scenario'].values())
 
-        # values iterator
-        valueiter = self.config['scenario'].itervalues()
+        # (deprecated) python 2 iterator
+        out = StringIO()
+        with LoggingIntercept(out):
+            valueiter = self.config['scenario'].itervalues()
+            # iterators are independent
+            self.assertIsNot(valueiter, self.config['scenario'].itervalues())
+        self.assertIn("The itervalues method is deprecated", out.getvalue())
         self.assertIsNot(type(valueiter), list)
         self.assertEqual(list(valueiter), ref)
-        # iterators are independent
-        self.assertFalse(valueiter is self.config['scenario'].itervalues())
 
     def test_block_items(self):
         ref = [('scenario file', 'Net3.tsg'), ('merlion', False),
                ('detection', [1, 2, 3])]
 
-        # list of items
+        # items iterator
         items = self.config['scenario'].items()
         self.assertIsNot(type(items), list)
         self.assertEqual(list(items), ref)
         # lists are independent
-        self.assertFalse(items is self.config['scenario'].items())
+        self.assertIsNot(items, self.config['scenario'].items())
 
-        # items iterator
-        itemiter = self.config['scenario'].iteritems()
+        # (deprecated) python 2 iterator
+        out = StringIO()
+        with LoggingIntercept(out):
+            itemiter = self.config['scenario'].iteritems()
+            # iterators are independent
+            self.assertIsNot(itemiter, self.config['scenario'].iteritems())
+        self.assertIn("The iteritems method is deprecated", out.getvalue())
         self.assertIsNot(type(itemiter), list)
         self.assertEqual(list(itemiter), ref)
-        # iterators are independent
-        self.assertFalse(itemiter is self.config['scenario'].iteritems())
 
     def test_value(self):
         #print(self.config.value())
@@ -1538,25 +1737,32 @@ def test_value(self):
     def test_list_manipulation(self):
         self.assertEqual(len(self.config['scenarios']), 0)
         self.config['scenarios'].append()
-        self.assertEqual(len(self.config['scenarios']), 1)
-        self.config['scenarios'].append({'merlion': True, 'detection': []})
+        os = StringIO()
+        with LoggingIntercept(os):
+            self.config['scenarios'].add()
+        self.assertIn("ConfigList.add() has been deprecated.  Use append()",
+                      os.getvalue())
         self.assertEqual(len(self.config['scenarios']), 2)
+        self.config['scenarios'].append({'merlion': True, 'detection': []})
+        self.assertEqual(len(self.config['scenarios']), 3)
         test = _display(self.config, 'userdata')
         sys.stdout.write(test)
         self.assertEqual(test, """scenarios:
   -
+  -
   -
     merlion: true
     detection: []
 """)
         self.config['scenarios'][0] = {'merlion': True, 'detection': []}
-        self.assertEqual(len(self.config['scenarios']), 2)
+        self.assertEqual(len(self.config['scenarios']), 3)
         test = _display(self.config, 'userdata')
         sys.stdout.write(test)
         self.assertEqual(test, """scenarios:
   -
     merlion: true
     detection: []
+  -
   -
     merlion: true
     detection: []
@@ -1567,6 +1773,10 @@ def test_list_manipulation(self):
   scenario file: Net3.tsg
   merlion: true
   detection: []
+-
+  scenario file: Net3.tsg
+  merlion: false
+  detection: [1, 2, 3]
 -
   scenario file: Net3.tsg
   merlion: true
@@ -1575,17 +1785,21 @@ def test_list_manipulation(self):
 
     def test_list_get(self):
         X = ConfigDict(implicit=True)
-        X.config = ConfigList()
+        X.declare('config', ConfigList())
         self.assertEqual(_display(X, 'userdata'), "")
-        self.assertIs(X.config.get(0), None )
+        with self.assertRaisesRegex(IndexError, 'list index out of range'):
+            self.assertIs(X.config.get(0), None)
         self.assertIs(X.config.get(0,None).value(), None )
+        val = X.config.get(0, 1)
+        self.assertIsInstance(val, ConfigValue)
+        self.assertEqual(val.value(), 1)
         self.assertRaisesRegexp(
             IndexError, '.*out of range', X.config.__getitem__, 0 )
         # get() shouldn't change the userdata flag...
         self.assertEqual(_display(X, 'userdata'), "")
 
         X = ConfigDict(implicit=True)
-        X.config = ConfigList([42], int)
+        X.declare('config', ConfigList([42], int))
         self.assertEqual(_display(X, 'userdata'), "")
         val = X.config.get(0)
         self.assertEqual(val.value(), 42)
@@ -1598,27 +1812,44 @@ def test_list_get(self):
         # get() shouldn't change the userdata flag...
         self.assertEqual(_display(X, 'userdata'), "")
 
-        self.assertIs(X.config.get(1), None )
+        with self.assertRaisesRegex(IndexError, 'list index out of range'):
+            self.assertIs(X.config.get(1), None)
         self.assertRaisesRegexp(
             IndexError, '.*out of range', X.config.__getitem__, 1)
 
         # this should ONLY change the userSet flag on the item (and not
         # the list)
         X.config.get(0).set_value(20)
-        self.assertEqual(_display(X, 'userdata'), "  - 20\n")
+        self.assertEqual(_display(X, 'userdata'), "config:\n  - 20\n")
+        self.assertEqual([_.name(True) for _ in X.user_values()],
+                         ["config[0]"])
 
         # this should ONLY change the userSet flag on the item (and not
         # the list)
         X = ConfigDict(implicit=True)
-        X.config = ConfigList([42], int)
+        X.declare('config', ConfigList([42], int))
         X.config[0] = 20
-        self.assertEqual(_display(X, 'userdata'), "  - 20\n")
+        self.assertEqual(_display(X, 'userdata'), "config:\n  - 20\n")
+        self.assertEqual([_.name(True) for _ in X.user_values()],
+                         ["config[0]"])
 
-        # This should change both...
+        # this should ONLY change the userSet flag on the item (and not
+        # the list)
         X = ConfigDict(implicit=True)
-        X.config = ConfigList([42], int)
+        X.declare('config', ConfigList([42], int))
         X.config.append(20)
         self.assertEqual(_display(X, 'userdata'), "config:\n  - 20\n")
+        self.assertEqual([_.name(True) for _ in X.user_values()],
+                         ["config[1]"])
+
+        # This should change both... because the [42] was "declared" as
+        # the default for the List, it will *not* be a user-set value
+        X = ConfigDict(implicit=True)
+        X.add('config', ConfigList([42], int))
+        X.config.append(20)
+        self.assertEqual(_display(X, 'userdata'), "config:\n  - 20\n")
+        self.assertEqual([_.name(True) for _ in X.user_values()],
+                         ["config", "config[1]"])
 
     def test_implicit_entries(self):
         config = ConfigDict()
@@ -1633,10 +1864,10 @@ def test_implicit_entries(self):
         config['implicit_2'] = 5
         self.assertEqual(3, len(config))
         self.assertEqual(['implicit_1', 'formal', 'implicit_2'],
-                         list(config.iterkeys()))
+                         list(config.keys()))
         config.reset()
         self.assertEqual(1, len(config))
-        self.assertEqual(['formal'], list(config.iterkeys()))
+        self.assertEqual(['formal'], list(config.keys()))
 
     def test_argparse_help(self):
         parser = argparse.ArgumentParser(prog='tester')
@@ -1845,6 +2076,45 @@ def test_set_value(self):
         self.assertEqual(config.a_c, 20)
         self.assertEqual(config.a_d_e, 30)
 
+    def test_name_mapping(self):
+        config = ConfigDict(implicit=True)
+
+        config.a_b = 5
+        self.assertEqual(list(config), ['a_b'])
+        self.assertIs(config.get('a_b'), config.get('a b'))
+        config['a b'] = 10
+        self.assertEqual(config.a_b, 10)
+        self.assertEqual(list(config), ['a_b'])
+        self.assertIn('a b', config)
+        self.assertIn('a_b', config)
+
+        config['c d'] = 1
+        self.assertEqual(list(config), ['a_b', 'c d'])
+        self.assertIs(config.get('c_d'), config.get('c d'))
+        config.c_d = 2
+        self.assertEqual(config['c d'], 2)
+        self.assertEqual(list(config), ['a_b', 'c d'])
+        self.assertIn('c d', config)
+        self.assertIn('c_d', config)
+
+        config.declare('e_f', ConfigValue(5, domain=int))
+        self.assertEqual(list(config), ['a_b', 'c d', 'e_f'])
+        self.assertIs(config.get('e_f'), config.get('e f'))
+        config['e f'] = 10
+        self.assertEqual(config.e_f, 10)
+        self.assertEqual(list(config), ['a_b', 'c d', 'e_f'])
+        self.assertIn('e f', config)
+        self.assertIn('e_f', config)
+
+        config['g h'] = 1
+        self.assertEqual(list(config), ['a_b', 'c d', 'e_f', 'g h'])
+        self.assertIs(config.get('g_h'), config.get('g h'))
+        config.g_h = 2
+        self.assertEqual(config['g h'], 2)
+        self.assertEqual(list(config), ['a_b', 'c d', 'e_f', 'g h'])
+        self.assertIn('g h', config)
+        self.assertIn('g_h', config)
+
     def test_call_options(self):
         config = ConfigDict(description="base description",
                              doc="base doc",
@@ -1900,6 +2170,134 @@ def test_call_options(self):
         self.assertEqual(mod_copy._description, "new description")
         self.assertEqual(mod_copy._visibility, 0)
 
+    def test_pickle(self):
+        def anon_domain(domain):
+            def cast(x):
+                return domain(x)
+            return cast
+        cfg = ConfigDict()
+        cfg.declare('int', ConfigValue(domain=int, default=10))
+        cfg.declare('in', ConfigValue(domain=In([1,3,5]), default=1))
+        cfg.declare('anon', ConfigValue(domain=anon_domain(int), default=1))
+        cfg.declare('lambda', ConfigValue(domain=lambda x: int(x), default=1))
+        cfg.declare('list', ConfigList(domain=str))
+
+        out = StringIO()
+        with LoggingIntercept(out, module=None):
+            cfg.set_value(
+                {'int': 100, 'in': 3, 'anon': 2.5, 'lambda': 1.5,
+                 'list': [2, 'a']}
+            )
+            self.assertEqual(
+                cfg.value(),
+                {'int': 100, 'in': 3, 'anon': 2, 'lambda': 1,
+                 'list': ['2', 'a']}
+            )
+
+            cfg2 = pickle.loads(pickle.dumps(cfg))
+            self.assertEqual(
+                cfg2.value(),
+                {'int': 100, 'in': 3, 'anon': 2, 'lambda': 1,
+                 'list': ['2', 'a']}
+            )
+
+            cfg2.list.append(10)
+            self.assertEqual(
+                cfg2.value(),
+                {'int': 100, 'in': 3, 'anon': 2, 'lambda': 1,
+                 'list': ['2', 'a', '10']}
+            )
+        # No warnings due to anything above.
+        self.assertEqual(out.getvalue(), "")
+
+        # On some platforms (notably, pypy3) if dill has been imported,
+        # then lambda and anonymous functions are actually picklable
+        # using the standard pickle.dumps() method.  We will check for
+        # one of two cases: either the domain was not picklable and was
+        # replaced by a passthrough "_UnpickleableDomain" object, OR it
+        # was picklable and the original domain was enforced.
+
+        out = StringIO()
+        with LoggingIntercept(out, module=None):
+            cfg2['anon'] = 5.5
+        if type(cfg2.get('anon')._domain) is _UnpickleableDomain:
+            self.assertIn(
+                "ConfigValue 'anon' was pickled with an unpicklable domain",
+                out.getvalue()
+            )
+            self.assertEqual(cfg2['anon'], 5.5)
+        else:
+            self.assertEqual(out.getvalue(), "")
+            self.assertIn('dill', sys.modules)
+            self.assertEqual(cfg2['anon'], 5)
+
+        out = StringIO()
+        with LoggingIntercept(out, module=None):
+            cfg2['lambda'] = 6.5
+        if type(cfg2.get('lambda')._domain) is _UnpickleableDomain:
+            self.assertIn(
+                "ConfigValue 'lambda' was pickled with an unpicklable domain",
+                out.getvalue()
+            )
+            self.assertEqual(cfg2['lambda'], 6.5)
+        else:
+            self.assertEqual(out.getvalue(), "")
+            self.assertIn('dill', sys.modules)
+            self.assertEqual(cfg2['lambda'], 6)
+
+
+    def test_self_assignment(self):
+        cfg = ConfigDict()
+        self.assertNotIn('d', dir(cfg))
+        cfg.d = cfg.declare('d', ConfigValue(10, int))
+        self.assertIn('d', dir(cfg))
+        cfg.aa = cfg.declare('aa', ConfigValue(1, int))
+        self.assertIn('aa', dir(cfg))
+        # test that dir is sorted
+        self.assertEqual(dir(cfg), sorted(dir(cfg)))
+        # check that inconsistent name is flagged
+        with self.assertRaisesRegexp(
+                ValueError, "Key 'b' not defined in ConfigDict ''"):
+            cfg.b = cfg.declare('bb', ConfigValue(2, int))
+
+
+    def test_declaration_errors(self):
+        cfg = ConfigDict()
+        cfg.b = cfg.declare('b', ConfigValue(2, int))
+        with self.assertRaisesRegexp(
+                ValueError, "duplicate config 'b' defined for ConfigDict ''"):
+            cfg.b = cfg.declare('b', ConfigValue(2, int))
+        with self.assertRaisesRegexp(
+                ValueError, "config 'dd' is already assigned to ConfigDict ''"):
+            cfg.declare('dd', cfg.get('b'))
+        with self.assertRaisesRegexp(
+                ValueError, "Illegal character in config 'd\[1\]'"):
+            cfg.declare('d[1]', ConfigValue(1))
+
+
+    def test_declare_from(self):
+        cfg = ConfigDict()
+        cfg.declare('a', ConfigValue(default=1, domain=int))
+        cfg.declare('b', ConfigValue(default=2, domain=int))
+        cfg2 = ConfigDict()
+        cfg2.declare_from(cfg)
+        self.assertEqual(cfg.value(), cfg2.value())
+        self.assertIsNot(cfg.get('a'), cfg2.get('a'))
+        self.assertIsNot(cfg.get('b'), cfg2.get('b'))
+
+        cfg2 = ConfigDict()
+        cfg2.declare_from(cfg, skip={'a'})
+        self.assertEqual(cfg.value()['b'], cfg2.value()['b'])
+        self.assertNotIn('a', cfg2)
+
+        with self.assertRaisesRegex(
+                ValueError, "passed a block with a duplicate field, 'b'"):
+            cfg2.declare_from(cfg)
+
+        with self.assertRaisesRegex(
+                ValueError, "only accepts other ConfigDicts"):
+            cfg2.declare_from({})
+
 
 if __name__ == "__main__":
     unittest.main()