Repository: Pyomo/pyomo

Issue:
Incorrect value from Param.default() when there is no default
When no default value is provided `Param.default()` returns `pyomo.core.base.param._NotValid`. According to its documentation the method should return `None`, as it used to do. Please update the documentation, in particular document how to determine if a default value exists.


Discussion:
See the related discussions in #300 and #301. @tkorvola , could you add a link to the documentation that you think is wrong?
I was referring to the docstring of `Param.default()`. It is also unclear how one should now check if a default is set: #301 just replaced `is None` with `is _NotFound` but underscore names are usually not regarded as public API.
Alternatively, perhaps preferably, please change `Param.default()` to behave (again) as documented, i.e., return `None` if no default is provided. `None` is after all the canonical missing value in Python.


Resolving patch:
diff --git a/pyomo/core/base/param.py b/pyomo/core/base/param.py
--- a/pyomo/core/base/param.py
+++ b/pyomo/core/base/param.py
@@ -83,11 +83,6 @@ def __contains__(self, val):
                 version='5.6.9', remove_in='6.0')
         return True
 
-class _NotValid(object):
-    """A dummy type that is pickle-safe that we can use as the default
-    value for Params to indicate that no valid value is present."""
-    pass
-
 
 class _ParamData(ComponentData, NumericValue):
     """
@@ -113,7 +108,7 @@ def __init__(self, component):
         # The following is equivalent to calling the
         # base NumericValue constructor.
         #
-        self._value = _NotValid
+        self._value = Param.NoValue
 
     def __getstate__(self):
         """
@@ -129,7 +124,7 @@ def __getstate__(self):
 
     def clear(self):
         """Clear the data in this component"""
-        self._value = _NotValid
+        self._value = Param.NoValue
 
     # FIXME: ComponentData need to have pointers to their index to make
     # operations like validation efficient.  As it stands now, if
@@ -145,7 +140,7 @@ def __call__(self, exception=True):
         """
         Return the value of this object.
         """
-        if self._value is _NotValid:
+        if self._value is Param.NoValue:
             if exception:
                 raise ValueError(
                     "Error evaluating Param value (%s):\n\tThe Param value is "
@@ -234,6 +229,11 @@ class Param(IndexedComponent):
 
     DefaultMutable = False
 
+    class NoValue(object):
+        """A dummy type that is pickle-safe that we can use as the default
+        value for Params to indicate that no valid value is present."""
+        pass
+
     def __new__(cls, *args, **kwds):
         if cls != Param:
             return super(Param, cls).__new__(cls)
@@ -243,13 +243,13 @@ def __new__(cls, *args, **kwds):
             return IndexedParam.__new__(IndexedParam)
 
     def __init__(self, *args, **kwd):
-        self._rule          = kwd.pop('rule', _NotValid )
+        self._rule          = kwd.pop('rule', Param.NoValue )
         self._rule          = kwd.pop('initialize', self._rule )
         self._validate      = kwd.pop('validate', None )
         self.domain         = kwd.pop('domain', None )
         self.domain         = kwd.pop('within', self.domain )
         self._mutable       = kwd.pop('mutable', Param.DefaultMutable )
-        self._default_val   = kwd.pop('default', _NotValid )
+        self._default_val   = kwd.pop('default', Param.NoValue )
         self._dense_initialize = kwd.pop('initialize_as_dense', False)
         self._units         = kwd.pop('units', None)
         if self._units is not None:
@@ -272,7 +272,7 @@ def __len__(self):
         component.  If a default value is specified, then the
         length equals the number of items in the component index.
         """
-        if self._default_val is _NotValid:
+        if self._default_val is Param.NoValue:
             return len(self._data)
         return len(self._index)
 
@@ -281,7 +281,7 @@ def __contains__(self, idx):
         Return true if the index is in the dictionary.  If the default value
         is specified, then all members of the component index are valid.
         """
-        if self._default_val is _NotValid:
+        if self._default_val is Param.NoValue:
             return idx in self._data
         return idx in self._index
 
@@ -290,7 +290,7 @@ def __iter__(self):
         Iterate over the keys in the dictionary.  If the default value is
         specified, then iterate over all keys in the component index.
         """
-        if self._default_val is _NotValid:
+        if self._default_val is Param.NoValue:
             return self._data.__iter__()
         return self._index.__iter__()
 
@@ -466,7 +466,7 @@ def set_default(self, val):
         NOTE: this test will not validate the value of function return values.
         """
         if self._constructed \
-                and val is not _NotValid \
+                and val is not Param.NoValue \
                 and type(val) in native_types \
                 and val not in self.domain:
             raise ValueError(
@@ -479,7 +479,7 @@ def default(self):
         Return the value of the parameter default.
 
         Possible values:
-            None            
+            Param.NoValue
                 No default value is provided.
             Numeric         
                 A constant value that is the default value for all undefined 
@@ -498,12 +498,12 @@ def _getitem_when_not_present(self, index):
         # Local values
         #
         val = self._default_val
-        if val is _NotValid:
+        if val is Param.NoValue:
             # We should allow the creation of mutable params without
             # a default value, as long as *solving* a model without
             # reasonable values produces an informative error.
             if self._mutable:
-                # Note: _ParamData defaults to _NotValid
+                # Note: _ParamData defaults to Param.NoValue
                 ans = self._data[index] = _ParamData(self)
                 return ans
             if self.is_indexed():
@@ -887,7 +887,7 @@ def construct(self, data=None):
         # the domain.
         #
         val = self._default_val
-        if val is not _NotValid \
+        if val is not Param.NoValue \
                 and type(val) in native_types \
                 and val not in self.domain:
             raise ValueError(
@@ -900,7 +900,7 @@ def construct(self, data=None):
         #
         # Step #1: initialize data from rule value
         #
-        if self._rule is not _NotValid:
+        if self._rule is not Param.NoValue:
             self._initialize_from(self._rule)
         #
         # Step #2: allow any user-specified (external) data to override
@@ -957,7 +957,7 @@ def _pprint(self):
         """
         Return data that will be printed for this component.
         """
-        if self._default_val is _NotValid:
+        if self._default_val is Param.NoValue:
             default = "None" # for backwards compatibility in reporting
         elif type(self._default_val) is types.FunctionType:
             default = "(function)"


Test patch:
diff --git a/pyomo/core/tests/unit/test_param.py b/pyomo/core/tests/unit/test_param.py
--- a/pyomo/core/tests/unit/test_param.py
+++ b/pyomo/core/tests/unit/test_param.py
@@ -33,10 +33,12 @@
                            cosh, tanh, asinh, acosh, atanh)
 from pyomo.common.log import LoggingIntercept
 from pyomo.common.tempfiles import TempfileManager
-from pyomo.core.base.param import _NotValid, _ParamData 
+from pyomo.core.base.param import _ParamData 
 
 from io import StringIO
 
+NoValue = Param.NoValue
+
 class ParamTester(object):
 
     def setUp(self, **kwds):
@@ -71,7 +73,7 @@ def test_value(self):
             self.assertRaises(TypeError, float, self.instance.A)
             self.assertRaises(TypeError, int, self.instance.A)
 
-        if self.instance.A._default_val is _NotValid:
+        if self.instance.A._default_val is NoValue:
             val_list = self.sparse_data.items()
         else:
             val_list = self.data.items()
@@ -121,7 +123,7 @@ def test_getitem(self):
                 test = self.instance.A[key]
                 self.assertEqual( value(test), val )
             except ValueError:
-                if val is not _NotValid:
+                if val is not NoValue:
                     raise
 
     def test_setitem_index_error(self):
@@ -208,7 +210,7 @@ def test_setitem_default_override(self):
         if len(keys) == len(sparse_keys):
             # No default value possible
             return
-        if self.instance.A._default_val is _NotValid:
+        if self.instance.A._default_val is NoValue:
             # No default value defined
             return
 
@@ -291,7 +293,7 @@ def test_is_indexed(self):
     def test_keys(self):
         test = self.instance.A.keys()
         #self.assertEqual( type(test), list )
-        if self.instance.A._default_val is _NotValid:
+        if self.instance.A._default_val is NoValue:
             self.assertEqual( sorted(test), sorted(self.sparse_data.keys()) )
         else:
             self.assertEqual( sorted(test), sorted(self.data.keys()) )
@@ -304,7 +306,7 @@ def test_values(self):
             test = self.instance.A.values()
             #self.assertEqual( type(test), list )
             test = zip(self.instance.A.keys(), test)
-            if self.instance.A._default_val is _NotValid:
+            if self.instance.A._default_val is NoValue:
                 self.validateDict(self.sparse_data.items(), test)
             else:
                 self.validateDict(self.data.items(), test)
@@ -316,12 +318,12 @@ def test_values(self):
     def test_items(self):
         expectException = False
         #                  len(self.sparse_data) < len(self.data) and \
-        #                  not self.instance.A._default_val is _NotValid and \
+        #                  not self.instance.A._default_val is NoValue and \
         #                  not self.instance.A.mutable
         try:
             test = self.instance.A.items()
             #self.assertEqual( type(test), list )
-            if self.instance.A._default_val is _NotValid:
+            if self.instance.A._default_val is NoValue:
                 self.validateDict(self.sparse_data.items(), test)
             else:
                 self.validateDict(self.data.items(), test)
@@ -342,7 +344,7 @@ def test_itervalues(self):
         try:
             test = self.instance.A.values()
             test = zip(self.instance.A.keys(), test)
-            if self.instance.A._default_val is _NotValid:
+            if self.instance.A._default_val is NoValue:
                 self.validateDict(self.sparse_data.items(), test)
             else:
                 self.validateDict(self.data.items(), test)
@@ -358,7 +360,7 @@ def test_iteritems(self):
         #                  not self.instance.A.mutable
         try:
             test = self.instance.A.items()
-            if self.instance.A._default_val is _NotValid:
+            if self.instance.A._default_val is NoValue:
                 self.validateDict(self.sparse_data.items(), test)
             else:
                 self.validateDict(self.data.items(), test)
@@ -404,7 +406,7 @@ def test_sparse_iteritems(self):
 
     def test_len(self):
         #"""Check the use of len"""
-        if self.instance.A._default_val is _NotValid:
+        if self.instance.A._default_val is NoValue:
             self.assertEqual( len(self.instance.A), len(self.sparse_data) )
             self.assertEqual( len(list(self.instance.A.keys())), len(self.sparse_data) )
         else:
@@ -421,14 +423,14 @@ def test_get_default(self):
             # nothing to test
             return
         idx = list(set(self.data) - set(self.sparse_data))[0]
-        expectException = self.instance.A._default_val is _NotValid \
+        expectException = self.instance.A._default_val is NoValue \
                           and not self.instance.A.mutable
         try:
             test = self.instance.A[idx]
             if expectException:
                 self.fail("Expected the test to raise an exception")
             self.assertFalse(expectException)
-            expectException = self.instance.A._default_val is _NotValid
+            expectException = self.instance.A._default_val is NoValue
             try:
                 ans = value(test)
                 self.assertEquals(ans, value(self.instance.A._default_val))
@@ -452,7 +454,7 @@ def setUp(self, **kwds):
         ParamTester.setUp(self, mutable=True, initialize={1:1.3}, **kwds)
 
         self.sparse_data = {1:1.3}
-        self.data = {1:1.3, 3:_NotValid}
+        self.data = {1:1.3, 3:NoValue}
 
 class ArrayParam_mutable_sparse_intDefault\
           (ParamTester, unittest.TestCase):
@@ -581,7 +583,7 @@ def setUp(self, **kwds):
         ParamTester.setUp(self, mutable=False, initialize={1:1.3}, **kwds)
 
         self.sparse_data = {1:1.3}
-        self.data = {1:1.3, 3:_NotValid}
+        self.data = {1:1.3, 3:NoValue}
 
 
 class ArrayParam_immutable_sparse_intDefault\
@@ -811,7 +813,7 @@ def setUp(self, **kwds):
 
     def test_value_scalar(self):
         #"""Check the value of the parameter"""
-        if self.data.get(None,_NotValid) is _NotValid:
+        if self.data.get(None,NoValue) is NoValue:
             self.assertRaises(ValueError, value, self.instance.A)
             self.assertRaises(TypeError, float, self.instance.A)
             self.assertRaises(TypeError, int, self.instance.A)
@@ -826,8 +828,8 @@ def test_value_scalar(self):
 
     def test_call(self):
         #"""Check the use of the __call__ method"""
-        if self.sparse_data.get(None,0) is _NotValid or \
-           self.data.get(None,_NotValid) is _NotValid: #not self.sparse_data:
+        if self.sparse_data.get(None,0) is NoValue or \
+           self.data.get(None,NoValue) is NoValue: #not self.sparse_data:
             self.assertRaisesRegexp(
                 ValueError, ".*currently set to an invalid value",
                 self.instance.A.__call__ )
@@ -836,8 +838,8 @@ def test_call(self):
 
     def test_get_valueattr(self):
         self.assertEqual( self.instance.A._value,
-                          self.sparse_data.get(None,_NotValid) )
-        if self.data.get(None,0) is _NotValid: #not self.sparse_data:
+                          self.sparse_data.get(None,NoValue) )
+        if self.data.get(None,0) is NoValue: #not self.sparse_data:
             try:
                 value(self.instance.A)
                 self.fail("Expected value error")
@@ -852,8 +854,8 @@ def test_set_valueattr(self):
         self.assertEqual(self.instance.A(), 4.3)
 
     def test_get_value(self):
-        if self.sparse_data.get(None,0) is _NotValid or \
-           self.data.get(None,_NotValid) is _NotValid: #not self.sparse_data:
+        if self.sparse_data.get(None,0) is NoValue or \
+           self.data.get(None,NoValue) is NoValue: #not self.sparse_data:
             try:
                 value(self.instance.A)
                 self.fail("Expected value error")
@@ -886,7 +888,7 @@ def setUp(self, **kwds):
         ScalarTester.setUp(self, mutable=True, **kwds)
 
         self.sparse_data = {}
-        self.data = {None:_NotValid}
+        self.data = {None:NoValue}
 
 
 class ScalarParam_mutable_init(ScalarTester, unittest.TestCase):
@@ -1355,7 +1357,7 @@ def test_nonnumeric(self):
 q : Size=3, Index=Any, Domain=Any, Default=None, Mutable=True
     Key : Value
       1 : None
-      2 : <class 'pyomo.core.base.param._NotValid'>
+      2 : <class 'pyomo.core.base.param.Param.NoValue'>
       a : b
             """.strip())