Repository: lmfit/lmfit-py

Issue:
Inconsistencies with Minimizer "nan_policy" and Model "missing"
There is an inconsistency between Model and Minimizer in setting a policy for NaNs.  When creating a Model, you can specify "missing", but the usage is not consistent with "nan_policy".  Furthermore, you cannot easily set the policy when doing a fit, only when setting up a Model.  

I think Model "missing" should be deprecated (suggestions for how to do that effectively most welcome!)  and "nan_policy" be added to `Model.fit()`.   Any thoughts or suggestions? 


Discussion:

Resolving patch:
diff --git a/examples/doc_model_with_nan_policy.py b/examples/doc_model_with_nan_policy.py
new file mode 100644
--- /dev/null
+++ b/examples/doc_model_with_nan_policy.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+#<examples/doc_model_with_nan_policy.py>
+import numpy as np
+
+from lmfit.models import  GaussianModel
+
+import matplotlib.pyplot as plt
+
+data = np.loadtxt('model1d_gauss.dat')
+x = data[:, 0]
+y = data[:, 1]
+
+y[44] = np.nan
+y[65] = np.nan
+
+# nan_policy = 'raise'
+# nan_policy = 'propagate'
+nan_policy = 'omit'
+
+gmodel = GaussianModel()
+result = gmodel.fit(y, x=x, amplitude=5, center=6, sigma=1,
+                    nan_policy=nan_policy)
+
+print(result.fit_report())
+
+# make sure nans are removed for plotting:
+
+x_ = x[np.where(np.isfinite(y))]
+y_ = y[np.where(np.isfinite(y))]
+
+plt.plot(x_, y_,         'bo')
+plt.plot(x_, result.init_fit, 'k--')
+plt.plot(x_, result.best_fit, 'r-')
+plt.show()
+#<end examples/doc_model_with_nan_policy.py>
diff --git a/lmfit/minimizer.py b/lmfit/minimizer.py
--- a/lmfit/minimizer.py
+++ b/lmfit/minimizer.py
@@ -31,6 +31,11 @@
 from scipy.stats import norm as norm_dist
 import six
 
+try:
+    from pandas import isnull
+except ImportError:
+    isnull = np.isnan
+
 # use locally modified version of uncertainties package
 from . import uncertainties
 from .parameter import Parameter, Parameters
@@ -486,15 +491,17 @@ def __residual(self, fvars, apply_bounds_transformation=True):
         self.result.nfev += 1
 
         out = self.userfcn(params, *self.userargs, **self.userkws)
-        out = _nan_policy(out, nan_policy=self.nan_policy)
 
         if callable(self.iter_cb):
             abort = self.iter_cb(params, self.result.nfev, out,
                                  *self.userargs, **self.userkws)
             self._abort = self._abort or abort
         self._abort = self._abort and self.result.nfev > len(fvars)
+
         if not self._abort:
-            return np.asarray(out).ravel()
+            return _nan_policy(np.asarray(out).ravel(),
+                               nan_policy=self.nan_policy)
+
 
     def __jacobian(self, fvars):
         """Reuturn analytical jacobian to be used with Levenberg-Marquardt.
@@ -1783,17 +1790,36 @@ def _make_random_gen(seed):
                      ' instance' % seed)
 
 
-def _nan_policy(a, nan_policy='raise', handle_inf=True):
+VALID_NAN_POLICIES = ('propagate', 'omit', 'raise')
+def validate_nan_policy(policy):
+    """validate, rationalize nan_policy, for back compatibility
+    and compatibility with Pandas missing convention
+    """
+    if policy in VALID_NAN_POLICIES:
+        return policy
+    if policy is None:
+        policy = 'propagate'
+
+    policy = policy.lower()
+    if policy == 'drop':
+        policy = 'omit'
+    if policy == 'none':
+        policy = 'propagate'
+    if policy not in VALID_NAN_POLICIES:
+        raise ValueError("nan_policy must be 'propagate', 'omit', or 'raise'.")
+    return policy
+
+def _nan_policy(arr, nan_policy='raise', handle_inf=True):
     """Specify behaviour when an array contains numpy.nan or numpy.inf.
 
     Parameters
     ----------
-    a : array_like
+    arr : array_like
         Input array to consider.
     nan_policy : str, optional
         One of:
 
-        'raise' - raise a `ValueError` if `a` contains NaN
+        'raise' - raise a `ValueError` if `arr` contains NaN (default)
         'propagate' - propagate NaN
         'omit' - filter NaN from input array
     handle_inf : bool, optional
@@ -1809,41 +1835,36 @@ def _nan_policy(a, nan_policy='raise', handle_inf=True):
     scipy/stats/stats.py/_contains_nan
 
     """
-    policies = ['propagate', 'raise', 'omit']
+    nan_policy = validate_nan_policy(nan_policy)
 
     if handle_inf:
-        handler_func = lambda a: ~np.isfinite(a)
+        handler_func = lambda x: ~np.isfinite(x)
     else:
-        handler_func = np.isnan
-
-    if nan_policy == 'propagate':
-        # nan values are ignored.
-        return a
-    elif nan_policy == 'raise':
+        handler_func = isnull
+
+    if nan_policy == 'omit':
+        # mask locates any values to remove
+        mask = ~handler_func(arr)
+        if not np.all(mask):  # there are some NaNs/infs/missing values
+            return arr[mask]
+    if nan_policy == 'raise':
         try:
             # Calling np.sum to avoid creating a huge array into memory
             # e.g. np.isnan(a).any()
             with np.errstate(invalid='ignore'):
-                contains_nan = handler_func(np.sum(a))
+                contains_nan = handler_func(np.sum(arr))
         except TypeError:
             # If the check cannot be properly performed we fallback to omiting
             # nan values and raising a warning. This can happen when attempting to
             # sum things that are not numbers (e.g. as in the function `mode`).
             contains_nan = False
-            warnings.warn("The input array could not be properly checked for nan "
-                          "values. nan values will be ignored.", RuntimeWarning)
+            warnings.warn("The input array could not be checked for NaNs. "
+                          "NaNs will be ignored.", RuntimeWarning)
 
         if contains_nan:
             raise ValueError("The input contains nan values")
-        return a
+    return arr
 
-    elif nan_policy == 'omit':
-        # nans are filtered
-        mask = handler_func(a)
-        return a[~mask]
-    else:
-        raise ValueError("nan_policy must be one of {%s}" %
-                         ', '.join("'%s'" % s for s in policies))
 
 
 def minimize(fcn, params, method='leastsq', args=None, kws=None,
diff --git a/lmfit/model.py b/lmfit/model.py
--- a/lmfit/model.py
+++ b/lmfit/model.py
@@ -13,6 +13,7 @@
 from scipy.stats import t
 
 from . import Minimizer, Parameter, Parameters
+from .minimizer import validate_nan_policy
 from .confidence import conf_interval
 from .printfuncs import ci_report, fit_report
 
@@ -53,29 +54,21 @@ def no_op(*args, **kwargs):
 
         return no_op
 
-
 class Model(object):
-    """Create a model from a user-supplied model function.
-
-    The model function will normally take an independent variable
-    (generally, the first argument) and a series of arguments that are
-    meant to be parameters for the model. It will return an array of
-    data to model some data as for a curve-fitting problem.
-
-    """
-
     _forbidden_args = ('data', 'weights', 'params')
     _invalid_ivar = "Invalid independent variable name ('%s') for function %s"
     _invalid_par = "Invalid parameter name ('%s') for function %s"
-    _invalid_missing = "missing must be None, 'none', 'drop', or 'raise'."
-    _valid_missing = (None, 'none', 'drop', 'raise')
-
     _invalid_hint = "unknown parameter hint '%s' for param '%s'"
     _hint_names = ('value', 'vary', 'min', 'max', 'expr')
 
     def __init__(self, func, independent_vars=None, param_names=None,
-                 missing='none', prefix='', name=None, **kws):
-        """
+                 nan_policy='raise', missing=None, prefix='', name=None, **kws):
+        """Create a model from a user-supplied model function.
+        The model function will normally take an independent variable
+        (generally, the first argument) and a series of arguments that are
+        meant to be parameters for the model. It will return an array of
+        data to model some data as for a curve-fitting problem.
+
         Parameters
         ----------
         func : callable
@@ -85,16 +78,11 @@ def __init__(self, func, independent_vars=None, param_names=None,
         param_names : list of str, optional
             Names of arguments to func that are to be made into parameters
             (default is None).
+        nan_policy : str, optional
+            How to handle NaN and missing values in data. Must be one of
+            'raise' (default), 'propagate', or 'omit'. See Note below.
         missing : str, optional
-            How to handle NaN and missing values in data. One of:
-
-            - 'none' or None : Do not check for null or missing values (default).
-
-            - 'drop' : Drop null or missing observations in data. If pandas is
-              installed, `pandas.isnull` is used, otherwise `numpy.isnan` is used.
-            - 'raise' : Raise a (more helpful) exception when data contains
-              null or missing values.
-
+            Synonym for 'nan_policy' for backward compatibility
         prefix : str, optional
             Prefix used for the model.
         name : str, optional
@@ -111,6 +99,20 @@ def __init__(self, func, independent_vars=None, param_names=None,
         2. The model function must return an array that will be the same
         size as the data being modeled.
 
+        3. nan_policy sets what to do when a NaN or missing value is
+        seen in the data. Should be one of:
+
+           - 'raise' : Raise a ValueError (default)
+
+           - 'propagate' : do nothing.
+
+           -  'omit' : (was 'drop') drop missing data.
+
+        4. The `missing` argument is deprecated in lmfit 0.9.8 and will be
+        removed in a later version. Use `nan_policy instead, as it is
+        consistent with the Minimizer class.
+
+
         Examples
         --------
         The model function will normally take an independent variable (generally,
@@ -132,6 +134,7 @@ def __init__(self, func, independent_vars=None, param_names=None,
         >>> print(gmodel.param_names, gmodel.independent_vars)
         ['amp', 'cen', 'wid'], ['x']
 
+
         """
         self.func = func
         self._prefix = prefix
@@ -139,9 +142,10 @@ def __init__(self, func, independent_vars=None, param_names=None,
         self.independent_vars = independent_vars
         self._func_allargs = []
         self._func_haskeywords = False
-        if missing not in self._valid_missing:
-            raise ValueError(self._invalid_missing)
-        self.missing = missing
+        if missing is not None:
+            nan_policy = missing
+        self.nan_policy = validate_nan_policy(nan_policy)
+
         self.opts = kws
         self.param_hints = OrderedDict()
         # the following has been changed from OrderedSet for the time being
@@ -495,17 +499,6 @@ def _residual(self, params, data, weights, **kwargs):
             diff *= weights
         return np.asarray(diff).ravel()  # for compatibility with pandas.Series
 
-    def _handle_missing(self, data):
-        """Handle missing data."""
-        if self.missing == 'raise':
-            if np.any(isnull(data)):
-                raise ValueError("Data contains a null value.")
-        elif self.missing == 'drop':
-            mask = ~isnull(data)
-            if np.all(mask):
-                return None  # short-circuit this -- no missing values
-            mask = np.asarray(mask)  # for compatibility with pandas.Series
-            return mask
 
     def _strip_prefix(self, name):
         npref = len(self._prefix)
@@ -602,7 +595,7 @@ def eval_components(self, params=None, **kwargs):
 
     def fit(self, data, params=None, weights=None, method='leastsq',
             iter_cb=None, scale_covar=True, verbose=False, fit_kws=None,
-            **kwargs):
+            nan_policy=None, **kwargs):
         """Fit the model to the data using the supplied Parameters.
 
         Parameters
@@ -617,18 +610,20 @@ def fit(self, data, params=None, weights=None, method='leastsq',
         method : str, optional
             Name of fitting method to use (default is `'leastsq'`).
         iter_cb : callable, optional
-             Callback function to call at each iteration (default is None).
+            Callback function to call at each iteration (default is None).
         scale_covar : bool, optional
-             Whether to automatically scale the covariance matrix when
-             calculating uncertainties (default is True, `leastsq` method only).
+            Whether to automatically scale the covariance matrix when
+            calculating uncertainties (default is True, `leastsq` method only).
         verbose: bool, optional
-             Whether to print a message when a new parameter is added because
-             of a hint (default is True).
+            Whether to print a message when a new parameter is added because
+            of a hint (default is True).
+        nan_policy : str, optional, one of 'raise' (default), 'propagate', or 'omit'.
+            What to do when encountering NaNs when fitting Model
         fit_kws: dict, optional
-             Options to pass to the minimizer being used.
+            Options to pass to the minimizer being used.
         **kwargs: optional
-             Arguments to pass to the  model function, possibly overriding
-             params.
+            Arguments to pass to the  model function, possibly overriding
+            params.
 
         Returns
         -------
@@ -713,9 +708,12 @@ def fit(self, data, params=None, weights=None, method='leastsq',
                 kwargs[var] = np.asfarray(var_data)
 
         # Handle null/missing values.
+        if nan_policy is not None:
+            self.nan_policy = validate_nan_policy(nan_policy)
+
         mask = None
-        if self.missing not in (None, 'none'):
-            mask = self._handle_missing(data)  # This can raise.
+        if self.nan_policy == 'omit':
+            mask = ~isnull(data)
             if mask is not None:
                 data = data[mask]
             if weights is not None:
@@ -723,8 +721,10 @@ def fit(self, data, params=None, weights=None, method='leastsq',
 
         # If independent_vars and data are alignable (pandas), align them,
         # and apply the mask from above if there is one.
+
         for var in self.independent_vars:
             if not np.isscalar(kwargs[var]):
+                # print("Model fit align ind dep ", var, mask.sum())
                 kwargs[var] = _align(kwargs[var], mask, data)
 
         if fit_kws is None:
@@ -732,7 +732,7 @@ def fit(self, data, params=None, weights=None, method='leastsq',
 
         output = ModelResult(self, params, method=method, iter_cb=iter_cb,
                              scale_covar=scale_covar, fcn_kws=kwargs,
-                             **fit_kws)
+                             nan_policy=self.nan_policy, **fit_kws)
         output.fit(data=data, weights=weights)
         output.components = self.components
         return output
@@ -817,8 +817,8 @@ def __init__(self, left, right, op, **kws):
         # we assume that all the sub-models have the same independent vars
         if 'independent_vars' not in kws:
             kws['independent_vars'] = self.left.independent_vars
-        if 'missing' not in kws:
-            kws['missing'] = self.left.missing
+        if 'nan_policy' not in kws:
+            kws['nan_policy'] = self.left.nan_policy
 
         def _tmp(self, *args, **kws):
             pass
@@ -883,7 +883,8 @@ class ModelResult(Minimizer):
 
     def __init__(self, model, params, data=None, weights=None,
                  method='leastsq', fcn_args=None, fcn_kws=None,
-                 iter_cb=None, scale_covar=True, **fit_kws):
+                 iter_cb=None, scale_covar=True, nan_policy='raise',
+                 **fit_kws):
         """
         Parameters
         ----------
@@ -905,6 +906,8 @@ def __init__(self, model, params, data=None, weights=None,
             Function to call on each iteration of fit.
         scale_covar : bool, optional
             Whether to scale covariance matrix for uncertainty evaluation.
+        nan_policy : str, optional, one of 'raise' (default), 'propagate', or 'omit'.
+            What to do when encountering NaNs when fitting Model
         **fit_kws : optional
             Keyword arguments to send to minimization routine.
         """
@@ -915,10 +918,11 @@ def __init__(self, model, params, data=None, weights=None,
         self.ci_out = None
         self.init_params = deepcopy(params)
         Minimizer.__init__(self, model._residual, params, fcn_args=fcn_args,
-                           fcn_kws=fcn_kws, iter_cb=iter_cb,
+                           fcn_kws=fcn_kws, iter_cb=iter_cb, nan_policy=nan_policy,
                            scale_covar=scale_covar, **fit_kws)
 
-    def fit(self, data=None, params=None, weights=None, method=None, **kwargs):
+    def fit(self, data=None, params=None, weights=None, method=None,
+            nan_policy=None, **kwargs):
         """Re-perform fit for a Model, given data and params.
 
         Parameters
@@ -931,6 +935,8 @@ def fit(self, data=None, params=None, weights=None, method=None, **kwargs):
             Weights to multiply (data-model) for fit residual.
         method : str, optional
             Name of minimization method to use (default is `'leastsq'`).
+        nan_policy : str, optional, one of 'raise' (default), 'propagate', or 'omit'.
+            What to do when encountering NaNs when fitting Model
         **kwargs : optional
             Keyword arguments to send to minimization routine.
 
@@ -943,11 +949,13 @@ def fit(self, data=None, params=None, weights=None, method=None, **kwargs):
             self.weights = weights
         if method is not None:
             self.method = method
+        if nan_policy is not None:
+            self.nan_policy = validate_nan_policy(nan_policy)
+
         self.ci_out = None
         self.userargs = (self.data, self.weights)
         self.userkws.update(kwargs)
         self.init_fit = self.model.eval(params=self.params, **self.userkws)
-
         _ret = self.minimize(method=self.method)
 
         for attr in dir(_ret):


Test patch:
diff --git a/tests/test_model.py b/tests/test_model.py
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -563,6 +563,44 @@ def gauss(x, sigma, mu, A):
         self.assertTrue(result.params['pos_delta'].value > 0)
 
 
+    def test_model_nan_policy(self):
+
+        x = np.linspace(0, 10, 201)
+        np.random.seed(0)
+        y  = gaussian(x, 10.0,  6.15, 0.8)
+        y += gaussian(x,  8.0,  6.35, 1.1)
+        y += gaussian(x,  0.25, 6.00, 7.5)
+        y += np.random.normal(size=len(x), scale=0.5)
+
+        y[55] = y[91] = np.nan
+        mod = PseudoVoigtModel()
+        params = mod.make_params(amplitude=20, center=5.5,
+                                  sigma=1, fraction=0.25)
+        params['fraction'].vary=False
+        # with raise, should get a ValueError
+        result = lambda: mod.fit(y, params, x=x, nan_policy='raise')
+        self.assertRaises(ValueError, result)
+
+        # with propagate, should get no error, but bad results
+        result = mod.fit(y, params, x=x, nan_policy='propagate')
+        self.assertTrue(result.success)
+        self.assertTrue(np.isnan(result.chisqr))
+        self.assertTrue(np.isnan(result.aic))
+        self.assertFalse(result.errorbars)
+        self.assertTrue(result.params['amplitude'].stderr==0)
+        self.assertTrue(abs(result.params['amplitude'].value - 20.0) < 0.001)
+
+        # with omit, should get good results
+        result = mod.fit(y, params, x=x, nan_policy='omit')
+        self.assertTrue(result.success)
+        self.assertTrue(result.chisqr > 2.0)
+        self.assertTrue(result.aic  < -100)
+        self.assertTrue(result.errorbars)
+        self.assertTrue(result.params['amplitude'].stderr > 0.1)
+        self.assertTrue(abs(result.params['amplitude'].value - 20.0) < 5.0)
+        self.assertTrue(abs(result.params['center'].value   - 6.0) < 0.5)
+
+
 class TestLinear(CommonTests, unittest.TestCase):
 
     def setUp(self):