def __init__(self, x, d, filter_size=None, iterations=<NUM_LIT:4>, clip=False,<EOL>clipsize=<NUM_LIT:3>, damp=<NUM_LIT>, blocksize=None, donorm=True):

self.x = x<EOL>self.d = d<EOL>self.damp = damp<EOL>self.clip = clip<EOL>self.iterations = iterations<EOL>self.donorm = donorm<EOL>self.blocksize = blocksize<EOL>if filter_size is None:<EOL><INDENT>self.filter_size = self._default_filter_size()<EOL><DEDENT>else:<EOL><INDENT>self.filter_size = filter_size<EOL><DEDENT>self.clipsize = clipsize * self.filter_size<EOL>

1-d barnes interpolation. Give data points d at locations x.

        See [1]_, equations 1-7 for implementation.

        Parameters
        ----------
        x : ndarray, 1-dimensional
            input positions, x values for data points

        d : ndarray, 1-dimensional
            input values, y values for data points

        filter_size : float, optional.
            control parameter for weight function (sigma), should be the
            average data spacing. Defaults to the average distance for
            sorted ``x``.

        iterations : integer, optional
            how many iterations to perform. only two needed with a high damping
            Defaults to 4

        clip : boolean, optional
            whether to clip the number of data points used by the filtersize
            Default is False

        clipsize : float, optional
            Total clipsize is determined by clipsize * filter_size
            Default is 3

        damp : float, optional
            the damping parameter used in accelerating convergence. Default
            is 0.95

        blocksize : Int or None, optional
            Memory-based performance feature. For very large ``x`` or ``d``
            calculating the distance matrix between each point can require
            excessive memory overhead. To avoid this, set ``blocksize`` to
            a moderate int, causing the Barnes to compute the distance
            matrix in blocks. Does not change the final results. Default is
            None, which uses all the data all at once, using lots of mem.

        donorm : bool, optional
            If False, uses an old, incorrect method to evaluate the interpolant
            rather than the correct version. Default is True. If you're using
            this, set it to True.

        Examples
        --------
        >>> import numpy as np
        >>> x_known = np.linspace(0, np.pi, 10)
        >>> d_known = np.sin(x_known)
        >>> barnes = BarnesInterpolation1D(x_known, d_known)
        >>> x_check = np.linspace(0, np.pi, 11)
        >>> np.allclose(barnes(x_check), np.sin(x_check), atol=1e-2)
        True

        References
        ----------
        .. [1] S. E. Koch, M. DesJardins, P. J. Kocin, J. Climate Appl.
                Meteor. 22 1487-1503 (1983)


return (a[:, None] - b[None, :])**<NUM_LIT:2><EOL>

Pairwise distance between each point in `a` and each point in `b`


sigma = sigma or self.filter_size<EOL>if not self.clip:<EOL><INDENT>o = np.exp(-rsq / (<NUM_LIT:2>*sigma**<NUM_LIT:2>))<EOL><DEDENT>else:<EOL><INDENT>o = np.zeros(rsq.shape, dtype='<STR_LIT:float>')<EOL>m = (rsq < self.clipsize**<NUM_LIT:2>)<EOL>o[m] = np.exp(-rsq[m] / (<NUM_LIT:2>*sigma**<NUM_LIT:2>))<EOL><DEDENT>return o<EOL>


if self.donorm:<EOL><INDENT>return self._newcall(rvecs)<EOL><DEDENT>else:<EOL><INDENT>return self._oldcall(rvecs)<EOL><DEDENT>

Get the values interpolated at positions rvecs


def _eval_firstorder(self, rvecs, data, sigma):

if not self.blocksize:<EOL><INDENT>dist_between_points = self._distance_matrix(rvecs, self.x)<EOL>gaussian_weights = self._weight(dist_between_points, sigma=sigma)<EOL>return gaussian_weights.dot(data) / gaussian_weights.sum(axis=<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>ans = np.zeros(rvecs.shape[<NUM_LIT:0>], dtype='<STR_LIT:float>')<EOL>bs = self.blocksize<EOL>for a in range(<NUM_LIT:0>, rvecs.shape[<NUM_LIT:0>], bs):<EOL><INDENT>dist = self._distance_matrix(rvecs[a:a+bs], self.x)<EOL>weights = self._weight(dist, sigma=sigma)<EOL>ans[a:a+bs] += weights.dot(data) / weights.sum(axis=<NUM_LIT:1>)<EOL><DEDENT>return ans<EOL><DEDENT>


<EOL>sigma = <NUM_LIT:1>*self.filter_size<EOL>out = self._eval_firstorder(rvecs, self.d, sigma)<EOL>ondata = self._eval_firstorder(self.x, self.d, sigma)<EOL>for i in range(self.iterations):<EOL><INDENT>out += self._eval_firstorder(rvecs, self.d-ondata, sigma)<EOL>ondata += self._eval_firstorder(self.x, self.d-ondata, sigma)<EOL>sigma *= self.damp<EOL><DEDENT>return out<EOL>


g = self.filter_size<EOL>dist0 = self._distance_matrix(self.x, self.x)<EOL>dist1 = self._distance_matrix(rvecs, self.x)<EOL>tmp = self._weight(dist0, g).dot(self.d)<EOL>out = self._weight(dist1, g).dot(self.d)<EOL>for i in range(self.iterations):<EOL><INDENT>out = out + self._weight(dist1, g).dot(self.d - tmp)<EOL>tmp = tmp + self._weight(dist0, g).dot(self.d - tmp)<EOL>g *= self.damp<EOL><DEDENT>return out<EOL>


super(BarnesInterpolationND, self).__init__(*args, **kwargs)<EOL>

A class for barnes interpolation in N dimensions.

Parameters
----------
x : ndarray, 2-dimensional
    input positions, x values for data points. x[i] is the ith position

d : ndarray, 1-dimensional
    input values, y values for data points. Same number of points as
    x has positions.

Examples
--------
>>> import numpy as np
>>> np.random.seed(0)
>>> x_known = np.random.randn(20, 2)
>>> func = lambda x: np.exp(-(x[:, 0]**2 + x[:, 1]**2))
>>> d_known = func(x_known)
>>> barnes_nd = BarnesInterpolationND(x_known, d_known, iterations=35)
>>> np.allclose(barnes_nd(x_known), func(x_known), atol=1e-2)
True

See Also
--------
BarnesInterpolation1D


def sq(x): return (x * x)<EOL>matrix = sq(a[:, <NUM_LIT:0>][:, None] - b[:, <NUM_LIT:0>][None, :])<EOL>for x, y in zip(a.T[<NUM_LIT:1>:], b.T[<NUM_LIT:1>:]):<EOL><INDENT>matrix += sq(x[:, None] - y[None, :])<EOL><DEDENT>return matrix<EOL>


def __init__(self, func, args=(), window=(<NUM_LIT:0.>, <NUM_LIT:1.>), degree=<NUM_LIT:3>, evalpts=<NUM_LIT:4>):

self.args = args<EOL>self.func = func<EOL>self.window = window<EOL>self.set_order(evalpts, degree)<EOL>

A 1D Chebyshev approximation / interpolation for an ND function,
        approximating (N-1)D in in the last dimension.

        Parameters
        ----------
        func : callable
            A function that takes scalar arguments (1D) and returns a N
            dimensional array corresponding to that scalar. Make it such that,
            for an array x, f(x)[.....,a] corresponds to f(x[a])

        args : tuple [optional]
            extra arguments to pass to func

        window : tuple (length 2)
            The bounds of the function over which we desire the interpolation

        degree : integer
            Degree of the Chebyshev interpolating polynomial

        evalpts : integer
            Number of Chebyshev points to evaluate the function at

        Examples
        --------
        >>> import numpy as np
        >>> func = lambda x: np.sin(x)
        >>> cheb = ChebyshevInterpolation1D(func, window=(0, np.pi), degree=9,
        >>>                                 evalpts=11)
        >>> np.allclose(cheb(1.0), np.sin(1.0), atol=1e-7)
        True


return ((<NUM_LIT:2> * x - self.window[<NUM_LIT:1>] - self.window[<NUM_LIT:0>]) /<EOL>(self.window[<NUM_LIT:1>] - self.window[<NUM_LIT:0>]))<EOL>

Convert windowdow coordinates to cheb coordinates [-1,1]


return <NUM_LIT:0.5> * (self.window[<NUM_LIT:0>] + self.window[<NUM_LIT:1>] +<EOL>c * (self.window[<NUM_LIT:1>] - self.window[<NUM_LIT:0>]))<EOL>

Convert cheb coordinates to windowdow coordinates


coeffs = [<NUM_LIT:0>]*self.degree<EOL>N = float(self.evalpts)<EOL>lvals = np.arange(self.evalpts).astype('<STR_LIT:float>')<EOL>xpts = self._c2x(np.cos(np.pi*(lvals + <NUM_LIT:0.5>)/N))<EOL>fpts = np.rollaxis(self.func(xpts, *self.args), -<NUM_LIT:1>)<EOL>for a in range(self.degree):<EOL><INDENT>inner = [<EOL>fpts[b] * np.cos(np.pi*a*(lvals[b]+<NUM_LIT:0.5>)/N)<EOL>for b in range(self.evalpts)<EOL>]<EOL>coeffs[a] = <NUM_LIT>/N * np.sum(inner, axis=<NUM_LIT:0>)<EOL><DEDENT>coeffs[<NUM_LIT:0>] *= <NUM_LIT:0.5><EOL>self._coeffs = np.array(coeffs)<EOL>

Calculate the coefficients based on the func, degree, and
        interpolating points.
        _coeffs is a [order, N,M,....] array

        Notes
        -----
        Moved the -c0 to the coefficients defintion
        app -= 0.5 * self._coeffs[0] -- moving this to the coefficients


weights = np.diag(np.ones(k+<NUM_LIT:1>))[k]<EOL>return np.polynomial.chebyshev.chebval(self._x2c(x), weights)<EOL>

Evaluates an individual Chebyshev polynomial `k` in coordinate space
with proper transformation given the window


return np.polynomial.chebyshev.chebval(<EOL>self._x2c(x), self._coeffs, tensor=True)<EOL>

Approximates `func` at the coordinates x, which must be in the window.

.. math::
    f(x) = \sum_{k=0}^{N-1} c_k T_k(x) - co/2

Output is in the format [A,...,x]


def __init__(self, modelstr, varmap, registry={}):

self.modelstr = modelstr<EOL>self.varmap = varmap<EOL>self.registry = registry<EOL>self.ivarmap = {v:k for k, v in iteritems(self.varmap)}<EOL>self.check_consistency()<EOL>

An abstraction for defining how to combine components into a complete
model as well as derivatives with respect to different variables.

Parameters
-----------
modelstr : dict of strings
    (Using a custom model) The actual equations used to defined the model.
    At least one eq. is required under the key `full`. Other partial
    update equations can be added where the variable name (e.g. 'dI')
    denotes a partial update of I, for example::

        {'full': 'H(P)', 'dP': 'H(dP)'}

varmap : dict
    (Using a custom model) The mapping of variables in the modelstr
    equations to actual component types. For example::

        {'P': 'obj', 'H': 'psf'}

registry : dict
    A mapping of categories to components which this model will
    accept. For example::

        {
            'obj': [objs.PlatonicSpheresCollection],
            'psf': [psfs.Gaussian4DPoly, psfs.GaussianMomentExpansion]
        }


error = False<EOL>regex = re.compile('<STR_LIT>')<EOL>if '<STR_LIT>' not in self.modelstr:<EOL><INDENT>raise ModelError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>for name, eq in iteritems(self.modelstr):<EOL><INDENT>var = regex.findall(eq)<EOL>for v in var:<EOL><INDENT>v = re.sub(r"<STR_LIT>", '<STR_LIT>', v)<EOL>if v not in self.varmap:<EOL><INDENT>log.error(<EOL>"<STR_LIT>" %<EOL>(v, name, eq, self.varmap)<EOL>)<EOL>error = True<EOL><DEDENT><DEDENT><DEDENT>if error:<EOL><INDENT>raise ModelError('<STR_LIT>')<EOL><DEDENT>

Make sure that the required comps are included in the list of
components supplied by the user. Also check that the parameters are
consistent across the many components.


error = False<EOL>compcats = [c.category for c in comps]<EOL>for k, v in iteritems(self.varmap):<EOL><INDENT>if k not in self.modelstr['<STR_LIT>']:<EOL><INDENT>log.warn('<STR_LIT>' % (k,v))<EOL><DEDENT>if v not in compcats:<EOL><INDENT>log.error('<STR_LIT>' % (k,v))<EOL>error = True<EOL><DEDENT><DEDENT>if error:<EOL><INDENT>raise ModelError('<STR_LIT>')<EOL><DEDENT>

Check that the list of components `comp` is compatible with both the
varmap and modelstr for this Model


return '<STR_LIT:d>'+name<EOL>

Transform a variable name into a derivative


return self.modelstr['<STR_LIT>']<EOL>


name = self.diffname(self.ivarmap[category])<EOL>return self.modelstr.get(name)<EOL>

Get the equation corresponding to a variation wrt category. For example
if::

    modelstr = {
        'full' :'H(I) + B',
        'dH' : 'dH(I)',
        'dI' : 'H(dI)',
        'dB' : 'dB'
    }

    varmap = {'H': 'psf', 'I': 'obj', 'B': 'bkg'}

then ``get_difference_model('obj') == modelstr['dI'] == 'H(dI)'``


def map_vars(self, comps, funcname='<STR_LIT>', diffmap=None, **kwargs):

out = {}<EOL>diffmap = diffmap or {}<EOL>for c in comps:<EOL><INDENT>cat = c.category<EOL>if cat in diffmap:<EOL><INDENT>symbol = self.diffname(self.ivarmap[cat])<EOL>out[symbol] = diffmap[cat]<EOL><DEDENT>else:<EOL><INDENT>symbol = self.ivarmap[cat]<EOL>out[symbol] = getattr(c, funcname)(**kwargs)<EOL><DEDENT><DEDENT>return out<EOL>

Map component function ``funcname`` result into model variables
dictionary for use in eval of the model. If ``diffmap`` is provided then
that symbol is translated into 'd'+diffmap.key and is replaced by
diffmap.value. ``**kwargs` are passed to the ``comp.funcname(**kwargs)``.


def evaluate(self, comps, funcname='<STR_LIT>', diffmap=None, **kwargs):

evar = self.map_vars(comps, funcname, diffmap=diffmap)<EOL>if diffmap is None:<EOL><INDENT>return eval(self.get_base_model(), evar)<EOL><DEDENT>else:<EOL><INDENT>compname = list(diffmap.keys())[<NUM_LIT:0>]<EOL>return eval(self.get_difference_model(compname), evar)<EOL><DEDENT>

Calculate the output of a model. It is recommended that at some point
before using `evaluate`, that you make sure the inputs are valid using
:class:`~peri.models.Model.check_inputs`

Parameters
-----------
comps : list of :class:`~peri.comp.comp.Component`
    Components which will be used to evaluate the model

funcname : string (default: 'get')
    Name of the function which to evaluate for the components which
    represent their output. That is, when each component is used in the
    evaluation, it is really their ``attr(comp, funcname)`` which is used.

diffmap : dictionary
    Extra mapping of derivatives or other symbols to extra variables.
    For example, the difference in a component has been evaluated as
    diff_obj so we set ``{'I': diff_obj}``

``**kwargs``:
    Arguments passed to ``funcname`` of component objects


varmap = {<EOL>'<STR_LIT:B>': '<STR_LIT>', '<STR_LIT:I>': '<STR_LIT>', '<STR_LIT:H>': '<STR_LIT>', '<STR_LIT:P>': '<STR_LIT>', '<STR_LIT:C>': '<STR_LIT>'<EOL>}<EOL>modelstr = {<EOL>'<STR_LIT>' : '<STR_LIT>',<EOL>'<STR_LIT>' : '<STR_LIT>',<EOL>'<STR_LIT>' : '<STR_LIT>',<EOL>'<STR_LIT>' : '<STR_LIT>',<EOL>'<STR_LIT>' : '<STR_LIT>',<EOL>}<EOL>registry = {<EOL>'<STR_LIT>': allfields,<EOL>'<STR_LIT>': allfields,<EOL>'<STR_LIT>': allpsfs,<EOL>'<STR_LIT>': allobjs,<EOL>'<STR_LIT>': {'<STR_LIT>': GlobalScalar}<EOL>}<EOL>Model.__init__(self, modelstr=modelstr, varmap=varmap, registry=registry)<EOL>

Confocal microscope image with simplifications made for performance.
In particular, the sensing and illumination point spread functions
are combined into one:

.. math::
    \\mathcal{M} = H(I(1-P) + CP) + B


def get_rand_Japprox(s, params, num_inds=<NUM_LIT:1000>, include_cost=False, **kwargs):

start_time = time.time()<EOL>tot_pix = s.residuals.size<EOL>if num_inds < tot_pix:<EOL><INDENT>inds = np.random.choice(tot_pix, size=num_inds, replace=False)<EOL>slicer = None<EOL>return_inds = np.sort(inds)<EOL><DEDENT>else:<EOL><INDENT>inds = None<EOL>return_inds = slice(<NUM_LIT:0>, None)<EOL>slicer = [slice(<NUM_LIT:0>, None)]*len(s.residuals.shape)<EOL><DEDENT>if include_cost:<EOL><INDENT>Jact, ge = s.gradmodel_e(params=params, inds=inds, slicer=slicer,flat=False,<EOL>**kwargs)<EOL>Jact *= -<NUM_LIT:1><EOL>J = [Jact, ge]<EOL><DEDENT>else:<EOL><INDENT>J = -s.gradmodel(params=params, inds=inds, slicer=slicer, flat=False,<EOL>**kwargs)<EOL><DEDENT>CLOG.debug('<STR_LIT>' % (time.time()-start_time))<EOL>return J, return_inds<EOL>

Calculates a random approximation to J by returning J only at a
set of random pixel/voxel locations.

Parameters
----------
    s : :class:`peri.states.State`
        The state to calculate J for.
    params : List
        The list of parameter names to calculate the gradient of.
    num_inds : Int, optional.
        The number of pix/voxels at which to calculate the random
        approximation to J. Default is 1000.
    include_cost : Bool, optional
        Set to True to append a finite-difference measure of the full
        cost gradient onto the returned J.

Other Parameters
----------------
    All kwargs parameters get passed to s.gradmodel only.

Returns
-------
    J : numpy.ndarray
        [d, num_inds] array of J, at the given indices.

    return_inds : numpy.ndarray or slice
        [num_inds] element array or slice(0, None) of the model
        indices at which J was evaluated.


all_params = s.params<EOL>for p in s.param_particle(np.arange(s.obj_get_positions().shape[<NUM_LIT:0>])):<EOL><INDENT>all_params.remove(p)<EOL><DEDENT>if remove_params is not None:<EOL><INDENT>for p in set(remove_params):<EOL><INDENT>all_params.remove(p)<EOL><DEDENT><DEDENT>return all_params<EOL>

Returns a list of the global parameter names.

Parameters
----------
    s : :class:`peri.states.ImageState`
        The state to name the globals of.
    remove_params : Set or None
        A set of unique additional parameters to remove from the globals
        list.

Returns
-------
    all_params : list
        The list of the global parameter names, with each of
        remove_params removed.


def get_num_px_jtj(s, nparams, decimate=<NUM_LIT:1>, max_mem=<NUM_LIT>, min_redundant=<NUM_LIT:20>):

<EOL>px_mem = int(max_mem // <NUM_LIT:8> // nparams) <EOL>px_red = min_redundant*nparams<EOL>px_dec = s.residuals.size//decimate<EOL>if px_red > px_mem:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>num_px = np.clip(px_dec, px_red, px_mem).astype('<STR_LIT:int>')<EOL>return num_px<EOL>

Calculates the number of pixels to use for J at a given memory usage.

Tries to pick a number of pixels as (size of image / `decimate`).
However, clips this to a maximum size and minimum size to ensure that
(1) too much memory isn't used and (2) J has enough elements so that
the inverse of JTJ will be well-conditioned.

Parameters
----------
    s : :class:`peri.states.State`
        The state on which to calculate J.
    nparams : Int
        The number of parameters that will be included in J.
    decimate : Int, optional
        The amount to decimate the number of pixels in the image by,
        i.e. tries to pick num_px = size of image / decimate.
        Default is 1
    max_mem : Numeric, optional
        The maximum allowed memory, in bytes, for J to occupy at
        double-precision. Default is 1e9.
    min_redundant : Int, optional
        The number of pixels must be at least `min_redundant` *
        `nparams`. If not, an error is raised. Default is 20

Returns
-------
    num_px : Int
        The number of pixels at which to calcualte J.


def vectorize_damping(params, damping=<NUM_LIT:1.0>, increase_list=[['<STR_LIT>', <NUM_LIT>]]):

damp_vec = np.ones(len(params)) * damping<EOL>for nm, fctr in increase_list:<EOL><INDENT>for a in range(damp_vec.size):<EOL><INDENT>if nm in params[a]:<EOL><INDENT>damp_vec[a] *= fctr<EOL><DEDENT><DEDENT><DEDENT>return damp_vec<EOL>

Returns a non-constant damping vector, allowing certain parameters to be
more strongly damped than others.

Parameters
----------
    params : List
        The list of parameter names, in order.
    damping : Float
        The default value of the damping.
    increase_list: List
        A nested 2-element list of the params to increase and their
        scale factors. All parameters containing the string
        increase_list[i][0] are increased by a factor increase_list[i][1].
Returns
-------
    damp_vec : np.ndarray
        The damping vector to use.


if not m.flags.c_contiguous:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>mmt = np.zeros([m.shape[<NUM_LIT:0>], m.shape[<NUM_LIT:0>]])  <EOL>for a in range(<NUM_LIT:0>, m.shape[<NUM_LIT:0>], step):<EOL><INDENT>mx = min(a+step, m.shape[<NUM_LIT:1>])<EOL>mmt[:, a:mx] = np.dot(m, m[a:mx].T)<EOL><DEDENT>return mmt<EOL>

np.dot(m, m.T) with low mem usage, by doing it in small steps


bools = tile.contains(positions)<EOL>return np.arange(bools.size)[bools]<EOL>

Finds the particles in a tile, as numpy.ndarray of ints.

Parameters
----------
    positions : `numpy.ndarray`
        [N,3] array of the particle positions to check in the tile
    tile : :class:`peri.util.Tile` instance
        Tile of the region inside which to check for particles.

Returns
-------
    numpy.ndarray, int
        The indices of the particles in the tile.


def separate_particles_into_groups(s, region_size=<NUM_LIT>, bounds=None,<EOL>doshift=False):

imtile = s.oshape.translate(-s.pad)<EOL>bounding_tile = (imtile if bounds is None else Tile(bounds[<NUM_LIT:0>], bounds[<NUM_LIT:1>]))<EOL>rs = (np.ones(bounding_tile.dim, dtype='<STR_LIT:int>')*region_size if<EOL>np.size(region_size) == <NUM_LIT:1> else np.array(region_size))<EOL>n_translate = np.ceil(bounding_tile.shape.astype('<STR_LIT:float>')/rs).astype('<STR_LIT:int>')<EOL>particle_groups = []<EOL>tile = Tile(left=bounding_tile.l, right=bounding_tile.l + rs)<EOL>if doshift == '<STR_LIT>':<EOL><INDENT>doshift = np.random.choice([True, False])<EOL><DEDENT>if doshift:<EOL><INDENT>shift = rs // <NUM_LIT:2><EOL>n_translate += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>shift = <NUM_LIT:0><EOL><DEDENT>deltas = np.meshgrid(*[np.arange(i) for i in n_translate])<EOL>positions = s.obj_get_positions()<EOL>if bounds is None:<EOL><INDENT>positions = np.clip(positions, imtile.l+<NUM_LIT>, imtile.r-<NUM_LIT>)<EOL><DEDENT>groups = list(map(lambda *args: find_particles_in_tile(positions,<EOL>tile.translate( np.array(args) * rs - shift)), *[d.ravel()<EOL>for d in deltas]))<EOL>for i in range(len(groups)-<NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>if groups[i].size == <NUM_LIT:0>:<EOL><INDENT>groups.pop(i)<EOL><DEDENT><DEDENT>assert _check_groups(s, groups)<EOL>return groups<EOL>

Separates particles into convenient groups for optimization.

Given a state, returns a list of groups of particles. Each group of
particles are located near each other in the image. Every particle
located in the desired region is contained in exactly 1 group.

Parameters
----------
s : :class:`peri.states.ImageState`
    The peri state to find particles in.
region_size : Int or 3-element list-like of ints, optional
    The size of the box. Groups particles into boxes of shape
    (region_size[0], region_size[1], region_size[2]). If region_size
    is a scalar, the box is a cube of length region_size.
    Default is 40.
bounds : 2-element list-like of 3-element lists, optional
    The sub-region of the image over which to look for particles.
        bounds[0]: The lower-left  corner of the image region.
        bounds[1]: The upper-right corner of the image region.
    Default (None -> ([0,0,0], s.oshape.shape)) is a box of the entire
    image size, i.e. the default places every particle in the image
    somewhere in the groups.
doshift : {True, False, `'rand'`}, optional
    Whether or not to shift the tile boxes by half a region size, to
    prevent the same particles to be chosen every time. If `'rand'`,
    randomly chooses either True or False. Default is False

Returns
-------
particle_groups : List
    Each element of particle_groups is an int numpy.ndarray of the
    group of nearby particles. Only contains groups with a nonzero
    number of particles, so the elements don't necessarily correspond
    to a given image region.


ans = []<EOL>for g in groups:<EOL><INDENT>ans.extend(g)<EOL><DEDENT>if np.unique(ans).size != np.size(ans):<EOL><INDENT>return False<EOL><DEDENT>elif np.unique(ans).size != s.obj_get_positions().shape[<NUM_LIT:0>]:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return (np.arange(s.obj_get_radii().size) == np.sort(ans)).all()<EOL><DEDENT>

Ensures that all particles are included in exactly 1 group


def calc_particle_group_region_size(s, region_size=<NUM_LIT>, max_mem=<NUM_LIT>, **kwargs):

region_size = np.array(region_size).astype('<STR_LIT:int>')<EOL>def calc_mem_usage(region_size):<EOL><INDENT>rs = np.array(region_size)<EOL>particle_groups = separate_particles_into_groups(s, region_size=<EOL>rs.tolist(), **kwargs)<EOL>numpart = [np.size(g) for g in particle_groups]<EOL>biggroups = [particle_groups[i] for i in np.argsort(numpart)[-<NUM_LIT:5>:]]<EOL>def get_tile_jsize(group):<EOL><INDENT>nms = s.param_particle(group)<EOL>tile = s.get_update_io_tiles(nms, s.get_values(nms))[<NUM_LIT:2>]<EOL>return tile.shape.prod() * len(nms)<EOL><DEDENT>mems = [<NUM_LIT:8>*get_tile_jsize(g) for g in biggroups]  <EOL>return np.max(mems)<EOL><DEDENT>im_shape = s.oshape.shape<EOL>if calc_mem_usage(region_size) > max_mem:<EOL><INDENT>while ((calc_mem_usage(region_size) > max_mem) and<EOL>np.any(region_size > <NUM_LIT:2>)):<EOL><INDENT>region_size = np.clip(region_size-<NUM_LIT:1>, <NUM_LIT:2>, im_shape)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>while ((calc_mem_usage(region_size) < max_mem) and<EOL>np.any(region_size < im_shape)):<EOL><INDENT>region_size = np.clip(region_size+<NUM_LIT:1>, <NUM_LIT:2>, im_shape)<EOL><DEDENT>region_size -= <NUM_LIT:1> <EOL><DEDENT>return region_size<EOL>

Finds the biggest region size for LM particle optimization with a
given memory constraint.

Input Parameters
----------------
    s : :class:`peri.states.ImageState`
        The state with the particles
    region_size : Int or 3-element list-like of ints, optional.
        The initial guess for the region size. Default is 40
    max_mem : Numeric, optional
        The maximum memory for the optimizer to take. Default is 1e9

Other Parameters
----------------
    bounds: 2-element list-like of 3-element lists.
        The sub-region of the image over which to look for particles.
            bounds[0]: The lower-left  corner of the image region.
            bounds[1]: The upper-right corner of the image region.
        Default (None -> ([0,0,0], s.oshape.shape)) is a box of the entire
        image size, i.e. the default places every particle in the image
        somewhere in the groups.
Returns
-------
    region_size : numpy.ndarray of ints of the region size.


inner_tile = st.ishape.intersection([st.ishape, padded_tile])<EOL>return inner_tile.translate(-st.pad)<EOL>

Translates a tile in the padded image to the unpadded image.

Given a state and a tile that corresponds to the padded image, returns
a tile that corresponds to the the corresponding pixels of the difference
image

Parameters
----------
    st : :class:`peri.states.State`
        The state
    padded_tile : :class:`peri.util.Tile`
        The tile in the padded image.

Returns
-------
    :class:`peri.util.Tile`
        The tile corresponding to padded_tile in the unpadded image.


if np.all(np.isnan(err_vals)):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>return np.nanargmin(err_vals)<EOL>

Returns the index of the lowest of the passed values. Catches nans etc.


def do_levmarq(s, param_names, damping=<NUM_LIT:0.1>, decrease_damp_factor=<NUM_LIT>,<EOL>run_length=<NUM_LIT:6>, eig_update=True, collect_stats=False, rz_order=<NUM_LIT:0>,<EOL>run_type=<NUM_LIT:2>, **kwargs):

if rz_order > <NUM_LIT:0>:<EOL><INDENT>aug = AugmentedState(s, param_names, rz_order=rz_order)<EOL>lm = LMAugmentedState(aug, damping=damping, run_length=run_length,<EOL>decrease_damp_factor=decrease_damp_factor, eig_update=<EOL>eig_update, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>lm = LMGlobals(s, param_names, damping=damping, run_length=run_length,<EOL>decrease_damp_factor=decrease_damp_factor, eig_update=<EOL>eig_update, **kwargs)<EOL><DEDENT>if run_type == <NUM_LIT:2>:<EOL><INDENT>lm.do_run_2()<EOL><DEDENT>elif run_type == <NUM_LIT:1>:<EOL><INDENT>lm.do_run_1()<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if collect_stats:<EOL><INDENT>return lm.get_termination_stats()<EOL><DEDENT>

Runs Levenberg-Marquardt optimization on a state.

Convenience wrapper for LMGlobals. Same keyword args, but the defaults
have been set to useful values for optimizing globals.
See LMGlobals and LMEngine for documentation.

See Also
--------
    do_levmarq_particles : Levenberg-Marquardt optimization of a
        specified set of particles.

    do_levmarq_all_particle_groups : Levenberg-Marquardt optimization
        of all the particles in the state.

    LMGlobals : Optimizer object; the workhorse of do_levmarq.

    LMEngine : Engine superclass for all the optimizers.


def do_levmarq_particles(s, particles, damping=<NUM_LIT:1.0>, decrease_damp_factor=<NUM_LIT>,<EOL>run_length=<NUM_LIT:4>, collect_stats=False, max_iter=<NUM_LIT:2>, **kwargs):

lp = LMParticles(s, particles, damping=damping, run_length=run_length,<EOL>decrease_damp_factor=decrease_damp_factor, max_iter=max_iter,<EOL>**kwargs)<EOL>lp.do_run_2()<EOL>if collect_stats:<EOL><INDENT>return lp.get_termination_stats()<EOL><DEDENT>

Levenberg-Marquardt optimization on a set of particles.

Convenience wrapper for LMParticles. Same keyword args, but the
defaults have been set to useful values for optimizing particles.
See LMParticles and LMEngine for documentation.

See Also
--------
    do_levmarq_all_particle_groups : Levenberg-Marquardt optimization
        of all the particles in the state.

    do_levmarq : Levenberg-Marquardt optimization of the entire state;
        useful for optimizing global parameters.

    LMParticles : Optimizer object; the workhorse of do_levmarq_particles.

    LMEngine : Engine superclass for all the optimizers.


def do_levmarq_all_particle_groups(s, region_size=<NUM_LIT>, max_iter=<NUM_LIT:2>, damping=<NUM_LIT:1.0>,<EOL>decrease_damp_factor=<NUM_LIT>, run_length=<NUM_LIT:4>, collect_stats=False, **kwargs):

lp = LMParticleGroupCollection(s, region_size=region_size, damping=damping,<EOL>run_length=run_length, decrease_damp_factor=decrease_damp_factor,<EOL>get_cos=collect_stats, max_iter=max_iter, **kwargs)<EOL>lp.do_run_2()<EOL>if collect_stats:<EOL><INDENT>return lp.stats<EOL><DEDENT>

Levenberg-Marquardt optimization for every particle in the state.

Convenience wrapper for LMParticleGroupCollection. Same keyword args,
but I've set the defaults to what I've found to be useful values for
optimizing particles. See LMParticleGroupCollection for documentation.

See Also
--------
    do_levmarq_particles : Levenberg-Marquardt optimization of a
        specified set of particles.

    do_levmarq : Levenberg-Marquardt optimization of the entire state;
        useful for optimizing global parameters.

    LMParticleGroupCollection : The workhorse of do_levmarq.

    LMEngine : Engine superclass for all the optimizers.


def do_levmarq_n_directions(s, directions, max_iter=<NUM_LIT:2>, run_length=<NUM_LIT:2>,<EOL>damping=<NUM_LIT>, collect_stats=False, marquardt_damping=True, **kwargs):

<EOL>normals = np.array([d/np.sqrt(np.dot(d,d)) for d in directions])<EOL>if np.isnan(normals).any():<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>obj = OptState(s, normals)<EOL>lo = LMOptObj(obj, max_iter=max_iter, run_length=run_length, damping=<EOL>damping, marquardt_damping=marquardt_damping, **kwargs)<EOL>lo.do_run_1()<EOL>if collect_stats:<EOL><INDENT>return lo.get_termination_stats()<EOL><DEDENT>

Optimization of a state along a specific set of directions in parameter
space.

Parameters
----------
    s : :class:`peri.states.State`
        The state to optimize
    directions : np.ndarray
        [n,d] element numpy.ndarray of the n directions in the d-
        dimensional space to optimize along. `directions` is trans-
        formed to a unit vector internally
Other Parameters
----------------
    Any parameters passed to LMEngine.


def burn(s, n_loop=<NUM_LIT:6>, collect_stats=False, desc='<STR_LIT>', rz_order=<NUM_LIT:0>, fractol=<NUM_LIT>,<EOL>errtol=<NUM_LIT>, mode='<STR_LIT>', max_mem=<NUM_LIT>, include_rad=True,<EOL>do_line_min='<STR_LIT:default>', partial_log=False, dowarn=True):

<EOL>mode = mode.lower()<EOL>if mode not in {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>'}:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if desc is '<STR_LIT>':<EOL><INDENT>desc = mode + '<STR_LIT>' if mode != '<STR_LIT>' else '<STR_LIT>'<EOL><DEDENT>eig_update = (mode != '<STR_LIT>')<EOL>glbl_run_length = <NUM_LIT:3> if mode == '<STR_LIT>' else <NUM_LIT:6><EOL>glbl_mx_itr = <NUM_LIT:2> if mode == '<STR_LIT>' else <NUM_LIT:1><EOL>use_accel = (mode == '<STR_LIT>')<EOL>rz_order = int(rz_order)<EOL>if do_line_min == '<STR_LIT:default>':<EOL><INDENT>do_line_min = False<EOL><DEDENT>if mode == '<STR_LIT>':<EOL><INDENT>glbl_nms = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>glbl_nms = [nm for nm in glbl_nms if nm in s.params]<EOL><DEDENT>else:<EOL><INDENT>if mode == '<STR_LIT>':<EOL><INDENT>remove_params = None<EOL><DEDENT>else:<EOL><INDENT>remove_params = s.get('<STR_LIT>').params<EOL>if ('<STR_LIT>' not in remove_params) and ('<STR_LIT>' in s.params):<EOL><INDENT>remove_params.append('<STR_LIT>')<EOL><DEDENT><DEDENT>glbl_nms = name_globals(s, remove_params=remove_params)<EOL><DEDENT>all_lp_stats = []<EOL>all_lm_stats = []<EOL>all_line_stats = []<EOL>all_loop_values = []<EOL>_delta_vals = []  <EOL>CLOG.info('<STR_LIT>' % (<NUM_LIT:0>, s.error))<EOL>for a in range(n_loop):<EOL><INDENT>start_err = s.error<EOL>start_params = np.copy(s.state[s.params])<EOL>BAD_DAMP = <NUM_LIT><EOL>BAD_LIST = [['<STR_LIT>', BAD_DAMP], ['<STR_LIT>', BAD_DAMP], ['<STR_LIT>',<EOL>BAD_DAMP], ['<STR_LIT>', BAD_DAMP]]<EOL>glbl_dmp = vectorize_damping(glbl_nms + ['<STR_LIT>']*rz_order, damping=<NUM_LIT:1.0>,<EOL>increase_list=[['<STR_LIT>', <NUM_LIT>]] + BAD_LIST)<EOL>if a != <NUM_LIT:0> or mode != '<STR_LIT>':<EOL><INDENT>if partial_log:<EOL><INDENT>log.set_level('<STR_LIT>')<EOL><DEDENT>gstats = do_levmarq(s, glbl_nms, max_iter=glbl_mx_itr, run_length=<EOL>glbl_run_length, eig_update=eig_update, num_eig_dirs=<NUM_LIT:10>,<EOL>eig_update_frequency=<NUM_LIT:3>, rz_order=rz_order, damping=<EOL>glbl_dmp, decrease_damp_factor=<NUM_LIT>, use_accel=use_accel,<EOL>collect_stats=collect_stats, fractol=<NUM_LIT:0.1>*fractol,<EOL>max_mem=max_mem)<EOL>if partial_log:<EOL><INDENT>log.set_level('<STR_LIT:info>')<EOL><DEDENT>all_lm_stats.append(gstats)<EOL><DEDENT>if desc is not None:<EOL><INDENT>states.save(s, desc=desc)<EOL><DEDENT>CLOG.info('<STR_LIT>'.format(a, s.error))<EOL>all_loop_values.append(s.values)<EOL>prtl_dmp = <NUM_LIT:1.0> if a==<NUM_LIT:0> else <NUM_LIT><EOL>pstats = do_levmarq_all_particle_groups(s, region_size=<NUM_LIT>, max_iter=<NUM_LIT:1>,<EOL>do_calc_size=True, run_length=<NUM_LIT:4>, eig_update=False,<EOL>damping=prtl_dmp, fractol=<NUM_LIT:0.1>*fractol, collect_stats=<EOL>collect_stats, max_mem=max_mem, include_rad=include_rad)<EOL>all_lp_stats.append(pstats)<EOL>if desc is not None:<EOL><INDENT>states.save(s, desc=desc)<EOL><DEDENT>CLOG.info('<STR_LIT>'.format(a, s.error))<EOL>gc.collect()<EOL>all_loop_values.append(s.values)<EOL>end_params = np.copy(s.state[s.params])<EOL>_delta_vals.append(start_params - end_params)<EOL>if do_line_min:<EOL><INDENT>all_line_stats.append(do_levmarq_n_directions(s, _delta_vals[-<NUM_LIT:3>:],<EOL>collect_stats=collect_stats))<EOL>if desc is not None:<EOL><INDENT>states.save(s, desc=desc)<EOL><DEDENT>CLOG.info('<STR_LIT>'.format(a, s.error))<EOL>all_loop_values.append(s.values)<EOL><DEDENT>new_err = s.error<EOL>derr = start_err - new_err<EOL>dobreak = (derr/new_err < fractol) or (derr < errtol)<EOL>if dobreak:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if dowarn and (not dobreak):<EOL><INDENT>CLOG.warn('<STR_LIT>')<EOL><DEDENT>d = {'<STR_LIT>':dobreak, '<STR_LIT>':all_loop_values}<EOL>if collect_stats:<EOL><INDENT>d.update({'<STR_LIT>':all_lm_stats, '<STR_LIT>':all_lp_stats,<EOL>'<STR_LIT>':all_line_stats})<EOL><DEDENT>return d<EOL>

Optimizes all the parameters of a state.

Burns a state through calling LMParticleGroupCollection and LMGlobals/
LMAugmentedState.

Parameters
----------
    s : :class:`peri.states.ImageState`
        The state to optimize
    n_loop : Int, optional
        The number of times to loop over in the optimizer. Default is 6.
    collect_stats : Bool, optional
        Whether or not to collect information on the optimizer's
        performance. Default is False.
    desc : string, optional
        Description to append to the states.save() call every loop.
        Set to None to avoid saving. Default is '', which selects
        one of 'burning', 'polishing', 'doing_positions'
    rz_order: Int, optional
        Set to an int > 0 to optimize with an augmented state (R(z) as
        a global parameter) vs. with the normal global parameters;
        rz_order is the order of the polynomial approximate for R(z).
        Default is 0 (no augmented state).
    fractol : Float, optional
        Fractional change in error at which to terminate. Default 1e-4
    errtol : Float, optional
        Absolute change in error at which to terminate. Default 1e-2
    mode : {'burn', 'do-particles', or 'polish'}, optional
        What mode to optimize with.
        * 'burn'          : Your state is far from the minimum.
        * 'do-particles'  : Positions far from minimum, globals well-fit.
        * 'polish'        : The state is close to the minimum.
        'burn' is the default. Only `polish` will get to the global
        minimum.
    max_mem : Numeric, optional
        The maximum amount of memory allowed for the optimizers' J's,
        for both particles & globals. Default is 1e9, i.e. 1GB per
        optimizer.
    do_line_min : Bool or 'default', optional
        Set to True to do an additional, third optimization per loop
        which optimizes along the subspace spanned by the last 3 steps
        of the burn()'s trajectory. In principle this should signifi-
        cantly speed up the convergence; in practice it sometimes does,
        sometimes doesn't. Default is 'default', which picks by mode:
        * 'burn'          : False
        * 'do-particles'  : False
        * 'polish'        : True
    dowarn : Bool, optional
        Whether to log a warning if termination results from finishing
        loops rather than from convergence. Default is True.

Returns
-------
    dictionary
        Dictionary of convergence information. Contains whether the
        optimization has converged (key ``'converged'``), the values of the
        state after each loop (key ``'all_loop_values'``).
        The values of the state's parameters after each part of the
        loop: globals, particles, linemin. If ``collect_stats`` is set,
        then also contains lists of termination dicts from globals,
        particles, and line minimization (keys ``'global_stats'``,
        ``'particle_stats'``, and ``'line_stats``').

Notes
-----
Proceeds by alternating between one Levenberg-Marquardt step
optimizing the globals, one optimizing the particles, and repeating
until termination.

In addition, if `do_line_min` is True, at the end of each loop
step an additional optimization is tried along the subspaced spanned
by the steps taken during the last 3 loops. Ideally, this changes the
convergence from linear to quadratic, but it doesn't always do much.

Each of the 3 options proceed by optimizing as follows:
* burn            : lm.do_run_2(), lp.do_run_2(). No psf, 2 loops on lm.
* do-particles    : lp.do_run_2(), scales for ilm, bkg's
* polish          : lm.do_run_2(), lp.do_run_2(). Everything, 1 loop each.
where lm is a globals LMGlobals instance, and lp a
LMParticleGroupCollection instance.


def finish(s, desc='<STR_LIT>', n_loop=<NUM_LIT:4>, max_mem=<NUM_LIT>, separate_psf=True,<EOL>fractol=<NUM_LIT>, errtol=<NUM_LIT>, dowarn=True):

values = [np.copy(s.state[s.params])]<EOL>remove_params = s.get('<STR_LIT>').params if separate_psf else None<EOL>global_params = name_globals(s, remove_params=remove_params)<EOL>gs = np.floor(max_mem / s.residuals.nbytes).astype('<STR_LIT:int>')<EOL>groups = [global_params[a:a+gs] for a in range(<NUM_LIT:0>, len(global_params), gs)]<EOL>CLOG.info('<STR_LIT>'.format(s.error))<EOL>for a in range(n_loop):<EOL><INDENT>start_err = s.error<EOL>for g in groups:<EOL><INDENT>do_levmarq(s, g, damping=<NUM_LIT:0.1>, decrease_damp_factor=<NUM_LIT>,<EOL>max_iter=<NUM_LIT:1>, max_mem=max_mem, eig_update=False)<EOL><DEDENT>if separate_psf:<EOL><INDENT>do_levmarq(s, remove_params, max_mem=max_mem, max_iter=<NUM_LIT:4>,<EOL>eig_update=False)<EOL><DEDENT>CLOG.info('<STR_LIT>'.format(a, s.error))<EOL>if desc is not None:<EOL><INDENT>states.save(s, desc=desc)<EOL><DEDENT>do_levmarq_all_particle_groups(s, max_iter=<NUM_LIT:1>, max_mem=max_mem)<EOL>CLOG.info('<STR_LIT>'.format(a, s.error))<EOL>if desc is not None:<EOL><INDENT>states.save(s, desc=desc)<EOL><DEDENT>values.append(np.copy(s.state[s.params]))<EOL>new_err = s.error<EOL>derr = start_err - new_err<EOL>dobreak = (derr/new_err < fractol) or (derr < errtol)<EOL>if dobreak:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if dowarn and (not dobreak):<EOL><INDENT>CLOG.warn('<STR_LIT>')<EOL><DEDENT>return {'<STR_LIT>':dobreak, '<STR_LIT>':np.array(values)}<EOL>

Crawls slowly to the minimum-cost state.

Blocks the global parameters into small enough sections such that each
can be optimized separately while including all the pixels (i.e. no
decimation). Optimizes the globals, then the psf separately if desired,
then particles, then a line minimization along the step direction to
speed up convergence.

Parameters
----------
    s : :class:`peri.states.ImageState`
        The state to optimize
    desc : string, optional
        Description to append to the states.save() call every loop.
        Set to `None` to avoid saving. Default is `'finish'`.
    n_loop : Int, optional
        The number of times to loop over in the optimizer. Default is 4.
    max_mem : Numeric, optional
        The maximum amount of memory allowed for the optimizers' J's,
        for both particles & globals. Default is 1e9.
    separate_psf : Bool, optional
        If True, does the psf optimization separately from the rest of
        the globals, since the psf has a more tortuous fit landscape.
        Default is True.
    fractol : Float, optional
        Fractional change in error at which to terminate. Default 1e-4
    errtol : Float, optional
        Absolute change in error at which to terminate. Default 1e-2
    dowarn : Bool, optional
        Whether to log a warning if termination results from finishing
        loops rather than from convergence. Default is True.

Returns
-------
    dictionary
        Information about the optimization. Has two keys: ``'converged'``,
        a Bool which of whether optimization stopped due to convergence
        (True) or due to max number of iterations (False), and
        ``'loop_values'``, a [n_loop+1, N] ``numpy.ndarray`` of the
        state's values, at the start of optimization and at the end of
        each loop, before the line minimization.


def fit_comp(new_comp, old_comp, **kwargs):

<EOL>new_cat = new_comp.category<EOL>new_comp.category = '<STR_LIT>'<EOL>fake_s = states.ImageState(Image(old_comp.get().copy()), [new_comp], pad=<NUM_LIT:0>,<EOL>mdl=mdl.SmoothFieldModel())<EOL>do_levmarq(fake_s, new_comp.params, **kwargs)<EOL>new_comp.category = new_cat<EOL>

Fits a new component to an old component

Calls do_levmarq to match the .get() fields of the two objects. The
parameters of new_comp are modified in place.

Parameters
----------
new_comp : :class:`peri.comps.comp`
    The new object, whose parameters to update to fit the field of
    `old_comp`. Must have a .get() attribute which returns an ndarray
old_comp : peri.comp
    The old ilm to match to.

Other Parameters
----------------
    Any keyword arguments to be passed to the optimizer LMGlobals
    through do_levmarq.

See Also
--------
do_levmarq : Levenberg-Marquardt minimization using a random subset
    of the image pixels.


self._num_iter = <NUM_LIT:0><EOL>self._inner_run_counter = <NUM_LIT:0><EOL>self._J_update_counter = self.update_J_frequency<EOL>self._fresh_JTJ = False<EOL>self._has_run = False<EOL>if new_damping is not None:<EOL><INDENT>self.damping = np.array(new_damping).astype('<STR_LIT:float>')<EOL><DEDENT>self._set_err_paramvals()<EOL>

Keeps all user supplied options the same, but resets counters etc.


raise NotImplementedError('<STR_LIT>')<EOL>

Must update:
    self.error, self._last_error, self.param_vals, self._last_vals


Takes an array param_vals, updates function, returns the new error


while not self.check_terminate():<EOL><INDENT>self._has_run = True<EOL>self._run1()<EOL>self._num_iter += <NUM_LIT:1>; self._inner_run_counter += <NUM_LIT:1><EOL><DEDENT>

LM run, evaluating 1 step at a time.

Broyden or eigendirection updates replace full-J updates until
a full-J update occurs. Does not run with the calculated J (no
internal run).


if self.check_update_J():<EOL><INDENT>self.update_J()<EOL><DEDENT>else:<EOL><INDENT>if self.check_Broyden_J():<EOL><INDENT>self.update_Broyden_J()<EOL><DEDENT>if self.check_update_eig_J():<EOL><INDENT>self.update_eig_J()<EOL><DEDENT><DEDENT>delta_vals = self.find_LM_updates(self.calc_grad())<EOL>er1 = self.update_function(self.param_vals + delta_vals)<EOL>good_step = (find_best_step([self.error, er1]) == <NUM_LIT:1>)<EOL>if not good_step:<EOL><INDENT>er0 = self.update_function(self.param_vals)<EOL>if np.abs(er0 -self.error)/er0 > <NUM_LIT>:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>CLOG.debug('<STR_LIT>')<EOL>CLOG.debug('<STR_LIT>' % (self.error, er1))<EOL>grad = self.calc_grad()<EOL>for _try in range(self._max_inner_loop):<EOL><INDENT>self.increase_damping()<EOL>delta_vals = self.find_LM_updates(grad)<EOL>er1 = self.update_function(self.param_vals + delta_vals)<EOL>good_step = (find_best_step([self.error, er1]) == <NUM_LIT:1>)<EOL>if good_step:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>er0 = self.update_function(self.param_vals)<EOL>CLOG.warn('<STR_LIT>')<EOL>if np.abs(er0 -self.error)/er0 > <NUM_LIT>:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>if good_step:<EOL><INDENT>self._last_error = self.error<EOL>self.error = er1<EOL>CLOG.debug('<STR_LIT>' % (self._last_error, self.error))<EOL>self.update_param_vals(delta_vals, incremental=True)<EOL>self.decrease_damping()<EOL><DEDENT>


while not self.check_terminate():<EOL><INDENT>self._has_run = True<EOL>self._run2()<EOL>self._num_iter += <NUM_LIT:1><EOL><DEDENT>

LM run evaluating 2 steps (damped and not) and choosing the best.

After finding the best of 2 steps, runs with that damping + Broyden
or eigendirection updates, until deciding to do a full-J update.
Only changes damping after full-J updates.


if self.check_update_J():<EOL><INDENT>self.update_J()<EOL><DEDENT>else:<EOL><INDENT>if self.check_Broyden_J():<EOL><INDENT>self.update_Broyden_J()<EOL><DEDENT>if self.check_update_eig_J():<EOL><INDENT>self.update_eig_J()<EOL><DEDENT><DEDENT>_last_residuals = self.calc_residuals().copy()<EOL>_last_error = <NUM_LIT:1>*self.error<EOL>_last_vals = self.param_vals.copy()<EOL>delta_params_1 = self.find_LM_updates(self.calc_grad(),<EOL>do_correct_damping=False)<EOL>self.decrease_damping()<EOL>delta_params_2 = self.find_LM_updates(self.calc_grad(),<EOL>do_correct_damping=False)<EOL>self.decrease_damping(undo_decrease=True)<EOL>er1 = self.update_function(self.param_vals + delta_params_1)<EOL>er2 = self.update_function(self.param_vals + delta_params_2)<EOL>triplet = (self.error, er1, er2)<EOL>best_step = find_best_step(triplet)<EOL>if best_step == <NUM_LIT:0>:<EOL><INDENT>_ = self.update_function(self.param_vals.copy())<EOL>grad = self.calc_grad()<EOL>CLOG.debug('<STR_LIT>')<EOL>CLOG.debug('<STR_LIT>' % triplet)<EOL>for _try in range(self._max_inner_loop):<EOL><INDENT>self.increase_damping()<EOL>delta_vals = self.find_LM_updates(grad)<EOL>er_new = self.update_function(self.param_vals + delta_vals)<EOL>good_step = er_new < self.error<EOL>if good_step:<EOL><INDENT>self.update_param_vals(delta_vals, incremental=True)<EOL>self.error = er_new<EOL>CLOG.debug('<STR_LIT>')<EOL>CLOG.debug('<STR_LIT>' % (triplet[<NUM_LIT:0>], self.error))<EOL>break<EOL><DEDENT><DEDENT>else: <EOL><INDENT>CLOG.warn('<STR_LIT>')<EOL>self.error = self.update_function(self.param_vals.copy())<EOL><DEDENT><DEDENT>elif best_step == <NUM_LIT:1>:<EOL><INDENT>good_step = True<EOL>CLOG.debug('<STR_LIT>')<EOL>CLOG.debug('<STR_LIT>' % triplet)<EOL>er1_1 = self.update_function(self.param_vals + delta_params_1)<EOL>if np.abs(er1_1 - er1) > <NUM_LIT>:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>self.update_param_vals(delta_params_1, incremental=True)<EOL>self.error = er1<EOL><DEDENT>elif best_step == <NUM_LIT:2>:<EOL><INDENT>good_step = True<EOL>self.error = er2<EOL>CLOG.debug('<STR_LIT>')<EOL>CLOG.debug('<STR_LIT>' % triplet)<EOL>self.update_param_vals(delta_params_2, incremental=True)<EOL>self.decrease_damping()<EOL><DEDENT>if good_step:<EOL><INDENT>self._last_residuals = _last_residuals<EOL>self._last_error = _last_error<EOL>self._last_vals = _last_vals<EOL>self.error<EOL>self.do_internal_run(initial_count=<NUM_LIT:1>)<EOL><DEDENT>


def do_internal_run(self, initial_count=<NUM_LIT:0>, subblock=None, update_derr=True):

self._inner_run_counter = initial_count; good_step = True<EOL>n_good_steps = <NUM_LIT:0><EOL>CLOG.debug('<STR_LIT>')<EOL>_last_residuals = self.calc_residuals().copy()<EOL>while ((self._inner_run_counter < self.run_length) & good_step &<EOL>(not self.check_terminate())):<EOL><INDENT>if self.check_Broyden_J() and self._inner_run_counter != <NUM_LIT:0>:<EOL><INDENT>self.update_Broyden_J()<EOL><DEDENT>if self.check_update_eig_J() and self._inner_run_counter != <NUM_LIT:0>:<EOL><INDENT>self.update_eig_J()<EOL><DEDENT>er0 = <NUM_LIT:1>*self.error<EOL>delta_vals = self.find_LM_updates(self.calc_grad(),<EOL>do_correct_damping=False, subblock=subblock)<EOL>er1 = self.update_function(self.param_vals + delta_vals)<EOL>good_step = er1 < er0<EOL>if good_step:<EOL><INDENT>n_good_steps += <NUM_LIT:1><EOL>CLOG.debug('<STR_LIT>' % (er0, er1))<EOL>self.update_param_vals(delta_vals, incremental=True)<EOL>self._last_residuals = _last_residuals.copy()<EOL>if update_derr:<EOL><INDENT>self._last_error = er0<EOL><DEDENT>self.error = er1<EOL>_last_residuals = self.calc_residuals().copy()<EOL><DEDENT>else:<EOL><INDENT>er0_0 = self.update_function(self.param_vals)<EOL>CLOG.debug('<STR_LIT>')<EOL>if np.abs(er0 - er0_0) > <NUM_LIT>:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT><DEDENT>self._inner_run_counter += <NUM_LIT:1><EOL><DEDENT>return n_good_steps<EOL>

Takes more steps without calculating J again.

Given a fixed damping, J, JTJ, iterates calculating steps, with
optional Broyden or eigendirection updates. Iterates either until
a bad step is taken or for self.run_length times.
Called internally by do_run_2() but is also useful on its own.

Parameters
----------
    initial_count : Int, optional
        The initial count of the run. Default is 0. Increasing from
        0 effectively temporarily decreases run_length.
    subblock : None or np.ndarray of bools, optional
        If not None, a boolean mask which determines which sub-
        block of parameters to run over. Default is None, i.e.
        all the parameters.
    update_derr : Bool, optional
        Set to False to not update the variable that determines
        delta_err, preventing premature termination through errtol.

Notes
-----
It might be good to do something similar to update_derr with the
parameter values, but this is trickier because of Broyden updates
and _fresh_J.


def find_LM_updates(self, grad, do_correct_damping=True, subblock=None):

if subblock is not None:<EOL><INDENT>if (subblock.sum() == <NUM_LIT:0>) or (subblock.size == <NUM_LIT:0>):<EOL><INDENT>CLOG.fatal('<STR_LIT>')<EOL>raise ValueError('<STR_LIT>')<EOL><DEDENT>j = self.J[subblock]<EOL>JTJ = np.dot(j, j.T)<EOL>damped_JTJ = self._calc_damped_jtj(JTJ, subblock=subblock)<EOL>grad = grad[subblock]  <EOL><DEDENT>else:<EOL><INDENT>damped_JTJ = self._calc_damped_jtj(self.JTJ, subblock=subblock)<EOL><DEDENT>delta = self._calc_lm_step(damped_JTJ, grad, subblock=subblock)<EOL>if self.use_accel:<EOL><INDENT>accel_correction = self.calc_accel_correction(damped_JTJ, delta)<EOL>nrm_d0 = np.sqrt(np.sum(delta**<NUM_LIT:2>))<EOL>nrm_corr = np.sqrt(np.sum(accel_correction**<NUM_LIT:2>))<EOL>CLOG.debug('<STR_LIT>' % (nrm_corr/nrm_d0))<EOL>if nrm_corr/nrm_d0 < self.max_accel_correction:<EOL><INDENT>delta += accel_correction<EOL><DEDENT>elif do_correct_damping:<EOL><INDENT>CLOG.debug('<STR_LIT>')<EOL>self.increase_damping()<EOL>damped_JTJ = self._calc_damped_jtj(self.JTJ, subblock=subblock)<EOL>delta = self._calc_lm_step(damped_JTJ, grad, subblock=subblock)<EOL><DEDENT><DEDENT>if np.any(np.isnan(delta)):<EOL><INDENT>CLOG.fatal('<STR_LIT>')<EOL>raise FloatingPointError('<STR_LIT>')<EOL><DEDENT>return delta<EOL>

Calculates LM updates, with or without the acceleration correction.

Parameters
----------
    grad : numpy.ndarray
        The gradient of the model cost.
    do_correct_damping : Bool, optional
        If `self.use_accel`, then set to True to correct damping
        if the acceleration correction is too big. Default is True
        Does nothing is `self.use_accel` is False
    subblock : slice, numpy.ndarray, or None, optional
        Set to a slice or a valide numpy.ndarray to use only a
        certain subset of the parameters. Default is None, i.e.
        use all the parameters.

Returns
-------
    delta : numpy.ndarray
        The Levenberg-Marquadt step, relative to the old
        parameters. Size is always self.param_vals.size.


def _calc_lm_step(self, damped_JTJ, grad, subblock=None):

delta0, res, rank, s = np.linalg.lstsq(damped_JTJ, -<NUM_LIT:0.5>*grad,<EOL>rcond=self.min_eigval)<EOL>if self._fresh_JTJ:<EOL><INDENT>CLOG.debug('<STR_LIT>' % (<EOL>delta0.size-rank, delta0.size))<EOL><DEDENT>if subblock is not None:<EOL><INDENT>delta = np.zeros(self.J.shape[<NUM_LIT:0>])<EOL>delta[subblock] = delta0<EOL><DEDENT>else:<EOL><INDENT>delta = delta0.copy()<EOL><DEDENT>return delta<EOL>

Calculates a Levenberg-Marquard step w/o acceleration


def update_param_vals(self, new_vals, incremental=False):

self._last_vals = self.param_vals.copy()<EOL>if incremental:<EOL><INDENT>self.param_vals += new_vals<EOL><DEDENT>else:<EOL><INDENT>self.param_vals = new_vals.copy()<EOL><DEDENT>self._fresh_JTJ = False<EOL>

Updates the current set of parameter values and previous values,
sets a flag to re-calculate J.

Parameters
----------
    new_vals : numpy.ndarray
        The new values to update to
    incremental : Bool, optional
        Set to True to make it an incremental update relative
        to the old parameters. Default is False


grad = self.calc_grad()<EOL>if list(delta_params) in [list('<STR_LIT>'), list('<STR_LIT>')]:<EOL><INDENT>jtj = (self.JTJ if delta_params == '<STR_LIT>' else<EOL>self._calc_damped_jtj(self.JTJ))<EOL>delta_params = self._calc_lm_step(jtj, self.calc_grad())<EOL><DEDENT>expected_error = (self.error + np.dot(grad, delta_params) +<EOL>np.dot(np.dot(self.JTJ, delta_params), delta_params))<EOL>return expected_error<EOL>

Returns the error expected after an update if the model were linear.

Parameters
----------
    delta_params : {numpy.ndarray, 'calc', or 'perfect'}, optional
        The relative change in parameters. If 'calc', uses update
        calculated from the current damping, J, etc; if 'perfect',
        uses the update calculated with zero damping.

Returns
-------
    numpy.float64
        The expected error after the update with `delta_params`


def calc_model_cosine(self, decimate=None, mode='<STR_LIT>'):

if mode == '<STR_LIT>':<EOL><INDENT>slicer = slice(<NUM_LIT:0>, None, decimate)<EOL>u, sig, v = np.linalg.svd(self.J[:,slicer], full_matrices=False) <EOL>r = self.calc_residuals()[slicer]<EOL>abs_r = np.sqrt((r*r).sum())<EOL>v_r = np.dot(v,r/abs_r)<EOL>projected = np.dot(v.T, v_r)<EOL>abs_cos = np.sqrt((projected*projected).sum())<EOL><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>expected_error = self.find_expected_error(delta_params='<STR_LIT>')<EOL>model_sine_2 = expected_error / self.error  <EOL>abs_cos = np.sqrt(<NUM_LIT:1> - model_sine_2)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>return abs_cos<EOL>

Calculates the cosine of the residuals with the model.

Parameters
----------
    decimate : Int or None, optional
        Decimate the residuals by `decimate` pixels. If None, no
        decimation is used. Valid only with mode='svd'. Default
        is None
    mode : {'svd', 'err'}
        Which mode to use; see Notes section. Default is 'err'.

Returns
-------
    abs_cos : numpy.float64
        The absolute value of the model cosine.

Notes
-----
The model cosine is defined in terms of the geometric view of
curve-fitting, as a model manifold embedded in a high-dimensional
space. The model cosine is the cosine of the residuals vector
with its projection on the tangent space: :math:`cos(phi) = |P^T r|/|r|`
where :math:`P^T` is the projection operator onto the model manifold
and :math:`r` the residuals. This can be calculated two ways: By
calculating the projection operator P directly with SVD (mode=`svd`),
or by using the expected error if the model were linear to calculate
a model sine first (mode=`err`). Since the SVD of a large matrix is
slow, mode=`err` is faster.

`decimate` allows for every nth pixel only to be counted in the
SVD matrix of J for speed. While this is n x faster, it is
considerably less accurate, so the default is no decimation.


delta_vals = self._last_vals - self.param_vals<EOL>delta_err = self._last_error - self.error<EOL>frac_err = delta_err / self.error<EOL>to_return = {'<STR_LIT>':delta_vals, '<STR_LIT>':delta_err,<EOL>'<STR_LIT>':<NUM_LIT:1>*self._num_iter, '<STR_LIT>':frac_err,<EOL>'<STR_LIT:error>':self.error, '<STR_LIT>':self._exp_err}<EOL>if get_cos:<EOL><INDENT>model_cosine = self.calc_model_cosine()<EOL>to_return.update({'<STR_LIT>':model_cosine})<EOL><DEDENT>return to_return<EOL>

Returns a dict of termination statistics

Parameters
----------
    get_cos : Bool, optional
        Whether or not to calcualte the cosine of the residuals
        with the tangent plane of the model using the current J.
        The calculation may take some time. Default is True

Returns
-------
    dict
        Has keys
            delta_vals  : The last change in parameter values.
            delta_err   : The last change in the error.
            exp_err     : The expected (last) change in the error.
            frac_err    : The fractional change in the error.
            num_iter    : The number of iterations completed.
            error       : The current error.


terminate = False<EOL>term_dict = self.get_termination_stats(get_cos=self.costol is not None)<EOL>terminate |= np.all(np.abs(term_dict['<STR_LIT>']) < self.paramtol)<EOL>terminate |= (term_dict['<STR_LIT>'] < self.errtol)<EOL>terminate |= (term_dict['<STR_LIT>'] < self.exptol)<EOL>terminate |= (term_dict['<STR_LIT>'] < self.fractol)<EOL>if self.costol is not None:<EOL><INDENT>terminate |= (curcos < term_dict['<STR_LIT>'])<EOL><DEDENT>return terminate<EOL>

Returns a Bool of whether the algorithm has found a satisfactory minimum


if not self._has_run:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>terminate = self.check_completion()<EOL>terminate |= (self._num_iter >= self.max_iter)<EOL>return terminate<EOL><DEDENT>

Returns a Bool of whether to terminate.

Checks whether a satisfactory minimum has been found or whether
too many iterations have occurred.


self._J_update_counter += <NUM_LIT:1><EOL>update = self._J_update_counter >= self.update_J_frequency<EOL>return update & (not self._fresh_JTJ)<EOL>

Checks if the full J should be updated.

Right now, just updates after update_J_frequency loops


self.calc_J()<EOL>step = np.ceil(<NUM_LIT> * self.J.shape[<NUM_LIT:1>]).astype('<STR_LIT:int>')  <EOL>self.JTJ = low_mem_sq(self.J, step=step)<EOL>self._fresh_JTJ = True<EOL>self._J_update_counter = <NUM_LIT:0><EOL>if np.any(np.isnan(self.JTJ)):<EOL><INDENT>raise FloatingPointError('<STR_LIT>')<EOL><DEDENT>self._exp_err = self.error - self.find_expected_error(delta_params='<STR_LIT>')<EOL>


residuals = self.calc_residuals()<EOL>return <NUM_LIT:2>*np.dot(self.J, residuals)<EOL>

The gradient of the cost w.r.t. the parameters.


def _rank_1_J_update(self, direction, values):

vals_to_sub = np.dot(direction, self.J)<EOL>delta_vals = values - vals_to_sub<EOL>for a in range(direction.size):<EOL><INDENT>self.J[a] += direction[a] * delta_vals<EOL><DEDENT>

Does J += np.outer(direction, new_values - old_values) without
using lots of memory


CLOG.debug('<STR_LIT>')<EOL>delta_vals = self.param_vals - self._last_vals<EOL>delta_residuals = self.calc_residuals() - self._last_residuals<EOL>nrm = np.sqrt(np.dot(delta_vals, delta_vals))<EOL>direction = delta_vals / nrm<EOL>vals = delta_residuals / nrm<EOL>self._rank_1_J_update(direction, vals)<EOL>self.JTJ = np.dot(self.J, self.J.T)<EOL>


CLOG.debug('<STR_LIT>')<EOL>vls, vcs = np.linalg.eigh(self.JTJ)<EOL>res0 = self.calc_residuals()<EOL>for a in range(min([self.num_eig_dirs, vls.size])):<EOL><INDENT>stif_dir = vcs[-(a+<NUM_LIT:1>)] <EOL>dl = self.eig_dl <EOL>_ = self.update_function(self.param_vals + dl*stif_dir)<EOL>res1 = self.calc_residuals()<EOL>grad_stif = (res1-res0)/dl<EOL>self._rank_1_J_update(stif_dir, grad_stif)<EOL><DEDENT>self.JTJ = np.dot(self.J, self.J.T)<EOL>_ = self.update_function(self.param_vals)<EOL>


def calc_accel_correction(self, damped_JTJ, delta0):

<EOL>_ = self.update_function(self.param_vals)<EOL>rm0 = self.calc_residuals().copy()<EOL>_ = self.update_function(self.param_vals + delta0)<EOL>rm1 = self.calc_residuals().copy()<EOL>_ = self.update_function(self.param_vals - delta0)<EOL>rm2 = self.calc_residuals().copy()<EOL>der2 = (rm2 + rm1 - <NUM_LIT:2>*rm0)<EOL>corr, res, rank, s = np.linalg.lstsq(damped_JTJ, np.dot(self.J, der2),<EOL>rcond=self.min_eigval)<EOL>corr *= -<NUM_LIT:0.5><EOL>return corr<EOL>

Geodesic acceleration correction to the LM step.

Parameters
----------
    damped_JTJ : numpy.ndarray
        The damped JTJ used to calculate the initial step.
    delta0 : numpy.ndarray
        The initial LM step.

Returns
-------
    corr : numpy.ndarray
        The correction to the original LM step.


p0 = self.param_vals.copy()<EOL>self.update_function(p0)  <EOL>r0 = self.calc_residuals().copy()<EOL>dl = np.zeros(p0.size, dtype='<STR_LIT:float>')<EOL>blk_J = []<EOL>for i in np.nonzero(blk)[<NUM_LIT:0>]:<EOL><INDENT>dl *= <NUM_LIT:0>; dl[i] = self.eig_dl<EOL>self.update_function(p0 + dl)<EOL>r1 = self.calc_residuals().copy()<EOL>blk_J.append((r1-r0)/self.eig_dl)<EOL><DEDENT>self.J[blk] = np.array(blk_J)<EOL>self.update_function(p0)<EOL>self.JTJ = np.dot(self.J, self.J.T)<EOL>if np.any(np.isnan(self.J)) or np.any(np.isnan(self.JTJ)):<EOL><INDENT>raise FloatingPointError('<STR_LIT>')<EOL><DEDENT>

Updates J only for certain parameters, described by the boolean
mask `blk`.


<EOL>self._last_vals = self.param_vals.copy()<EOL>self.error = self.update_function(self.param_vals)<EOL>self._last_error = (<NUM_LIT:1> + <NUM_LIT:2>*self.fractol) * self.error<EOL>

Must update:
    self.error, self._last_error, self.param_vals, self._last_vals


del self.J<EOL>self.J = np.zeros([self.param_vals.size, self.data.size])<EOL>dp = np.zeros_like(self.param_vals)<EOL>f0 = self.model.copy()<EOL>for a in range(self.param_vals.size):<EOL><INDENT>dp *= <NUM_LIT:0><EOL>dp[a] = self.dl[a]<EOL>f1 = self.func(self.param_vals + dp, *self.func_args, **self.func_kwargs)<EOL>grad_func = (f1 - f0) / dp[a]<EOL>self.J[a] = -grad_func<EOL><DEDENT>


self.model = self.func(param_vals, *self.func_args, **self.func_kwargs)<EOL>d = self.calc_residuals()<EOL>return np.dot(d.flat, d.flat)<EOL>

Takes an array param_vals, updates function, returns the new error


self.opt_obj.update_function(param_vals)<EOL>return self.opt_obj.get_error()<EOL>


dp = np.zeros(self.p0.size)<EOL>for a in range(param_vals.size):<EOL><INDENT>dp += param_vals[a] * self.directions[a]<EOL><DEDENT>self.state.update(self.state.params, self.p0 + dp)<EOL>self.param_vals[:] = param_vals<EOL>return None<EOL>

Updates with param_vals[i] = distance from self.p0 along self.direction[i].


return self.state.residuals.ravel().copy()<EOL>

See also
--------
:func:`peri.state.States.residuals`


r0 = self.state.residuals.copy().ravel()<EOL>dl = np.zeros(self.param_vals.size)<EOL>p0 = self.param_vals.copy()<EOL>J = []<EOL>for a in range(self.param_vals.size):<EOL><INDENT>dl *= <NUM_LIT:0><EOL>dl[a] += self.dl<EOL>self.update_function(p0 + dl)<EOL>r1 = self.state.residuals.copy().ravel()<EOL>J.append( (r1-r0)/self.dl)<EOL><DEDENT>self.update_function(p0)<EOL>return np.array(J)<EOL>


self.update_function(self.param_vals)<EOL>params = np.array(self.param_names)[blk].tolist()<EOL>blk_J = -self.state.gradmodel(params=params, inds=self._inds, flat=False)<EOL>self.J[blk] = blk_J<EOL>self.JTJ = np.dot(self.J, self.J.T)<EOL>if np.any(np.isnan(self.J)) or np.any(np.isnan(self.JTJ)):<EOL><INDENT>raise FloatingPointError('<STR_LIT>')<EOL><DEDENT>

Updates J only for certain parameters, described by the boolean
mask blk.


def find_expected_error(self, delta_params='<STR_LIT>', adjust=True):

expected_error = super(LMGlobals, self).find_expected_error(<EOL>delta_params=delta_params)<EOL>if adjust:<EOL><INDENT>derr = (expected_error - self.error) * (self.state.residuals.size /<EOL>float(self.num_pix))<EOL>expected_error = self.error + derr<EOL><DEDENT>return expected_error<EOL>


<EOL>if mode == '<STR_LIT>':<EOL><INDENT>expected_error = self.find_expected_error(delta_params='<STR_LIT>',<EOL>adjust=False)<EOL>derr = self.error - expected_error<EOL>residuals_err = lambda r: np.dot(r,r).sum()<EOL>current_partial_error = residuals_err(self.calc_residuals())<EOL>expected_partial_error = current_partial_error - derr<EOL>model_sine_2 = expected_partial_error / current_partial_error<EOL>abs_cos = np.sqrt(<NUM_LIT:1> - model_sine_2)<EOL><DEDENT>else:<EOL><INDENT>abs_cos = super(self.__class__, self).calc_model_cosine(decimate=<EOL>decimate, mode=mode)<EOL><DEDENT>return abs_cos<EOL>


if self._fresh_JTJ:<EOL><INDENT>return self._graderr<EOL><DEDENT>else:<EOL><INDENT>residuals = self.calc_residuals()<EOL>return <NUM_LIT:2>*np.dot(self.J, residuals)<EOL><DEDENT>


def reset(self, new_region_size=None, do_calc_size=True, new_damping=None,<EOL>new_max_mem=None):

if new_region_size is not None:<EOL><INDENT>self.region_size = new_region_size<EOL><DEDENT>if new_max_mem != None:<EOL><INDENT>self.max_mem = new_max_mem<EOL><DEDENT>if do_calc_size:<EOL><INDENT>self.region_size = calc_particle_group_region_size(self.state,<EOL>region_size=self.region_size, max_mem=self.max_mem)<EOL><DEDENT>self.stats = []<EOL>self.particle_groups = separate_particles_into_groups(self.state,<EOL>self.region_size, doshift='<STR_LIT>')<EOL>if new_damping is not None:<EOL><INDENT>self._kwargs.update({'<STR_LIT>':new_damping})<EOL><DEDENT>if self.save_J:<EOL><INDENT>if len(self.particle_groups) > <NUM_LIT>:<EOL><INDENT>CLOG.warn('<STR_LIT>')<EOL><DEDENT>self._tempfiles = []<EOL>self._has_saved_J = []<EOL>for a in range(len(self.particle_groups)):<EOL><INDENT>for _ in ['<STR_LIT>','<STR_LIT>']:<EOL><INDENT>self._tempfiles.append(tempfile.TemporaryFile(dir=os.getcwd()))<EOL><DEDENT>self._has_saved_J.append(False)<EOL><DEDENT><DEDENT>

Resets the particle groups and optionally the region size and damping.

Parameters
----------
    new_region_size : : Int or 3-element list-like of ints, optional
        The region size for sub-blocking particles. Default is 40
    do_calc_size : Bool, optional
        If True, calculates the region size internally based on
        the maximum allowed memory. Default is True
    new_damping : Float or None, optional
        The new damping of the optimizer. Set to None to leave
        as the default for LMParticles. Default is None.
    new_max_mem : Numeric, optional
        The maximum allowed memory for J to occupy. Default is 1e9


for a in range(len(self.particle_groups)):<EOL><INDENT>group = self.particle_groups[a]<EOL>lp = LMParticles(self.state, group, **self._kwargs)<EOL>if mode == '<STR_LIT>':<EOL><INDENT>lp.J, lp.JTJ, lp._dif_tile = self._load_j_diftile(a)<EOL><DEDENT>if mode == '<STR_LIT:1>':<EOL><INDENT>lp.do_run_1()<EOL><DEDENT>if mode == '<STR_LIT:2>':<EOL><INDENT>lp.do_run_2()<EOL><DEDENT>if mode == '<STR_LIT>':<EOL><INDENT>lp.do_internal_run()<EOL><DEDENT>self.stats.append(lp.get_termination_stats(get_cos=self.get_cos))<EOL>if self.save_J and (mode != '<STR_LIT>'):<EOL><INDENT>self._dump_j_diftile(a, lp.J, lp._dif_tile)<EOL>self._has_saved_J[a] = True<EOL><DEDENT><DEDENT>

workhorse for the self.do_run_xx methods.


Calls LMParticles.do_run_1 for each group of particles.


Calls LMParticles.do_run_2 for each group of particles.


if not self.save_J:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>if not np.all(self._has_saved_J):<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>self._do_run(mode='<STR_LIT>')<EOL>

Calls LMParticles.do_internal_run for each group of particles.


inds = list(range(self.state.obj_get_positions().shape[<NUM_LIT:0>]))<EOL>self._rad_nms = self.state.param_particle_rad(inds)<EOL>self._pos_nms = self.state.param_particle_pos(inds)<EOL>self._initial_rad = np.copy(self.state.state[self._rad_nms])<EOL>self._initial_pos = np.copy(self.state.state[self._pos_nms]).reshape((-<NUM_LIT:1>,<NUM_LIT:3>))<EOL>self.param_vals[self.rscale_mask] = <NUM_LIT:0><EOL>

Resets the initial radii used for updating the particles. Call
if any of the particle radii or positions have been changed
external to the augmented state.


return np.exp(self._poly(pos[:,<NUM_LIT:2>]))<EOL>


shp = self.state.oshape.shape<EOL>zmax = float(shp[<NUM_LIT:0>])<EOL>zmin = <NUM_LIT:0.0><EOL>zmid = zmax * <NUM_LIT:0.5><EOL>coeffs = self.param_vals[self.rscale_mask].copy()<EOL>if coeffs.size == <NUM_LIT:0>:<EOL><INDENT>ans = <NUM_LIT:0>*z<EOL><DEDENT>else:<EOL><INDENT>ans = np.polynomial.legendre.legval((z-zmid)/zmid,<EOL>self.param_vals[self.rscale_mask])<EOL><DEDENT>return ans<EOL>


self.update_rscl_x_params(param_vals[self.rscale_mask])<EOL>self.state.update(self.param_names, param_vals[self.globals_mask])<EOL>self.param_vals[:] = param_vals.copy()<EOL>if np.any(np.isnan(self.state.residuals)):<EOL><INDENT>raise FloatingPointError('<STR_LIT>')<EOL><DEDENT>

Updates all the parameters of the state + rscale(z)


self.aug_state.reset()<EOL>super(LMAugmentedState, self).reset(**kwargs)<EOL>


def feature_guess(st, rad, invert='<STR_LIT>', minmass=None, use_tp=False,<EOL>trim_edge=False, **kwargs):

<EOL>if invert == '<STR_LIT>':<EOL><INDENT>invert = guess_invert(st)<EOL><DEDENT>if invert:<EOL><INDENT>im = <NUM_LIT:1> - st.residuals<EOL><DEDENT>else:<EOL><INDENT>im = st.residuals<EOL><DEDENT>return _feature_guess(im, rad, minmass=minmass, use_tp=use_tp,<EOL>trim_edge=trim_edge)<EOL>

Makes a guess at particle positions using heuristic centroid methods.

Parameters
----------
st : :class:`peri.states.State`
    The state to check adding particles to.
rad : Float
    The feature size for featuring.
invert : {'guess', True, False}, optional
    Whether to invert the image; set to True for there are dark
    particles on a bright background, False for bright particles.
    The default is to guess from the state's current particles.
minmass : Float or None, optional
    The minimum mass/masscut of a particle. Default is ``None`` =
    calculated internally.
use_tp : Bool, optional
    Whether or not to use trackpy. Default is ``False``, since trackpy
    cuts out particles at the edge.
trim_edge : Bool, optional
    Whether to trim particles at the edge pixels of the image. Can be
    useful for initial featuring but is bad for adding missing particles
    as they are frequently at the edge. Default is ``False``.

Returns
-------
guess : [N,3] numpy.ndarray
    The featured positions of the particles, sorted in order of decreasing
    feature mass.
npart : Int
    The number of added particles.


def _feature_guess(im, rad, minmass=None, use_tp=False, trim_edge=False):

if minmass is None:<EOL><INDENT>minmass = rad**<NUM_LIT:3> * <NUM_LIT:4>/<NUM_LIT>*np.pi * <NUM_LIT><EOL><DEDENT>if use_tp:<EOL><INDENT>diameter = np.ceil(<NUM_LIT:2>*rad)<EOL>diameter += <NUM_LIT:1>-(diameter % <NUM_LIT:2>)<EOL>df = peri.trackpy.locate(im, int(diameter), minmass=minmass)<EOL>npart = np.array(df['<STR_LIT>']).size<EOL>guess = np.zeros([npart, <NUM_LIT:3>])<EOL>guess[:, <NUM_LIT:0>] = df['<STR_LIT:z>']<EOL>guess[:, <NUM_LIT:1>] = df['<STR_LIT:y>']<EOL>guess[:, <NUM_LIT:2>] = df['<STR_LIT:x>']<EOL>mass = df['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>guess, mass = initializers.local_max_featuring(<EOL>im, radius=rad, minmass=minmass, trim_edge=trim_edge)<EOL>npart = guess.shape[<NUM_LIT:0>]<EOL><DEDENT>inds = np.argsort(mass)[::-<NUM_LIT:1>]  <EOL>return guess[inds].copy(), npart<EOL>


def check_add_particles(st, guess, rad='<STR_LIT>', do_opt=True, im_change_frac=<NUM_LIT>,<EOL>min_derr='<STR_LIT>', **kwargs):

<EOL>if min_derr == '<STR_LIT>':<EOL><INDENT>min_derr = <NUM_LIT:3> * st.sigma<EOL><DEDENT>accepts = <NUM_LIT:0><EOL>new_poses = []<EOL>if rad == '<STR_LIT>':<EOL><INDENT>rad = guess_add_radii(st)<EOL><DEDENT>message = ('<STR_LIT:->'*<NUM_LIT:30> + '<STR_LIT>' + '<STR_LIT:->'*<NUM_LIT:30> +<EOL>'<STR_LIT>')<EOL>with log.noformat():<EOL><INDENT>CLOG.info(message)<EOL><DEDENT>for a in range(guess.shape[<NUM_LIT:0>]):<EOL><INDENT>p0 = guess[a]<EOL>absent_err = st.error<EOL>absent_d = st.residuals.copy()<EOL>ind = st.obj_add_particle(p0, rad)<EOL>if do_opt:<EOL><INDENT>opt.do_levmarq_particles(<EOL>st, ind, damping=<NUM_LIT:1.0>, max_iter=<NUM_LIT:1>, run_length=<NUM_LIT:3>,<EOL>eig_update=False, include_rad=False)<EOL><DEDENT>present_err = st.error<EOL>present_d = st.residuals.copy()<EOL>dont_kill = should_particle_exist(<EOL>absent_err, present_err, absent_d, present_d,<EOL>im_change_frac=im_change_frac, min_derr=min_derr)<EOL>if dont_kill:<EOL><INDENT>accepts += <NUM_LIT:1><EOL>p = tuple(st.obj_get_positions()[ind].ravel())<EOL>r = tuple(st.obj_get_radii()[ind].ravel())<EOL>new_poses.append(p)<EOL>part_msg = '<STR_LIT>' % (<EOL>p + r + (absent_err, st.error))<EOL>with log.noformat():<EOL><INDENT>CLOG.info(part_msg)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>st.obj_remove_particle(ind)<EOL>if np.abs(absent_err - st.error) > <NUM_LIT>:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>return accepts, new_poses<EOL>

Checks whether to add particles at a given position by seeing if adding
the particle improves the fit of the state.

Parameters
----------
st : :class:`peri.states.State`
    The state to check adding particles to.
guess : [N,3] list-like
    The positions of particles to check to add.
rad : {Float, ``'calc'``}, optional.
    The radius of the newly-added particles. Default is ``'calc'``,
    which uses the states current radii's median.
do_opt : Bool, optional
    Whether to optimize the particle position before checking if it
    should be kept. Default is True (optimizes position).
im_change_frac : Float
    How good the change in error needs to be relative to the change in
    the difference image. Default is 0.2; i.e. if the error does not
    decrease by 20% of the change in the difference image, do not add
    the particle.
min_derr : Float or '3sig'
    The minimal improvement in error to add a particle. Default
    is ``'3sig' = 3*st.sigma``.

Returns
-------
accepts : Int
    The number of added particles
new_poses : [N,3] list
    List of the positions of the added particles. If ``do_opt==True``,
    then these positions will differ from the input 'guess'.


def check_remove_particle(st, ind, im_change_frac=<NUM_LIT>, min_derr='<STR_LIT>',<EOL>**kwargs):

<EOL>if min_derr == '<STR_LIT>':<EOL><INDENT>min_derr = <NUM_LIT:3> * st.sigma<EOL><DEDENT>present_err = st.error<EOL>present_d = st.residuals.copy()<EOL>p, r = st.obj_remove_particle(ind)<EOL>p = p[<NUM_LIT:0>]<EOL>r = r[<NUM_LIT:0>]<EOL>absent_err = st.error<EOL>absent_d = st.residuals.copy()<EOL>if should_particle_exist(absent_err, present_err, absent_d, present_d,<EOL>im_change_frac=im_change_frac, min_derr=min_derr):<EOL><INDENT>st.obj_add_particle(p, r)<EOL>killed = False<EOL><DEDENT>else:<EOL><INDENT>killed = True<EOL><DEDENT>return killed, tuple(p), (r,)<EOL>

Checks whether to remove particle 'ind' from state 'st'. If removing the
particle increases the error by less than max( min_derr, change in image *
im_change_frac), then the particle is removed.

Parameters
----------
st : :class:`peri.states.State`
    The state to check adding particles to.
ind : Int
    The index of the particle to check to remove.
im_change_frac : Float
    How good the change in error needs to be relative to the change in
    the difference image. Default is 0.2; i.e. if the error does not
    decrease by 20% of the change in the difference image, do not add
    the particle.
min_derr : Float or '3sig'
    The minimal improvement in error to add a particle. Default is
    ``'3sig' = 3*st.sigma``.

Returns
-------
killed : Bool
    Whether the particle was removed.
p : Tuple
    The position of the removed particle.
r : Tuple
    The radius of the removed particle.


def should_particle_exist(absent_err, present_err, absent_d, present_d,<EOL>im_change_frac=<NUM_LIT>, min_derr=<NUM_LIT:0.1>):

delta_im = np.ravel(present_d - absent_d)<EOL>im_change = np.dot(delta_im, delta_im)<EOL>err_cutoff = max([im_change_frac * im_change, min_derr])<EOL>return (absent_err - present_err) >= err_cutoff<EOL>

Checks whether or not adding a particle should be present.

Parameters
----------
absent_err : Float
    The state error without the particle.
present_err : Float
    The state error with the particle.
absent_d : numpy.ndarray
    The state residuals without the particle.
present_d : numpy.ndarray
    The state residuals with the particle.
im_change_frac : Float, optional
    How good the change in error needs to be relative to the change in
    the residuals. Default is 0.2; i.e. return False if the error does
    not decrease by 0.2 x the change in the residuals.
min_derr : Float, optional
    The minimal improvement in error. Default is 0.1

Returns
-------
Bool
    True if the errors is better with the particle present.


def add_missing_particles(st, rad='<STR_LIT>', tries=<NUM_LIT:50>, **kwargs):

if rad == '<STR_LIT>':<EOL><INDENT>rad = guess_add_radii(st)<EOL><DEDENT>guess, npart = feature_guess(st, rad, **kwargs)<EOL>tries = np.min([tries, npart])<EOL>accepts, new_poses = check_add_particles(<EOL>st, guess[:tries], rad=rad, **kwargs)<EOL>return accepts, new_poses<EOL>

Attempts to add missing particles to the state.

Operates by:
(1) featuring the difference image using feature_guess,
(2) attempting to add the featured positions using check_add_particles.

Parameters
----------
st : :class:`peri.states.State`
    The state to check adding particles to.
rad : Float or 'calc', optional
    The radius of the newly-added particles and of the feature size for
    featuring. Default is 'calc', which uses the median of the state's
    current radii.
tries : Int, optional
    How many particles to attempt to add. Only tries to add the first
    ``tries`` particles, in order of mass. Default is 50.

Other Parameters
----------------
invert : Bool, optional
    Whether to invert the image. Default is ``True``, i.e. dark particles
minmass : Float or None, optionals
    The minimum mass/masscut of a particle. Default is ``None``=calcualted
    by ``feature_guess``.
use_tp : Bool, optional
    Whether to use trackpy in feature_guess. Default is False, since
    trackpy cuts out particles at the edge.

do_opt : Bool, optional
    Whether to optimize the particle position before checking if it
    should be kept. Default is True (optimizes position).
im_change_frac : Float, optional
    How good the change in error needs to be relative to the change
    in the difference image. Default is 0.2; i.e. if the error does
    not decrease by 20% of the change in the difference image, do
    not add the particle.

min_derr : Float or '3sig', optional
    The minimal improvement in error to add a particle. Default
    is ``'3sig' = 3*st.sigma``.

Returns
-------
accepts : Int
    The number of added particles
new_poses : [N,3] list
    List of the positions of the added particles. If ``do_opt==True``,
    then these positions will differ from the input 'guess'.


def remove_bad_particles(st, min_rad='<STR_LIT>', max_rad='<STR_LIT>', min_edge_dist=<NUM_LIT>,<EOL>check_rad_cutoff=[<NUM_LIT>, <NUM_LIT:15>], check_outside_im=True,<EOL>tries=<NUM_LIT:50>, im_change_frac=<NUM_LIT>, **kwargs):

is_near_im_edge = lambda pos, pad: (((pos + st.pad) < pad) | (pos ><EOL>np.array(st.ishape.shape) + st.pad - pad)).any(axis=<NUM_LIT:1>)<EOL>removed = <NUM_LIT:0><EOL>attempts = <NUM_LIT:0><EOL>n_tot_part = st.obj_get_positions().shape[<NUM_LIT:0>]<EOL>q10 = int(<NUM_LIT:0.1> * n_tot_part)  <EOL>r_sig = np.sort(st.obj_get_radii())[q10:-q10].std()<EOL>r_med = np.median(st.obj_get_radii())<EOL>if max_rad == '<STR_LIT>':<EOL><INDENT>max_rad = r_med + <NUM_LIT:15>*r_sig<EOL><DEDENT>if min_rad == '<STR_LIT>':<EOL><INDENT>min_rad = r_med - <NUM_LIT>*r_sig<EOL><DEDENT>if check_rad_cutoff == '<STR_LIT>':<EOL><INDENT>check_rad_cutoff = [r_med - <NUM_LIT>*r_sig, r_med + <NUM_LIT>*r_sig]<EOL><DEDENT>rad_wrong_size = np.nonzero(<EOL>(st.obj_get_radii() < min_rad) | (st.obj_get_radii() > max_rad))[<NUM_LIT:0>]<EOL>near_im_edge = np.nonzero(is_near_im_edge(st.obj_get_positions(),<EOL>min_edge_dist - st.pad))[<NUM_LIT:0>]<EOL>delete_inds = np.unique(np.append(rad_wrong_size, near_im_edge)).tolist()<EOL>delete_poses = st.obj_get_positions()[delete_inds].tolist()<EOL>message = ('<STR_LIT:->'*<NUM_LIT> + '<STR_LIT>' + '<STR_LIT:->'*<NUM_LIT> +<EOL>'<STR_LIT>')<EOL>with log.noformat():<EOL><INDENT>CLOG.info(message)<EOL><DEDENT>for pos in delete_poses:<EOL><INDENT>ind = st.obj_closest_particle(pos)<EOL>old_err = st.error<EOL>p, r = st.obj_remove_particle(ind)<EOL>p = p[<NUM_LIT:0>]<EOL>r = r[<NUM_LIT:0>]<EOL>part_msg = '<STR_LIT>' % (<EOL>tuple(p) + (r,) + (old_err, st.error))<EOL>with log.noformat():<EOL><INDENT>CLOG.info(part_msg)<EOL><DEDENT>removed += <NUM_LIT:1><EOL><DEDENT>check_rad_inds = np.nonzero((st.obj_get_radii() < check_rad_cutoff[<NUM_LIT:0>]) |<EOL>(st.obj_get_radii() > check_rad_cutoff[<NUM_LIT:1>]))[<NUM_LIT:0>]<EOL>if check_outside_im:<EOL><INDENT>check_edge_inds = np.nonzero(<EOL>is_near_im_edge(st.obj_get_positions(), st.pad))[<NUM_LIT:0>]<EOL>check_inds = np.unique(np.append(check_rad_inds, check_edge_inds))<EOL><DEDENT>else:<EOL><INDENT>check_inds = check_rad_inds<EOL><DEDENT>check_inds = check_inds[np.argsort(st.obj_get_radii()[check_inds])]<EOL>tries = np.min([tries, check_inds.size])<EOL>check_poses = st.obj_get_positions()[check_inds[:tries]].copy()<EOL>for pos in check_poses:<EOL><INDENT>old_err = st.error<EOL>ind = st.obj_closest_particle(pos)<EOL>killed, p, r = check_remove_particle(<EOL>st, ind, im_change_frac=im_change_frac)<EOL>if killed:<EOL><INDENT>removed += <NUM_LIT:1><EOL>check_inds[check_inds > ind] -= <NUM_LIT:1>  <EOL>delete_poses.append(pos)<EOL>part_msg = '<STR_LIT>' % (<EOL>p + r + (old_err, st.error))<EOL>with log.noformat():<EOL><INDENT>CLOG.info(part_msg)<EOL><DEDENT><DEDENT><DEDENT>return removed, delete_poses<EOL>

Removes improperly-featured particles from the state, based on a
combination of particle size and the change in error on removal.

Parameters
-----------
st : :class:`peri.states.State`
    The state to remove bad particles from.
min_rad : Float, optional
    All particles with radius below min_rad are automatically deleted.
    Set to 'calc' to make it the median rad - 25* radius std.
    Default is 'calc'.

max_rad : Float, optional
    All particles with radius above max_rad are automatically deleted.
    Set to 'calc' to make it the median rad + 15* radius std.
    Default is 'calc'.

min_edge_dist : Float, optional
    All particles within min_edge_dist of the (padded) image
    edges are automatically deleted. Default is 2.0

check_rad_cutoff : 2-element list of floats, optional
    Particles with radii < check_rad_cutoff[0] or > check_rad_cutoff[1]
    are checked if they should be deleted. Set to 'calc' to make it the
    median rad +- 3.5 * radius std. Default is [3.5, 15].

check_outside_im : Bool, optional
    If True, checks if particles located outside the unpadded image
    should be deleted. Default is True.

tries : Int, optional
    The maximum number of particles with radii < check_rad_cutoff
    to try to remove. Checks in increasing order of radius size.
    Default is 50.

im_change_frac : Float, , optional
    Number between 0 and 1. If removing a particle decreases the
    error by less than im_change_frac*the change in the image, then
    the particle is deleted. Default is 0.2

Returns
-------
removed: Int
    The cumulative number of particles removed.


def add_subtract(st, max_iter=<NUM_LIT:7>, max_npart='<STR_LIT>', max_mem=<NUM_LIT>,<EOL>always_check_remove=False, **kwargs):

if max_npart == '<STR_LIT>':<EOL><INDENT>max_npart = <NUM_LIT> * st.obj_get_positions().shape[<NUM_LIT:0>]<EOL><DEDENT>total_changed = <NUM_LIT:0><EOL>_change_since_opt = <NUM_LIT:0><EOL>removed_poses = []<EOL>added_poses0 = []<EOL>added_poses = []<EOL>nr = <NUM_LIT:1>  <EOL>for _ in range(max_iter):<EOL><INDENT>if (nr != <NUM_LIT:0>) or (always_check_remove):<EOL><INDENT>nr, rposes = remove_bad_particles(st, **kwargs)<EOL><DEDENT>na, aposes = add_missing_particles(st, **kwargs)<EOL>current_changed = na + nr<EOL>removed_poses.extend(rposes)<EOL>added_poses0.extend(aposes)<EOL>total_changed += current_changed<EOL>_change_since_opt += current_changed<EOL>if current_changed == <NUM_LIT:0>:<EOL><INDENT>break<EOL><DEDENT>elif _change_since_opt > max_npart:<EOL><INDENT>_change_since_opt *= <NUM_LIT:0><EOL>CLOG.info('<STR_LIT>')<EOL>opt.do_levmarq(st, opt.name_globals(st, remove_params=st.get(<EOL>'<STR_LIT>').params), max_iter=<NUM_LIT:1>, run_length=<NUM_LIT:4>, num_eig_dirs=<NUM_LIT:3>,<EOL>max_mem=max_mem, eig_update_frequency=<NUM_LIT:2>, rz_order=<NUM_LIT:0>,<EOL>use_accel=True)<EOL>CLOG.info('<STR_LIT>'.format(st.error))<EOL><DEDENT><DEDENT>for p in added_poses0:<EOL><INDENT>i = st.obj_closest_particle(p)<EOL>opt.do_levmarq_particles(st, np.array([i]), max_iter=<NUM_LIT:2>, damping=<NUM_LIT>)<EOL>added_poses.append(st.obj_get_positions()[i])<EOL><DEDENT>return total_changed, np.array(removed_poses), np.array(added_poses)<EOL>

Automatically adds and subtracts missing & extra particles.

Operates by removing bad particles then adding missing particles on
repeat, until either no particles are added/removed or after `max_iter`
attempts.

Parameters
----------
st: :class:`peri.states.State`
    The state to add and subtract particles to.
max_iter : Int, optional
    The maximum number of add-subtract loops to use. Default is 7.
    Terminates after either max_iter loops or when nothing has changed.
max_npart : Int or 'calc', optional
    The maximum number of particles to add before optimizing the non-psf
    globals. Default is ``'calc'``, which uses 5% of the initial number
    of particles.
max_mem : Int, optional
    The maximum memory to use for optimization after adding max_npart
    particles. Default is 2e8.
always_check_remove : Bool, optional
    Set to True to always check whether to remove particles. If ``False``,
    only checks for removal while particles were removed on the previous
    attempt. Default is False.

Other Parameters
----------------
invert : Bool, optional
    ``True`` if the particles are dark on a bright background, ``False``
    if they are bright on a dark background. Default is ``True``.
min_rad : Float, optional
    Particles with radius below ``min_rad`` are automatically deleted.
    Default is ``'calc'`` = median rad - 25* radius std.
max_rad : Float, optional
    Particles with radius above ``max_rad`` are automatically deleted.
    Default is ``'calc'`` = median rad + 15* radius std, but you should
    change this for your particle sizes.

min_edge_dist : Float, optional
    Particles closer to the edge of the padded image than this are
    automatically deleted. Default is 2.0.
check_rad_cutoff : 2-element float list.
    Particles with ``radii < check_rad_cutoff[0]`` or ``> check...[1]``
    are checked if they should be deleted (not automatic). Default is
    ``[3.5, 15]``.
check_outside_im : Bool, optional
    Set to True to check whether to delete particles whose positions are
    outside the un-padded image.

rad : Float, optional
    The initial radius for added particles; added particles radii are
    not fit until the end of ``add_subtract``. Default is ``'calc'``,
    which uses the median radii of active particles.

tries : Int, optional
    The number of particles to attempt to remove or add, per iteration.
    Default is 50.

im_change_frac : Float, optional
    How good the change in error needs to be relative to the change in
    the difference image. Default is 0.2; i.e. if the error does not
    decrease by 20% of the change in the difference image, do not add
    the particle.

min_derr : Float, optional
    The minimum change in the state's error to keep a particle in the
    image. Default is ``'3sig'`` which uses ``3*st.sigma``.

do_opt : Bool, optional
    Set to False to avoid optimizing particle positions after adding.
minmass : Float, optional
    The minimum mass for a particle to be identified as a feature,
    as used by trackpy. Defaults to a decent guess.

use_tp : Bool, optional
    Set to True to use trackpy to find missing particles inside the
    image. Not recommended since trackpy deliberately cuts out particles
    at the edge of the image. Default is ``False``.

Returns
-------
total_changed : Int
    The total number of adds and subtracts done on the data. Not the
    same as ``changed_inds.size`` since the same particle or particle
    index can be added/subtracted multiple times.
added_positions : [N_added,3] numpy.ndarray
    The positions of particles that have been added at any point in the
    add-subtract cycle.
removed_positions : [N_added,3] numpy.ndarray
    The positions of particles that have been removed at any point in
    the add-subtract cycle.

Notes
------
Occasionally after the intial featuring a cluster of particles is
featured as 1 big particle. To fix these mistakes, it helps to set
max_rad to a physical value. This removes the big particle and allows
it to be re-featured by (several passes of) the adds.

The added/removed positions returned are whether or not the position
has been added or removed ever. It's possible that a position is
added, then removed during a later iteration.


def identify_misfeatured_regions(st, filter_size=<NUM_LIT:5>, sigma_cutoff=<NUM_LIT>):

<EOL>r = st.residuals<EOL>weights = np.ones([filter_size]*len(r.shape), dtype='<STR_LIT:float>')<EOL>weights /= weights.sum()<EOL>f = np.sqrt(nd.filters.convolve(r*r, weights, mode='<STR_LIT>'))<EOL>if sigma_cutoff == '<STR_LIT>':<EOL><INDENT>max_ok = initializers.otsu_threshold(f)<EOL><DEDENT>else:<EOL><INDENT>max_ok = f.mean() + sigma_cutoff * f.std()<EOL><DEDENT>bad = f > max_ok<EOL>labels, n = nd.measurements.label(bad)<EOL>inds = []<EOL>for i in range(<NUM_LIT:1>, n+<NUM_LIT:1>):<EOL><INDENT>inds.append(np.nonzero(labels == i))<EOL><DEDENT>tiles = [Tile(np.min(ind, axis=<NUM_LIT:1>), np.max(ind, axis=<NUM_LIT:1>)+<NUM_LIT:1>) for ind in inds]<EOL>volumes = [t.volume for t in tiles]<EOL>return [tiles[i] for i in np.argsort(volumes)[::-<NUM_LIT:1>]]<EOL>

Identifies regions of missing/misfeatured particles based on the
residuals' local deviation from uniform Gaussian noise.

Parameters
----------
st : :class:`peri.states.State`
    The state in which to identify mis-featured regions.

filter_size : Int, best if odd.
    The size of the filter for calculating the local standard deviation;
    should approximately be the size of a poorly featured region in
    each dimension. Default is 5.

sigma_cutoff : Float or `otsu`, optional
    The max allowed deviation of the residuals from what is expected,
    in units of the residuals' standard deviation. Lower means more
    sensitive, higher = less sensitive. Default is 8.0, i.e. one pixel
    out of every 7*10^11 is mis-identified randomly. In practice the
    noise is not Gaussian so there are still some regions mis-identified
    as improperly featured. Set to ```otsu``` to calculate this number
    based on an automatic Otsu threshold.

Returns
-------
tiles : List of :class:`peri.util.Tile`
    Each tile is the smallest bounding tile that contains an improperly
    featured region. The list is sorted by the tile's volume.

Notes
-----
Algorithm is
1.  Create a field of the local standard deviation, as measured over
    a hypercube of size filter_size.
2.  Find the maximum reasonable value of the field. [The field should
    be a random variable with mean of r.std() and standard deviation
    of ~r.std() / sqrt(N), where r is the residuals and N is the
    number of pixels in the hypercube.]
3.  Label & Identify the misfeatured regions as portions where
    the local error is too large.
4.  Parse the misfeatured regions into tiles.
5.  Return the sorted tiles.
The Otsu option to calculate the sigma cutoff works well for images
that actually contain missing particles, returning a number similar
to one calculated with a sigma cutoff. However, if the image is
well-featured with Gaussian residuals, then the Otsu threshold
splits the Gaussian down the middle instead of at the tails, which
is very bad. So use with caution.


def add_subtract_misfeatured_tile(<EOL>st, tile, rad='<STR_LIT>', max_iter=<NUM_LIT:3>, invert='<STR_LIT>', max_allowed_remove=<NUM_LIT:20>,<EOL>minmass=None, use_tp=False, **kwargs):

if rad == '<STR_LIT>':<EOL><INDENT>rad = guess_add_radii(st)<EOL><DEDENT>if invert == '<STR_LIT>':<EOL><INDENT>invert = guess_invert(st)<EOL><DEDENT>initial_error = np.copy(st.error)<EOL>rinds = np.nonzero(tile.contains(st.obj_get_positions()))[<NUM_LIT:0>]<EOL>if rinds.size >= max_allowed_remove:<EOL><INDENT>CLOG.fatal('<STR_LIT>')<EOL>raise RuntimeError<EOL><DEDENT>elif rinds.size >= max_allowed_remove/<NUM_LIT:2>:<EOL><INDENT>CLOG.warn('<STR_LIT>')<EOL><DEDENT>elif rinds.size > <NUM_LIT:0>:<EOL><INDENT>rpos, rrad = st.obj_remove_particle(rinds)<EOL><DEDENT>n_added = -rinds.size<EOL>added_poses = []<EOL>for _ in range(max_iter):<EOL><INDENT>if invert:<EOL><INDENT>im = <NUM_LIT:1> - st.residuals[tile.slicer]<EOL><DEDENT>else:<EOL><INDENT>im = st.residuals[tile.slicer]<EOL><DEDENT>guess, _ = _feature_guess(im, rad, minmass=minmass, use_tp=use_tp)<EOL>accepts, poses = check_add_particles(<EOL>st, guess+tile.l, rad=rad, do_opt=True, **kwargs)<EOL>added_poses.extend(poses)<EOL>n_added += accepts<EOL>if accepts == <NUM_LIT:0>:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:  <EOL><INDENT>CLOG.warn('<STR_LIT>')<EOL><DEDENT>ainds = []<EOL>for p in added_poses:<EOL><INDENT>ainds.append(st.obj_closest_particle(p))<EOL><DEDENT>if len(ainds) > max_allowed_remove:<EOL><INDENT>for i in range(<NUM_LIT:0>, len(ainds), max_allowed_remove):<EOL><INDENT>opt.do_levmarq_particles(<EOL>st, np.array(ainds[i:i + max_allowed_remove]),<EOL>include_rad=True, max_iter=<NUM_LIT:3>)<EOL><DEDENT><DEDENT>elif len(ainds) > <NUM_LIT:0>:<EOL><INDENT>opt.do_levmarq_particles(st, ainds, include_rad=True, max_iter=<NUM_LIT:3>)<EOL><DEDENT>did_something = (rinds.size > <NUM_LIT:0>) or (len(ainds) > <NUM_LIT:0>)<EOL>if did_something & (st.error > initial_error):<EOL><INDENT>CLOG.info('<STR_LIT>'.format(<EOL>tile.l.tolist(), tile.r.tolist()))<EOL>if len(ainds) > <NUM_LIT:0>:<EOL><INDENT>_ = st.obj_remove_particle(ainds)<EOL><DEDENT>if rinds.size > <NUM_LIT:0>:<EOL><INDENT>for p, r in zip(rpos.reshape(-<NUM_LIT:1>, <NUM_LIT:3>), rrad.reshape(-<NUM_LIT:1>)):<EOL><INDENT>_ = st.obj_add_particle(p, r)<EOL><DEDENT><DEDENT>n_added = <NUM_LIT:0><EOL>ainds = []<EOL><DEDENT>return n_added, ainds<EOL>

Automatically adds and subtracts missing & extra particles in a region
of poor fit.

Parameters
----------
st: :class:`peri.states.State`
    The state to add and subtract particles to.
tile : :class:`peri.util.Tile`
    The poorly-fit region to examine.
rad : Float or 'calc', optional
    The initial radius for added particles; added particles radii are
    not fit until the end of add_subtract. Default is ``'calc'``, which
    uses the median radii of active particles.
max_iter : Int, optional
    The maximum number of loops for attempted adds at one tile location.
    Default is 3.
invert : {'guess', True, False}, optional
    Whether to invert the image for feature_guess -- True for dark
    particles on a bright background, False for bright particles. The
    default is to guess from the state's current particles.
max_allowed_remove : Int, optional
    The maximum number of particles to remove. If the misfeatured tile
    contains more than this many particles, raises an error. If it
    contains more than half as many particles, logs a warning. If more
    than this many particles are added, they are optimized in blocks of
    ``max_allowed_remove``. Default is 20.

Other Parameters
----------------
im_change_frac : Float on [0, 1], optional.
    If adding or removing a particle decreases the error less than
    ``im_change_frac``*the change in the image, the particle is deleted.
    Default is 0.2.

min_derr : {Float, ``'3sig'``}, optional
    The minimum change in the state's error to keep a particle in the
    image. Default is ``'3sig'`` which uses ``3*st.sigma``.

do_opt : Bool, optional
    Set to False to avoid optimizing particle positions after adding
    them. Default is True.

minmass : Float, optional
    The minimum mass for a particle to be identified as a feature, as
    used by trackpy. Defaults to a decent guess.

use_tp : Bool, optional
    Set to True to use trackpy to find missing particles inside the
    image. Not recommended since trackpy deliberately cuts out particles
    at the edge of the image. Default is False.

Outputs
-------
n_added : Int
    The change in the number of particles, i.e. ``n_added-n_subtracted``
ainds: List of ints
    The indices of the added particles.

Notes
--------
The added/removed positions returned are whether or not the
position has been added or removed ever. It's possible/probably that
a position is added, then removed during a later iteration.

Algorithm is:
1.  Remove all particles within the tile.
2.  Feature and add particles to the tile.
3.  Optimize the added particles positions only.
4.  Run 2-3 until no particles have been added.
5.  Optimize added particle radii
Because all the particles are removed within a tile, it is important
to set max_allowed_remove to a reasonable value. Otherwise, if the
tile is the size of the image it can take a long time to remove all
the particles and re-add them.