if isinstance(analyte, str):<EOL><INDENT>analyte = [analyte]<EOL><DEDENT>if isinstance(filt, (int, float)):<EOL><INDENT>filt = [filt]<EOL><DEDENT>elif isinstance(filt, str):<EOL><INDENT>filt = self.fuzzmatch(filt, multi=True)<EOL><DEDENT>if analyte is None:<EOL><INDENT>analyte = self.analytes<EOL><DEDENT>if filt is None:<EOL><INDENT>filt = list(self.index.values())<EOL><DEDENT>for a in analyte:<EOL><INDENT>for f in filt:<EOL><INDENT>if isinstance(f, (int, float)):<EOL><INDENT>f = self.index[int(f)]<EOL><DEDENT>try:<EOL><INDENT>self.switches[a][f] = True<EOL><DEDENT>except KeyError:<EOL><INDENT>f = self.fuzzmatch(f, multi=False)<EOL>self.switches[a][f] = True<EOL><DEDENT><DEDENT><DEDENT>return<EOL>

Turn on specified filter(s) for specified analyte(s).

Parameters
----------
analyte : optional, str or array_like
    Name or list of names of analytes.
    Defaults to all analytes.
filt : optional. int, str or array_like
    Name/number or iterable names/numbers of filters.

Returns
-------
None


if isinstance(analyte, str):<EOL><INDENT>analyte = [analyte]<EOL><DEDENT>if isinstance(filt, (int, float)):<EOL><INDENT>filt = [filt]<EOL><DEDENT>elif isinstance(filt, str):<EOL><INDENT>filt = self.fuzzmatch(filt, multi=True)<EOL><DEDENT>if analyte is None:<EOL><INDENT>analyte = self.analytes<EOL><DEDENT>if filt is None:<EOL><INDENT>filt = list(self.index.values())<EOL><DEDENT>for a in analyte:<EOL><INDENT>for f in filt:<EOL><INDENT>if isinstance(f, int):<EOL><INDENT>f = self.index[f]<EOL><DEDENT>try:<EOL><INDENT>self.switches[a][f] = False<EOL><DEDENT>except KeyError:<EOL><INDENT>f = self.fuzzmatch(f, multi=False)<EOL>self.switches[a][f] = False<EOL><DEDENT><DEDENT><DEDENT>return<EOL>

Turn off specified filter(s) for specified analyte(s).

Parameters
----------
analyte : optional, str or array_like
    Name or list of names of analytes.
    Defaults to all analytes.
filt : optional. int, list of int or str
    Number(s) or partial string that corresponds to filter name(s).

Returns
-------
None


if analyte is None:<EOL><INDENT>analyte = self.analytes<EOL><DEDENT>elif isinstance(analyte, str):<EOL><INDENT>analyte = [analyte]<EOL><DEDENT>out = []<EOL>for f in self.components.keys():<EOL><INDENT>for a in analyte:<EOL><INDENT>if self.switches[a][f]:<EOL><INDENT>out.append(f)<EOL><DEDENT><DEDENT><DEDENT>key = '<STR_LIT>'.join(sorted(out))<EOL>for a in analyte:<EOL><INDENT>self.keys[a] = key<EOL><DEDENT>return self.make_fromkey(key)<EOL>

Make filter for specified analyte(s).

Filter specified in filt.switches.

Parameters
----------
analyte : str or array_like
    Name or list of names of analytes.

Returns
-------
array_like
    boolean filter


def fuzzmatch(self, fuzzkey, multi=False):

keys, ratios = np.array([(f, seqm(None, fuzzkey, f).ratio()) for f in self.components.keys()]).T<EOL>mratio = max(ratios)<EOL>if multi:<EOL><INDENT>return keys[ratios == mratio]<EOL><DEDENT>else:<EOL><INDENT>if sum(ratios == mratio) == <NUM_LIT:1>:<EOL><INDENT>return keys[ratios == mratio][<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>".format(fuzzkey) + '<STR_LIT:U+002CU+0020>'.join(keys[ratios == mratio]) + "<STR_LIT>")<EOL><DEDENT><DEDENT>

Identify a filter by fuzzy string matching.

Partial ('fuzzy') matching performed by `fuzzywuzzy.fuzzy.ratio`

Parameters
----------
fuzzkey : str
    A string that partially matches one filter name more than the others.

Returns
-------
The name of the most closely matched filter. : str


if key != '<STR_LIT>':<EOL><INDENT>def make_runable(match):<EOL><INDENT>return "<STR_LIT>" + self.fuzzmatch(match.group(<NUM_LIT:0>)) + "<STR_LIT>"<EOL><DEDENT>runable = re.sub('<STR_LIT>', make_runable, key)<EOL>return eval(runable)<EOL><DEDENT>else:<EOL><INDENT>return ~np.zeros(self.size, dtype=bool)<EOL><DEDENT>

Make filter from logical expression.

Takes a logical expression as an input, and returns a filter. Used for advanced
filtering, where combinations of nested and/or filters are desired. Filter names must
exactly match the names listed by print(filt).

Example: ``key = '(Filter_1 | Filter_2) & Filter_3'``
is equivalent to:
``(Filter_1 OR Filter_2) AND Filter_3``
statements in parentheses are evaluated first.

Parameters
----------
key : str
    logical expression describing filter construction.

Returns
-------
array_like
    boolean filter


if analyte is None:<EOL><INDENT>analyte = self.analytes<EOL><DEDENT>elif isinstance(analyte, str):<EOL><INDENT>analyte = [analyte]<EOL><DEDENT>out = {}<EOL>for a in analyte:<EOL><INDENT>key = []<EOL>for f in self.components.keys():<EOL><INDENT>if self.switches[a][f]:<EOL><INDENT>key.append(f)<EOL><DEDENT><DEDENT>out[a] = '<STR_LIT>'.join(sorted(key))<EOL><DEDENT>self.keydict = out<EOL>return out<EOL>

Make logical expressions describing the filter(s) for specified analyte(s).

Parameters
----------
analyte : optional, str or array_like
    Name or list of names of analytes.
    Defaults to all analytes.

Returns
-------
dict
    containing the logical filter expression for each analyte.


def grab_filt(self, filt, analyte=None):

if isinstance(filt, str):<EOL><INDENT>if filt in self.components:<EOL><INDENT>if analyte is None:<EOL><INDENT>return self.components[filt]<EOL><DEDENT>else:<EOL><INDENT>if self.switches[analyte][filt]:<EOL><INDENT>return self.components[filt]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>ind = self.make_fromkey(filt)<EOL><DEDENT>except KeyError:<EOL><INDENT>print(("<STR_LIT>"<EOL>"<STR_LIT>"))<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(filt, dict):<EOL><INDENT>try:<EOL><INDENT>ind = self.make_fromkey(filt[analyte])<EOL><DEDENT>except ValueError:<EOL><INDENT>print(("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"))<EOL><DEDENT><DEDENT>elif filt:<EOL><INDENT>ind = self.make(analyte)<EOL><DEDENT>else:<EOL><INDENT>ind = ~np.zeros(self.size, dtype=bool)<EOL><DEDENT>return ind<EOL>

Flexible access to specific filter using any key format.

Parameters
----------
f : str, dict or bool
    either logical filter expression, dict of expressions,
    or a boolean
analyte : str
    name of analyte the filter is for.

Returns
-------
array_like
    boolean filter


def get_components(self, key, analyte=None):

out = {}<EOL>for k, v in self.components.items():<EOL><INDENT>if key in k:<EOL><INDENT>if analyte is None:<EOL><INDENT>out[k] = v<EOL><DEDENT>elif self.switches[analyte][k]:<EOL><INDENT>out[k] = v<EOL><DEDENT><DEDENT><DEDENT>return out<EOL>

Extract filter components for specific analyte(s).

Parameters
----------
key : str
    string present in one or more filter names.
    e.g. 'Al27' will return all filters with
    'Al27' in their names.
analyte : str
    name of analyte the filter is for

Returns
-------
boolean filter : array-like


out = '<STR_LIT>'<EOL>for k in sorted(self.components.keys()):<EOL><INDENT>out += '<STR_LIT>'.format(k, self.info[k]) + '<STR_LIT:\n>'<EOL><DEDENT>return(out)<EOL>


def cluster_meanshift(data, bandwidth=None, bin_seeding=False, **kwargs):

if bandwidth is None:<EOL><INDENT>bandwidth = cl.estimate_bandwidth(data)<EOL><DEDENT>ms = cl.MeanShift(bandwidth=bandwidth, bin_seeding=bin_seeding, **kwargs)<EOL>ms.fit(data)<EOL>labels = ms.labels_<EOL>return labels, [np.nan]<EOL>

Identify clusters using Meanshift algorithm.

Parameters
----------
data : array_like
    array of size [n_samples, n_features].
bandwidth : float or None
    If None, bandwidth is estimated automatically using
    sklean.cluster.estimate_bandwidth
bin_seeding : bool
    Setting this option to True will speed up the algorithm.
    See sklearn documentation for full description.

Returns
-------
dict
    boolean array for each identified cluster.


def cluster_kmeans(data, n_clusters, **kwargs):

km = cl.KMeans(n_clusters, **kwargs)<EOL>kmf = km.fit(data)<EOL>labels = kmf.labels_<EOL>return labels, [np.nan]<EOL>

Identify clusters using K - Means algorithm.

Parameters
----------
data : array_like
    array of size [n_samples, n_features].
n_clusters : int
    The number of clusters expected in the data.

Returns
-------
dict
    boolean array for each identified cluster.


def cluster_DBSCAN(data, eps=None, min_samples=None,<EOL>n_clusters=None, maxiter=<NUM_LIT:200>, **kwargs):

if n_clusters is None:<EOL><INDENT>if eps is None:<EOL><INDENT>eps = <NUM_LIT><EOL><DEDENT>db = cl.DBSCAN(eps=eps, min_samples=min_samples, **kwargs).fit(data)<EOL><DEDENT>else:<EOL><INDENT>clusters = <NUM_LIT:0><EOL>eps_temp = <NUM_LIT:1> / <NUM_LIT><EOL>niter = <NUM_LIT:0><EOL>while clusters < n_clusters:<EOL><INDENT>clusters_last = clusters<EOL>eps_temp *= <NUM_LIT><EOL>db = cl.DBSCAN(eps=eps_temp, min_samples=min_samples, **kwargs).fit(data)<EOL>clusters = (len(set(db.labels_)) -<EOL>(<NUM_LIT:1> if -<NUM_LIT:1> in db.labels_ else <NUM_LIT:0>))<EOL>if clusters < clusters_last:<EOL><INDENT>eps_temp *= <NUM_LIT:1> / <NUM_LIT><EOL>db = cl.DBSCAN(eps=eps_temp, min_samples=min_samples, **kwargs).fit(data)<EOL>clusters = (len(set(db.labels_)) -<EOL>(<NUM_LIT:1> if -<NUM_LIT:1> in db.labels_ else <NUM_LIT:0>))<EOL>warnings.warn(('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>').format(n_clusters, clusters, eps_temp))<EOL>break<EOL><DEDENT>niter += <NUM_LIT:1><EOL>if niter == maxiter:<EOL><INDENT>warnings.warn(('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>').format(maxiter, n_clusters))<EOL>break<EOL><DEDENT><DEDENT><DEDENT>labels = db.labels_<EOL>core_samples_mask = np.zeros_like(labels)<EOL>core_samples_mask[db.core_sample_indices_] = True<EOL>return labels, core_samples_mask<EOL>

Identify clusters using DBSCAN algorithm.

Parameters
----------
data : array_like
    array of size [n_samples, n_features].
eps : float
    The minimum 'distance' points must be apart for them to be in the
    same cluster. Defaults to 0.3. Note: If the data are normalised
    (they should be for DBSCAN) this is in terms of total sample
    variance.  Normalised data have a mean of 0 and a variance of 1.
min_samples : int
    The minimum number of samples within distance `eps` required
    to be considered as an independent cluster.
n_clusters : int
    The number of clusters expected. If specified, `eps` will be
    incrementally reduced until the expected number of clusters is
    found.
maxiter : int
    The maximum number of iterations DBSCAN will run.

Returns
-------
dict
    boolean array for each identified cluster and core samples.


def __init__(self, analytes, sort_by=<NUM_LIT:0>):

if isinstance(analytes, str):<EOL><INDENT>self.analytes = [analytes]<EOL><DEDENT>else:<EOL><INDENT>self.analytes = analytes<EOL><DEDENT>self.sort_by = sort_by<EOL>return<EOL>

Object to fit then apply a classifier.

Parameters
----------
analytes : str or array-like
    The analytes used by the clustring algorithm

Returns
-------
classifier object


def format_data(self, data, scale=True):

if len(self.analytes) == <NUM_LIT:1>:<EOL><INDENT>d = nominal_values(data[self.analytes[<NUM_LIT:0>]])<EOL>ds = np.array(list(zip(d, np.zeros(len(d)))))<EOL><DEDENT>else:<EOL><INDENT>d = [nominal_values(data[a]) for a in self.analytes]<EOL>ds = np.vstack(d).T<EOL><DEDENT>finite = np.isfinite(ds).sum(<NUM_LIT:1>) == ds.shape[<NUM_LIT:1>]<EOL>sampled = np.arange(data[self.analytes[<NUM_LIT:0>]].size)[finite]<EOL>ds = ds[finite]<EOL>if scale:<EOL><INDENT>ds = self.scaler.transform(ds)<EOL><DEDENT>return ds, sampled<EOL>

Function for converting a dict to an array suitable for sklearn.

Parameters
----------
data : dict
    A dict of data, containing all elements of
    `analytes` as items.
scale : bool
    Whether or not to scale the data. Should always be
    `True`, unless used by `classifier.fitting_data`
    where a scaler hasn't been created yet.

Returns
-------
A data array suitable for use with `sklearn.cluster`.


ds_fit, _ = self.format_data(data, scale=False)<EOL>self.scaler = preprocessing.StandardScaler().fit(ds_fit)<EOL>return self.scaler.transform(ds_fit)<EOL>

Function to format data for cluster fitting.

Parameters
----------
data : dict
    A dict of data, containing all elements of
    `analytes` as items.

Returns
-------
A data array for initial cluster fitting.


def fit_kmeans(self, data, n_clusters, **kwargs):

km = cl.KMeans(n_clusters=n_clusters, **kwargs)<EOL>km.fit(data)<EOL>return km<EOL>

Fit KMeans clustering algorithm to data.

Parameters
----------
data : array-like
    A dataset formatted by `classifier.fitting_data`.
n_clusters : int
    The number of clusters in the data.
**kwargs
    passed to `sklearn.cluster.KMeans`.

Returns
-------
Fitted `sklearn.cluster.KMeans` object.


def fit_meanshift(self, data, bandwidth=None, bin_seeding=False, **kwargs):

if bandwidth is None:<EOL><INDENT>bandwidth = cl.estimate_bandwidth(data)<EOL><DEDENT>ms = cl.MeanShift(bandwidth=bandwidth, bin_seeding=bin_seeding)<EOL>ms.fit(data)<EOL>return ms<EOL>

Fit MeanShift clustering algorithm to data.

Parameters
----------
data : array-like
    A dataset formatted by `classifier.fitting_data`.
bandwidth : float
    The bandwidth value used during clustering.
    If none, determined automatically. Note:
    the data are scaled before clutering, so
    this is not in the same units as the data.
bin_seeding : bool
    Whether or not to use 'bin_seeding'. See
    documentation for `sklearn.cluster.MeanShift`.
**kwargs
    passed to `sklearn.cluster.MeanShift`.

Returns
-------
Fitted `sklearn.cluster.MeanShift` object.


def fit(self, data, method='<STR_LIT>', **kwargs):

self.method = method<EOL>ds_fit = self.fitting_data(data)<EOL>mdict = {'<STR_LIT>': self.fit_kmeans,<EOL>'<STR_LIT>': self.fit_meanshift}<EOL>clust = mdict[method]<EOL>self.classifier = clust(data=ds_fit, **kwargs)<EOL>c0 = self.classifier.cluster_centers_.T[self.sort_by]<EOL>self.classifier.cluster_centers_ = self.classifier.cluster_centers_[np.argsort(c0)]<EOL>self.classifier.labels_ = self.classifier.predict(ds_fit)<EOL>self.classifier.ulabels_ = np.unique(self.classifier.labels_)<EOL>return<EOL>

fit classifiers from large dataset.

Parameters
----------
data : dict
    A dict of data for clustering. Must contain
    items with the same name as analytes used for
    clustering.
method : str
    A string defining the clustering method used. Can be:

    * 'kmeans' : K-Means clustering algorithm
    * 'meanshift' : Meanshift algorithm

n_clusters : int
    *K-Means only*. The numebr of clusters to identify
bandwidth : float
    *Meanshift only.*
    The bandwidth value used during clustering.
    If none, determined automatically. Note:
    the data are scaled before clutering, so
    this is not in the same units as the data.
bin_seeding : bool
    *Meanshift only.*
    Whether or not to use 'bin_seeding'. See
    documentation for `sklearn.cluster.MeanShift`.
**kwargs :
    passed to `sklearn.cluster.MeanShift`.

Returns
-------
list


size = data[self.analytes[<NUM_LIT:0>]].size<EOL>ds, sampled = self.format_data(data)<EOL>cs = self.classifier.predict(ds)<EOL>clusters = self.map_clusters(size, sampled, cs)<EOL>return clusters<EOL>

Label new data with cluster identities.

Parameters
----------
data : dict
    A data dict containing the same analytes used to
    fit the classifier.
sort_by : str
    The name of an analyte used to sort the resulting
    clusters. If None, defaults to the first analyte
    used in fitting.

Returns
-------
array of clusters the same length as the data.


def map_clusters(self, size, sampled, clusters):

ids = np.zeros(size, dtype=int)<EOL>ids[:] = -<NUM_LIT:2><EOL>ids[sampled] = clusters<EOL>return ids<EOL>

Translate cluster identity back to original data size.

Parameters
----------
size : int
    size of original dataset
sampled : array-like
    integer array describing location of finite values
    in original data.
clusters : array-like
    integer array of cluster identities

Returns
-------
list of cluster identities the same length as original
data. Where original data are non-finite, returns -2.


def sort_clusters(self, data, cs, sort_by):

<EOL>sdat = data[sort_by]<EOL>means = []<EOL>nclusts = np.arange(cs.max() + <NUM_LIT:1>)<EOL>for c in nclusts:<EOL><INDENT>means.append(np.nanmean(sdat[cs == c]))<EOL><DEDENT>means = np.array(means)<EOL>rank = np.zeros(means.size)<EOL>rank[np.argsort(means)] = np.arange(means.size)<EOL>csn = cs.copy()<EOL>for c, o in zip(nclusts, rank):<EOL><INDENT>csn[cs == c] = o<EOL><DEDENT>return csn<EOL>

Sort clusters by the concentration of a particular analyte.

Parameters
----------
data : dict
    A dataset containing sort_by as a key.
cs : array-like
    An array of clusters, the same length as values of data.
sort_by : str
    analyte to sort the clusters by

Returns
-------
array of clusters, sorted by mean value of sort_by analyte.


<EOL>ind = ~np.apply_along_axis(any, <NUM_LIT:1>, np.isnan(d))<EOL>if any(~ind):<EOL><INDENT>pcs = np.full((d.shape[<NUM_LIT:0>], nc), np.nan)<EOL>d = d[ind, :]<EOL><DEDENT>pca = PCA(nc).fit(d)<EOL>if any(~ind):<EOL><INDENT>pcs[ind, :] = pca.transform(d)<EOL><DEDENT>else:<EOL><INDENT>pcs = pca.transform(d)<EOL><DEDENT>return pca, pcs<EOL>

Calculates pca of d.

Parameters
----------
nc : int
    Number of components
d : np.ndarray
    An NxM array, containing M observations of N variables.
    Data must be floats. Can contain NaN values.

Returns
-------
pca, dt : tuple
    fitted PCA object, and transformed d (same size as d).


def pca_plot(pca, dt, xlabs=None, mode='<STR_LIT>', lognorm=True):

nc = pca.n_components<EOL>f = np.arange(pca.n_features_)<EOL>cs = list(itertools.combinations(range(nc), <NUM_LIT:2>))<EOL>ind = ~np.apply_along_axis(any, <NUM_LIT:1>, np.isnan(dt))<EOL>cylim = (pca.components_.min(), pca.components_.max())<EOL>yd = cylim[<NUM_LIT:1>] - cylim[<NUM_LIT:0>]<EOL>fig, axs = plt.subplots(nc, nc, figsize=[<NUM_LIT:3> * nc, nc * <NUM_LIT:3>], tight_layout=True)<EOL>for x, y in zip(*np.triu_indices(nc)):<EOL><INDENT>if x == y:<EOL><INDENT>tax = axs[x, y]<EOL>tax.bar(f, pca.components_[x], <NUM_LIT>)<EOL>tax.set_xticks([])<EOL>tax.axhline(<NUM_LIT:0>, zorder=-<NUM_LIT:1>, c=(<NUM_LIT:0>,<NUM_LIT:0>,<NUM_LIT:0>,<NUM_LIT>))<EOL>tax.set_ylim(cylim[<NUM_LIT:0>] - <NUM_LIT> * yd,<EOL>cylim[<NUM_LIT:1>] + <NUM_LIT> * yd)<EOL>for xi, yi, lab in zip(f, pca.components_[x], xlabs):<EOL><INDENT>if yi > <NUM_LIT:0>:<EOL><INDENT>yo = yd * <NUM_LIT><EOL>va = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>yo = yd * -<NUM_LIT><EOL>va = '<STR_LIT>'<EOL><DEDENT>tax.text(xi, yi + yo, lab, ha='<STR_LIT>', va=va, rotation=<NUM_LIT>, fontsize=<NUM_LIT:8>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>xv = dt[ind, x]<EOL>yv = dt[ind, y]<EOL>if mode == '<STR_LIT>':<EOL><INDENT>axs[x, y].scatter(xv, yv, alpha=<NUM_LIT>)<EOL>axs[y, x].scatter(yv, xv, alpha=<NUM_LIT>)<EOL><DEDENT>if mode == '<STR_LIT>':<EOL><INDENT>if lognorm:<EOL><INDENT>norm = mpl.colors.LogNorm()<EOL><DEDENT>else:<EOL><INDENT>norm = None<EOL><DEDENT>axs[x, y].hist2d(xv, yv, <NUM_LIT:50>, cmap=plt.cm.Blues, norm=norm)<EOL>axs[y, x].hist2d(yv, xv, <NUM_LIT:50>, cmap=plt.cm.Blues, norm=norm)<EOL><DEDENT><DEDENT>if x == <NUM_LIT:0>:<EOL><INDENT>axs[y, x].set_ylabel('<STR_LIT>'.format(y + <NUM_LIT:1>))<EOL><DEDENT>if y == nc - <NUM_LIT:1>:<EOL><INDENT>axs[y, x].set_xlabel('<STR_LIT>'.format(x + <NUM_LIT:1>))<EOL><DEDENT><DEDENT>return fig, axs, xv, yv<EOL>


def read_data(data_file, dataformat, name_mode):

with open(data_file) as f:<EOL><INDENT>lines = f.readlines()<EOL><DEDENT>if '<STR_LIT>' in dataformat.keys():<EOL><INDENT>meta = Bunch()<EOL>for k, v in dataformat['<STR_LIT>'].items():<EOL><INDENT>try:<EOL><INDENT>out = re.search(v[-<NUM_LIT:1>], lines[int(k)]).groups()<EOL><DEDENT>except:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(v[-<NUM_LIT:1>], lines[int(k)]))<EOL><DEDENT>for i in np.arange(len(v[<NUM_LIT:0>])):<EOL><INDENT>meta[v[<NUM_LIT:0>][i]] = out[i]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>meta = {}<EOL><DEDENT>if name_mode == '<STR_LIT>':<EOL><INDENT>sample = os.path.basename(data_file).split('<STR_LIT:.>')[<NUM_LIT:0>]<EOL><DEDENT>elif name_mode == '<STR_LIT>':<EOL><INDENT>sample = meta['<STR_LIT:name>']<EOL><DEDENT>else:<EOL><INDENT>sample = name_mode<EOL><DEDENT>columns = np.array(lines[dataformat['<STR_LIT>']['<STR_LIT>']].strip().split(<EOL>dataformat['<STR_LIT>']['<STR_LIT>']))<EOL>if '<STR_LIT>' in dataformat['<STR_LIT>'].keys():<EOL><INDENT>pr = re.compile(dataformat['<STR_LIT>']['<STR_LIT>'])<EOL>analytes = [pr.match(c).groups()[<NUM_LIT:0>] for c in columns if pr.match(c)]<EOL><DEDENT>if '<STR_LIT>' in dataformat.keys():<EOL><INDENT>with open(data_file) as f:<EOL><INDENT>fbuffer = f.read()<EOL><DEDENT>for k, v in dataformat['<STR_LIT>'].items():<EOL><INDENT>fbuffer = re.sub(k, v, fbuffer)<EOL><DEDENT>read_data = np.genfromtxt(BytesIO(fbuffer.encode()),<EOL>**dataformat['<STR_LIT>']).T<EOL><DEDENT>else:<EOL><INDENT>read_data = np.genfromtxt(data_file,<EOL>**dataformat['<STR_LIT>']).T<EOL><DEDENT>dind = np.zeros(read_data.shape[<NUM_LIT:0>], dtype=bool)<EOL>for a in analytes:<EOL><INDENT>dind[columns == a] = True<EOL><DEDENT>data = Bunch()<EOL>data['<STR_LIT>'] = read_data[dataformat['<STR_LIT>']['<STR_LIT>']]<EOL>if '<STR_LIT>' in dataformat['<STR_LIT>']:<EOL><INDENT>if isinstance(dataformat['<STR_LIT>']['<STR_LIT>'], (float, int)):<EOL><INDENT>time_mult = dataformat['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>elif isinstance(dataformat['<STR_LIT>']['<STR_LIT>'], str):<EOL><INDENT>unit_multipliers = {'<STR_LIT>': <NUM_LIT:1>/<NUM_LIT:1000>,<EOL>'<STR_LIT>': <NUM_LIT>/<NUM_LIT:1>,<EOL>'<STR_LIT:s>': <NUM_LIT:1>}<EOL>try:<EOL><INDENT>time_mult = unit_multipliers[dataformat['<STR_LIT>']['<STR_LIT>']]<EOL><DEDENT>except:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT>data['<STR_LIT>'] *= time_mult<EOL><DEDENT>data['<STR_LIT>'] = Bunch(zip(analytes, read_data[dind]))<EOL>data['<STR_LIT>'] = np.nansum(read_data[dind], <NUM_LIT:0>)<EOL>return sample, analytes, data, meta<EOL>

Load data_file described by a dataformat dict.

Parameters
----------
data_file : str
    Path to data file, including extension.
dataformat : dict
    A dataformat dict, see example below.
name_mode : str
    How to identyfy sample names. If 'file_names' uses the
    input name of the file, stripped of the extension. If
    'metadata_names' uses the 'name' attribute of the 'meta'
    sub-dictionary in dataformat. If any other str, uses this
    str as the sample name.

Example
-------
>>>
{'genfromtext_args': {'delimiter': ',',
                      'skip_header': 4},  # passed directly to np.genfromtxt
 'column_id': {'name_row': 3,  # which row contains the column names
               'delimiter': ',',  # delimeter between column names
               'timecolumn': 0,  # which column contains the 'time' variable
               'pattern': '([A-z]{1,2}[0-9]{1,3})'},  # a regex pattern which captures the column names
 'meta_regex': {  # a dict of (line_no: ([descriptors], [regexs])) pairs
                0: (['path'], '(.*)'),
                2: (['date', 'method'],  # MUST include date
                 '([A-Z][a-z]+ [0-9]+ [0-9]{4}[ ]+[0-9:]+ [amp]+).* ([A-z0-9]+\.m)')
               }
}

Returns
-------
sample, analytes, data, meta : tuple


def autorange(t, sig, gwin=<NUM_LIT:7>, swin=None, win=<NUM_LIT:30>,<EOL>on_mult=(<NUM_LIT>, <NUM_LIT:1.>), off_mult=(<NUM_LIT:1.>, <NUM_LIT>),<EOL>nbin=<NUM_LIT:10>, transform='<STR_LIT>', thresh=None):

failed = []<EOL>if swin is not None:<EOL><INDENT>sigs = fastsmooth(sig, swin)<EOL><DEDENT>else:<EOL><INDENT>sigs = sig<EOL><DEDENT>if transform == '<STR_LIT>':<EOL><INDENT>tsigs = np.log10(sigs)<EOL><DEDENT>else:<EOL><INDENT>tsigs = sigs<EOL><DEDENT>if thresh is None:<EOL><INDENT>bins = <NUM_LIT:50><EOL>kde_x = np.linspace(tsigs.min(), tsigs.max(), bins)<EOL>kde = gaussian_kde(tsigs)<EOL>yd = kde.pdf(kde_x)<EOL>mins = findmins(kde_x, yd)  <EOL>if len(mins) > <NUM_LIT:0>:<EOL><INDENT>bkg = tsigs < (mins[<NUM_LIT:0>])  <EOL><DEDENT>else:<EOL><INDENT>bkg = np.ones(tsigs.size, dtype=bool)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>bkg = tsigs < thresh<EOL><DEDENT>fbkg = bkg<EOL>fsig = ~bkg<EOL>zeros = bool_2_indices(fsig)<EOL>g = abs(fastgrad(sigs, gwin))  <EOL>if zeros is not None:<EOL><INDENT>zeros = zeros.flatten()<EOL>for z in zeros:  <EOL><INDENT>if z - win < <NUM_LIT:0>:<EOL><INDENT>lo = gwin // <NUM_LIT:2><EOL>hi = int(z + win)<EOL><DEDENT>elif z + win > (len(sig) - gwin // <NUM_LIT:2>):<EOL><INDENT>lo = int(z - win)<EOL>hi = len(sig) - gwin // <NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>lo = int(z - win)<EOL>hi = int(z + win)<EOL><DEDENT>xs = t[lo:hi]<EOL>ys = g[lo:hi]<EOL>mid = (hi + lo) // <NUM_LIT:2><EOL>tp = sigs[mid + <NUM_LIT:3>] > sigs[mid - <NUM_LIT:3>]  <EOL>c = t[z]  <EOL>width = (t[<NUM_LIT:1>] - t[<NUM_LIT:0>]) * <NUM_LIT:2><EOL>try:<EOL><INDENT>pg, _ = curve_fit(gauss, xs, ys,<EOL>p0=(np.nanmax(ys),<EOL>c,<EOL>width),<EOL>sigma=(xs - c)**<NUM_LIT:2> + <NUM_LIT>)<EOL>fwhm = abs(<NUM_LIT:2> * pg[-<NUM_LIT:1>] * np.sqrt(<NUM_LIT:2> * np.log(<NUM_LIT:2>)))<EOL>if tp:<EOL><INDENT>lim = np.array([-fwhm, fwhm]) * on_mult + pg[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>lim = np.array([-fwhm, fwhm]) * off_mult + pg[<NUM_LIT:1>]<EOL><DEDENT>fbkg[(t > lim[<NUM_LIT:0>]) & (t < lim[<NUM_LIT:1>])] = False<EOL>fsig[(t > lim[<NUM_LIT:0>]) & (t < lim[<NUM_LIT:1>])] = False<EOL><DEDENT>except RuntimeError:<EOL><INDENT>failed.append([c, tp])<EOL>pass<EOL><DEDENT><DEDENT><DEDENT>ftrn = ~fbkg & ~fsig<EOL>if len(failed) > <NUM_LIT:0>:<EOL><INDENT>trns = t[bool_2_indices(ftrn)]<EOL>tr_mean = (trns[:, <NUM_LIT:1>] - trns[:, <NUM_LIT:0>]).mean() / <NUM_LIT:2><EOL>for f, tp in failed:<EOL><INDENT>if tp:<EOL><INDENT>ind = (t >= f - tr_mean *<EOL>on_mult[<NUM_LIT:0>]) & (t <= f + tr_mean * on_mult[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>ind = (t >= f - tr_mean *<EOL>off_mult[<NUM_LIT:0>]) & (t <= f + tr_mean * off_mult[<NUM_LIT:0>])<EOL><DEDENT>fsig[ind] = False<EOL>fbkg[ind] = False<EOL>ftrn[ind] = False<EOL><DEDENT><DEDENT>return fbkg, fsig, ftrn, [f[<NUM_LIT:0>] for f in failed]<EOL>

Automatically separates signal and background in an on/off data stream.

**Step 1: Thresholding.**
The background signal is determined using a gaussian kernel density
estimator (kde) of all the data. Under normal circumstances, this
kde should find two distinct data distributions, corresponding to
'signal' and 'background'. The minima between these two distributions
is taken as a rough threshold to identify signal and background
regions. Any point where the trace crosses this thrshold is identified
as a 'transition'.

**Step 2: Transition Removal.**
The width of the transition regions between signal and background are
then determined, and the transitions are excluded from analysis. The
width of the transitions is determined by fitting a gaussian to the
smoothed first derivative of the analyte trace, and determining its
width at a point where the gaussian intensity is at at `conf` time the
gaussian maximum. These gaussians are fit to subsets of the data
centered around the transitions regions determined in Step 1, +/- `win`
data points. The peak is further isolated by finding the minima and
maxima of a second derivative within this window, and the gaussian is
fit to the isolated peak.

Parameters
----------
t : array-like
    Independent variable (usually time).
sig : array-like
    Dependent signal, with distinctive 'on' and 'off' regions.
gwin : int
    The window used for calculating first derivative.
    Defaults to 7.
swin : int
    The window used for signal smoothing. If None, ``gwin // 2``.
win : int
    The width (c +/- win) of the transition data subsets.
    Defaults to 20.
on_mult and off_mult : tuple, len=2
    Control the width of the excluded transition regions, which is defined
    relative to the peak full-width-half-maximum (FWHM) of the transition
    gradient. The region n * FHWM below the transition, and m * FWHM above
    the tranision will be excluded, where (n, m) are specified in `on_mult`
    and `off_mult`.
    `on_mult` and `off_mult` apply to the off-on and on-off transitions,
    respectively.
    Defaults to (1.5, 1) and (1, 1.5).
transform : str
    How to transform the data. Default is 'log'.

Returns
-------
fbkg, fsig, ftrn, failed : tuple
    where fbkg, fsig and ftrn are boolean arrays the same length as sig,
    that are True where sig is background, signal and transition, respecively.
    failed contains a list of transition positions where gaussian fitting
    has failed.


def autorange_components(t, sig, transform='<STR_LIT>', gwin=<NUM_LIT:7>, swin=None,<EOL>win=<NUM_LIT:30>, on_mult=(<NUM_LIT>, <NUM_LIT:1.>), off_mult=(<NUM_LIT:1.>, <NUM_LIT>),<EOL>thresh=None):

failed = []<EOL>if swin is not None:<EOL><INDENT>sigs = fastsmooth(sig, swin)<EOL><DEDENT>else:<EOL><INDENT>sigs = sig<EOL><DEDENT>if transform == '<STR_LIT>':<EOL><INDENT>tsigs = np.log10(sigs)<EOL>tsig = np.log10(sig)<EOL><DEDENT>else:<EOL><INDENT>tsigs = sigs<EOL>tsig = sig<EOL><DEDENT>if thresh is None:<EOL><INDENT>bins = <NUM_LIT:50><EOL>kde_x = np.linspace(tsigs.min(), tsigs.max(), bins)<EOL>kde = gaussian_kde(tsigs)<EOL>yd = kde.pdf(kde_x)<EOL>mins = findmins(kde_x, yd)  <EOL>if len(mins) > <NUM_LIT:0>:<EOL><INDENT>bkg = tsigs < (mins[<NUM_LIT:0>])  <EOL>thresh = mins[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>bkg = np.ones(tsigs.size, dtype=bool)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>bkg = tsigs < thresh<EOL><DEDENT>fbkg = bkg<EOL>fsig = ~bkg<EOL>zeros = bool_2_indices(fsig)<EOL>g = abs(fastgrad(sigs, gwin))  <EOL>if zeros is not None:<EOL><INDENT>zeros = zeros.flatten()<EOL>trans = dict(zeros=zeros.flatten(),<EOL>lohi=[],<EOL>pgs=[],<EOL>excl=[],<EOL>tps=[],<EOL>failed=[],<EOL>xs=[],<EOL>ys=[])<EOL>for z in zeros:  <EOL><INDENT>if z - win < <NUM_LIT:0>:<EOL><INDENT>lo = gwin // <NUM_LIT:2><EOL>hi = int(z + win)<EOL><DEDENT>elif z + win > (len(sig) - gwin // <NUM_LIT:2>):<EOL><INDENT>lo = int(z - win)<EOL>hi = len(sig) - gwin // <NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>lo = int(z - win)<EOL>hi = int(z + win)<EOL><DEDENT>xs = t[lo:hi]<EOL>ys = g[lo:hi]<EOL>trans['<STR_LIT>'].append(xs)<EOL>trans['<STR_LIT>'].append(ys)<EOL>trans['<STR_LIT>'].append([lo, hi])<EOL>mid = (hi + lo) // <NUM_LIT:2><EOL>tp = sigs[mid + <NUM_LIT:3>] > sigs[mid - <NUM_LIT:3>]  <EOL>trans['<STR_LIT>'].append(tp)<EOL>c = t[z]  <EOL>width = (t[<NUM_LIT:1>] - t[<NUM_LIT:0>]) * <NUM_LIT:2>  <EOL>try:<EOL><INDENT>pg, _ = curve_fit(gauss, xs, ys,<EOL>p0=(np.nanmax(ys),<EOL>c,<EOL>width),<EOL>sigma=(xs - c)**<NUM_LIT:2> + <NUM_LIT>)<EOL>trans['<STR_LIT>'].append(pg)<EOL>fwhm = abs(<NUM_LIT:2> * pg[-<NUM_LIT:1>] * np.sqrt(<NUM_LIT:2> * np.log(<NUM_LIT:2>)))<EOL>if tp:<EOL><INDENT>lim = np.array([-fwhm, fwhm]) * on_mult + pg[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>lim = np.array([-fwhm, fwhm]) * off_mult + pg[<NUM_LIT:1>]<EOL><DEDENT>trans['<STR_LIT>'].append(lim)<EOL>fbkg[(t > lim[<NUM_LIT:0>]) & (t < lim[<NUM_LIT:1>])] = False<EOL>fsig[(t > lim[<NUM_LIT:0>]) & (t < lim[<NUM_LIT:1>])] = False<EOL>failed.append(False)<EOL><DEDENT>except RuntimeError:<EOL><INDENT>failed.append(True)<EOL>trans['<STR_LIT>'].append([np.nan, np.nan])<EOL>trans['<STR_LIT>'].append([np.nan, np.nan, np.nan])<EOL>trans['<STR_LIT>'].append([np.nan, np.nan])<EOL>trans['<STR_LIT>'].append(tp)<EOL>pass<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>zeros = []<EOL><DEDENT>return t, sig, sigs, tsig, tsigs, kde_x, yd, g, trans, thresh<EOL>

Returns the components underlying the autorange algorithm.

Returns
-------
t : array-like
    Time axis (independent variable)
sig : array-like
    Raw signal (dependent variable)
sigs : array-like
    Smoothed signal (swin)
tsig : array-like
    Transformed raw signal (transform)
tsigs : array-like
    Transformed smoothed signal (transform, swin)
kde_x : array-like
    kernel density estimate of smoothed signal.
yd : array-like
    bins of kernel density estimator.
g : array-like
    gradient of smoothed signal (swin, gwin)
trans : dict
    per-transition data.
thresh : float
    threshold identified from kernel density plot


def noise_despike(sig, win=<NUM_LIT:3>, nlim=<NUM_LIT>, maxiter=<NUM_LIT:4>):

if win % <NUM_LIT:2> != <NUM_LIT:1>:<EOL><INDENT>win += <NUM_LIT:1>  <EOL><DEDENT>kernel = np.ones(win) / win  <EOL>over = np.ones(len(sig), dtype=bool)  <EOL>npad = int((win - <NUM_LIT:1>) / <NUM_LIT:2>)<EOL>over[:npad] = False<EOL>over[-npad:] = False<EOL>nloops = <NUM_LIT:0><EOL>while any(over) and (nloops < maxiter):<EOL><INDENT>rmean = np.convolve(sig, kernel, '<STR_LIT>')  <EOL>rstd = rmean**<NUM_LIT:0.5>  <EOL>over[npad:-npad] = (sig[npad:-npad] > rmean + nlim * rstd)<EOL>if any(over):<EOL><INDENT>sig[npad:-npad][over[npad:-npad]] = rmean[over[npad:-npad]]<EOL>nloops += <NUM_LIT:1><EOL><DEDENT><DEDENT>return sig<EOL>

Apply standard deviation filter to remove anomalous values.

Parameters
----------
win : int
    The window used to calculate rolling statistics.
nlim : float
    The number of standard deviations above the rolling
    mean above which data are considered outliers.

Returns
-------
None


def expdecay_despike(sig, expdecay_coef, tstep, maxiter=<NUM_LIT:3>):

<EOL>noise = np.std(sig[:<NUM_LIT:5>])  <EOL>for i in [<NUM_LIT:10>, <NUM_LIT:20>, <NUM_LIT:30>, <NUM_LIT:50>]:<EOL><INDENT>inoise = np.std(sig[:i])<EOL>if inoise < <NUM_LIT> * noise:<EOL><INDENT>noise = inoise<EOL><DEDENT><DEDENT>rms_noise3 = <NUM_LIT:3> * noise<EOL>i = <NUM_LIT:0><EOL>f = True<EOL>while (i < maxiter) and f:<EOL><INDENT>siglo = np.roll(sig * np.exp(tstep * expdecay_coef), <NUM_LIT:1>)<EOL>sighi = np.roll(sig * np.exp(-tstep * expdecay_coef), -<NUM_LIT:1>)<EOL>loind = (sig < siglo - rms_noise3) & (sig < np.roll(sig, -<NUM_LIT:1>) - rms_noise3)<EOL>hiind = (sig > sighi + rms_noise3) & (sig > np.roll(sig, <NUM_LIT:1>) + rms_noise3)<EOL>sig[loind] = sig[np.roll(loind, -<NUM_LIT:1>)]<EOL>sig[hiind] = sig[np.roll(hiind, -<NUM_LIT:1>)]<EOL>f = any(np.concatenate([loind, hiind]))<EOL>i += <NUM_LIT:1><EOL><DEDENT>return sig<EOL>

Apply exponential decay filter to remove physically impossible data based on instrumental washout.

The filter is re-applied until no more points are removed, or maxiter is reached.

Parameters
----------
exponent : float
    Exponent used in filter
tstep : float
    The time increment between data points.
maxiter : int
    The maximum number of times the filter should be applied.

Returns
-------
None


self.focus = self.data[focus]<EOL>self.focus_stage = focus<EOL>self.__dict__.update(self.focus)<EOL>

Set the 'focus' attribute of the data file.

The 'focus' attribute of the object points towards data from a
particular stage of analysis. It is used to identify the 'working
stage' of the data. Processing functions operate on the 'focus'
stage, so if steps are done out of sequence, things will break.

Names of analysis stages:

* 'rawdata': raw data, loaded from csv file when object
  is initialised.
* 'despiked': despiked data.
* 'signal'/'background': isolated signal and background data,
  padded with np.nan. Created by self.separate, after
  signal and background regions have been identified by
  self.autorange.
* 'bkgsub': background subtracted data, created by
  self.bkg_correct
* 'ratios': element ratio data, created by self.ratio.
* 'calibrated': ratio data calibrated to standards, created by
  self.calibrate.

Parameters
----------
focus : str
    The name of the analysis stage desired.

Returns
-------
None


@_log<EOL><INDENT>def despike(self, expdecay_despiker=True, exponent=None,<EOL>noise_despiker=True, win=<NUM_LIT:3>, nlim=<NUM_LIT>, maxiter=<NUM_LIT:3>):<DEDENT>

if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self.data['<STR_LIT>'] = Bunch()<EOL><DEDENT>out = {}<EOL>for a, v in self.focus.items():<EOL><INDENT>if '<STR_LIT:time>' not in a.lower():<EOL><INDENT>sig = v.copy()  <EOL>if expdecay_despiker:<EOL><INDENT>if exponent is not None:<EOL><INDENT>sig = proc.expdecay_despike(sig, exponent, self.tstep, maxiter)<EOL><DEDENT>else:<EOL><INDENT>warnings.warn('<STR_LIT>')<EOL><DEDENT><DEDENT>if noise_despiker:<EOL><INDENT>sig = proc.noise_despike(sig, int(win), nlim, maxiter)<EOL><DEDENT>out[a] = sig<EOL><DEDENT><DEDENT>self.data['<STR_LIT>'].update(out)<EOL>self.data['<STR_LIT>'] = sum(self.data['<STR_LIT>'].values())<EOL>self.setfocus('<STR_LIT>')<EOL>return<EOL>

Applies expdecay_despiker and noise_despiker to data.

Parameters
----------
expdecay_despiker : bool
    Whether or not to apply the exponential decay filter.
exponent : None or float
    The exponent for the exponential decay filter. If None,
    it is determined automatically using `find_expocoef`.
noise_despiker : bool
    Whether or not to apply the standard deviation spike filter.
win : int
    The rolling window over which the spike filter calculates
    the trace statistics.
nlim : float
    The number of standard deviations above the rolling mean
    that data are excluded.
maxiter : int
    The max number of times that the fitler is applied.

Returns
-------
None


@_log<EOL><INDENT>def autorange(self, analyte='<STR_LIT>', gwin=<NUM_LIT:5>, swin=<NUM_LIT:3>, win=<NUM_LIT:30>,<EOL>on_mult=[<NUM_LIT:1.>, <NUM_LIT:1.>], off_mult=[<NUM_LIT:1.>, <NUM_LIT>],<EOL>ploterrs=True, transform='<STR_LIT>', **kwargs):<DEDENT>

if analyte is None:<EOL><INDENT>sig = self.data['<STR_LIT>']<EOL><DEDENT>elif analyte == '<STR_LIT>':<EOL><INDENT>sig = self.data['<STR_LIT>']<EOL><DEDENT>elif analyte in self.analytes:<EOL><INDENT>sig = self.focus[analyte]<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>(self.bkg, self.sig,<EOL>self.trn, failed) = proc.autorange(self.Time, sig, gwin=gwin, swin=swin, win=win,<EOL>on_mult=on_mult, off_mult=off_mult,<EOL>transform=transform)<EOL>self.mkrngs()<EOL>errs_to_plot = False<EOL>if len(failed) > <NUM_LIT:0>:<EOL><INDENT>errs_to_plot = True<EOL>plotlines = []<EOL>for f in failed:<EOL><INDENT>if f != self.Time[-<NUM_LIT:1>]:<EOL><INDENT>plotlines.append(f)<EOL><DEDENT><DEDENT><DEDENT>if ploterrs and errs_to_plot and len(plotlines) > <NUM_LIT:0>:<EOL><INDENT>f, ax = self.tplot(ranges=True)<EOL>for pl in plotlines:<EOL><INDENT>ax.axvline(pl, c='<STR_LIT:r>', alpha=<NUM_LIT>, lw=<NUM_LIT:3>, ls='<STR_LIT>')<EOL><DEDENT>return f, plotlines<EOL><DEDENT>else:<EOL><INDENT>return<EOL><DEDENT>

Automatically separates signal and background data regions.

Automatically detect signal and background regions in the laser
data, based on the behaviour of a single analyte. The analyte used
should be abundant and homogenous in the sample.

**Step 1: Thresholding.**
The background signal is determined using a gaussian kernel density
estimator (kde) of all the data. Under normal circumstances, this
kde should find two distinct data distributions, corresponding to
'signal' and 'background'. The minima between these two distributions
is taken as a rough threshold to identify signal and background
regions. Any point where the trace crosses this thrshold is identified
as a 'transition'.

**Step 2: Transition Removal.**
The width of the transition regions between signal and background are
then determined, and the transitions are excluded from analysis. The
width of the transitions is determined by fitting a gaussian to the
smoothed first derivative of the analyte trace, and determining its
width at a point where the gaussian intensity is at at `conf` time the
gaussian maximum. These gaussians are fit to subsets of the data
centered around the transitions regions determined in Step 1, +/- `win`
data points. The peak is further isolated by finding the minima and
maxima of a second derivative within this window, and the gaussian is
fit to the isolated peak.

Parameters
----------
analyte : str
    The analyte that autorange should consider. For best results,
    choose an analyte that is present homogeneously in high
    concentrations.
gwin : int
    The smoothing window used for calculating the first derivative.
    Must be odd.
win : int
    Determines the width (c +/- win) of the transition data subsets.
on_mult and off_mult : tuple, len=2
    Factors to control the width of the excluded transition regions.
    A region n times the full - width - half - maximum of the transition
    gradient will be removed either side of the transition center.
    `on_mult` and `off_mult` refer to the laser - on and laser - off
    transitions, respectively. See manual for full explanation.
    Defaults to (1.5, 1) and (1, 1.5).

Returns
-------
Outputs added as instance attributes. Returns None.
bkg, sig, trn : iterable, bool
    Boolean arrays identifying background, signal and transision
    regions
bkgrng, sigrng and trnrng : iterable
    (min, max) pairs identifying the boundaries of contiguous
    True regions in the boolean arrays.


def autorange_plot(self, analyte='<STR_LIT>', gwin=<NUM_LIT:7>, swin=None, win=<NUM_LIT:20>,<EOL>on_mult=[<NUM_LIT>, <NUM_LIT:1.>], off_mult=[<NUM_LIT:1.>, <NUM_LIT>],<EOL>transform='<STR_LIT>'):

if analyte is None:<EOL><INDENT>sig = self.data['<STR_LIT>']<EOL><DEDENT>elif analyte == '<STR_LIT>':<EOL><INDENT>sig = self.data['<STR_LIT>']<EOL><DEDENT>elif analyte in self.analytes:<EOL><INDENT>sig = self.focus[analyte]<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if transform == '<STR_LIT>':<EOL><INDENT>sig = np.log10(sig)<EOL><DEDENT>fig, axs = plot.autorange_plot(t=self.Time, sig=sig, gwin=gwin,<EOL>swin=swin, win=win, on_mult=on_mult,<EOL>off_mult=off_mult)<EOL>return fig, axs<EOL>

Plot a detailed autorange report for this sample.


bbool = bool_2_indices(self.bkg)<EOL>if bbool is not None:<EOL><INDENT>self.bkgrng = self.Time[bbool]<EOL><DEDENT>else:<EOL><INDENT>self.bkgrng = [[np.nan, np.nan]]<EOL><DEDENT>sbool = bool_2_indices(self.sig)<EOL>if sbool is not None:<EOL><INDENT>self.sigrng = self.Time[sbool]<EOL><DEDENT>else:<EOL><INDENT>self.sigrng = [[np.nan, np.nan]]<EOL><DEDENT>tbool = bool_2_indices(self.trn)<EOL>if tbool is not None:<EOL><INDENT>self.trnrng = self.Time[tbool]<EOL><DEDENT>else:<EOL><INDENT>self.trnrng = [[np.nan, np.nan]]<EOL><DEDENT>self.ns = np.zeros(self.Time.size)<EOL>n = <NUM_LIT:1><EOL>for i in range(len(self.sig) - <NUM_LIT:1>):<EOL><INDENT>if self.sig[i]:<EOL><INDENT>self.ns[i] = n<EOL><DEDENT>if self.sig[i] and ~self.sig[i + <NUM_LIT:1>]:<EOL><INDENT>n += <NUM_LIT:1><EOL><DEDENT><DEDENT>self.n = int(max(self.ns))  <EOL>return<EOL>

Transform boolean arrays into list of limit pairs.

Gets Time limits of signal/background boolean arrays and stores them as
sigrng and bkgrng arrays. These arrays can be saved by 'save_ranges' in
the analyse object.


@_log<EOL><INDENT>def bkg_subtract(self, analyte, bkg, ind=None, focus_stage='<STR_LIT>'):<DEDENT>

if '<STR_LIT>' not in self.data.keys():<EOL><INDENT>self.data['<STR_LIT>'] = Bunch()<EOL><DEDENT>self.data['<STR_LIT>'][analyte] = self.data[focus_stage][analyte] - bkg<EOL>if ind is not None:<EOL><INDENT>self.data['<STR_LIT>'][analyte][ind] = np.nan<EOL><DEDENT>return<EOL>

Subtract provided background from signal (focus stage).

Results is saved in new 'bkgsub' focus stage

Returns
-------
None


@_log<EOL><INDENT>def correct_spectral_interference(self, target_analyte, source_analyte, f):<DEDENT>

if target_analyte not in self.analytes:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(target_analyte, '<STR_LIT:U+002CU+0020>'.join(self.analytes)))<EOL><DEDENT>if source_analyte not in self.analytes:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(source_analyte, '<STR_LIT:U+002CU+0020>'.join(self.analytes)))<EOL><DEDENT>self.data['<STR_LIT>'][target_analyte] -= self.data['<STR_LIT>'][source_analyte] * f<EOL>

Correct spectral interference.

Subtract interference counts from target_analyte, based on the
intensity of a source_analayte and a known fractional contribution (f).

Correction takes the form:
target_analyte -= source_analyte * f

Only operates on background-corrected data ('bkgsub').

To undo a correction,
rerun `self.bkg_subtract()`.

Parameters
----------
target_analyte : str
    The name of the analyte to modify.
source_analyte : str
    The name of the analyte to base the correction on.
f : float
    The fraction of the intensity of the source_analyte to
    subtract from the target_analyte. Correction is:
    target_analyte - source_analyte * f

Returns
-------
None


if internal_standard is not None:<EOL><INDENT>self.internal_standard = internal_standard<EOL><DEDENT>self.data['<STR_LIT>'] = Bunch()<EOL>for a in self.analytes:<EOL><INDENT>self.data['<STR_LIT>'][a] = (self.data['<STR_LIT>'][a] /<EOL>self.data['<STR_LIT>'][self.internal_standard])<EOL><DEDENT>self.setfocus('<STR_LIT>')<EOL>return<EOL>

Divide all analytes by a specified internal_standard analyte.

Parameters
----------
internal_standard : str
    The analyte used as the internal_standard.

Returns
-------
None


@_log<EOL><INDENT>def calibrate(self, calib_ps, analytes=None):<DEDENT>

<EOL>if analytes is None:<EOL><INDENT>analytes = self.analytes<EOL><DEDENT>if '<STR_LIT>' not in self.data.keys():<EOL><INDENT>self.data['<STR_LIT>'] = Bunch()<EOL><DEDENT>for a in analytes:<EOL><INDENT>m = calib_ps[a]['<STR_LIT:m>'].new(self.uTime)<EOL>if '<STR_LIT:c>' in calib_ps[a]:<EOL><INDENT>c = calib_ps[a]['<STR_LIT:c>'].new(self.uTime)<EOL><DEDENT>else:<EOL><INDENT>c = <NUM_LIT:0><EOL><DEDENT>self.data['<STR_LIT>'][a] = self.data['<STR_LIT>'][a] * m + c<EOL><DEDENT>if self.internal_standard not in analytes:<EOL><INDENT>self.data['<STR_LIT>'][self.internal_standard] =np.empty(len(self.data['<STR_LIT>'][self.internal_standard]))<EOL><DEDENT>self.setfocus('<STR_LIT>')<EOL>return<EOL>

Apply calibration to data.

The `calib_dict` must be calculated at the `analyse` level,
and passed to this calibrate function.

Parameters
----------
calib_dict : dict
    A dict of calibration values to apply to each analyte.

Returns
-------
None


@_log<EOL><INDENT>def sample_stats(self, analytes=None, filt=True,<EOL>stat_fns={},<EOL>eachtrace=True):<DEDENT>

if analytes is None:<EOL><INDENT>analytes = self.analytes<EOL><DEDENT>elif isinstance(analytes, str):<EOL><INDENT>analytes = [analytes]<EOL><DEDENT>self.stats = Bunch()<EOL>self.stats['<STR_LIT>'] = analytes<EOL>with warnings.catch_warnings():<EOL><INDENT>warnings.simplefilter("<STR_LIT:ignore>", category=RuntimeWarning)<EOL>for n, f in stat_fns.items():<EOL><INDENT>self.stats[n] = []<EOL>for a in analytes:<EOL><INDENT>ind = self.filt.grab_filt(filt, a)<EOL>dat = nominal_values(self.focus[a])<EOL>if eachtrace:<EOL><INDENT>sts = []<EOL>for t in np.arange(self.n) + <NUM_LIT:1>:<EOL><INDENT>sts.append(f(dat[ind & (self.ns == t)]))<EOL><DEDENT>self.stats[n].append(sts)<EOL><DEDENT>else:<EOL><INDENT>self.stats[n].append(f(dat[ind]))<EOL><DEDENT><DEDENT>self.stats[n] = np.array(self.stats[n])<EOL><DEDENT><DEDENT>return<EOL>

Calculate sample statistics

Returns samples, analytes, and arrays of statistics
of shape (samples, analytes). Statistics are calculated
from the 'focus' data variable, so output depends on how
the data have been processed.

Parameters
----------
analytes : array_like
    List of analytes to calculate the statistic on
filt : bool or str
    The filter to apply to the data when calculating sample statistics.
        bool: True applies filter specified in filt.switches.
        str: logical string specifying a partucular filter
stat_fns : dict
    Dict of {name: function} pairs. Functions that take a single
    array_like input, and return a single statistic. Function should
    be able to cope with NaN values.
eachtrace : bool
    True: per - ablation statistics
    False: whole sample statistics

Returns
-------
None


ats = {}<EOL>for n in np.arange(self.n) + <NUM_LIT:1>:<EOL><INDENT>t = self.Time[self.ns == n]<EOL>ats[n - <NUM_LIT:1>] = t.max() - t.min()<EOL><DEDENT>return ats<EOL>

Function for calculating the ablation time for each
ablation.

Returns
-------
    dict of times for each ablation.


@_log<EOL><INDENT>def filter_threshold(self, analyte, threshold):<DEDENT>

params = locals()<EOL>del(params['<STR_LIT>'])<EOL>below, above = filters.threshold(self.focus[analyte], threshold)<EOL>setn = self.filt.maxset + <NUM_LIT:1><EOL>self.filt.add(analyte + '<STR_LIT>',<EOL>below,<EOL>'<STR_LIT>'.format(threshold) + analyte,<EOL>params, setn=setn)<EOL>self.filt.add(analyte + '<STR_LIT>',<EOL>above,<EOL>'<STR_LIT>'.format(threshold) + analyte,<EOL>params, setn=setn)<EOL>

Apply threshold filter.

Generates threshold filters for the given analytes above and below
the specified threshold.

Two filters are created with prefixes '_above' and '_below'.
    '_above' keeps all the data above the threshold.
    '_below' keeps all the data below the threshold.

i.e. to select data below the threshold value, you should turn the
'_above' filter off.

Parameters
----------
analyte : TYPE
    Description of `analyte`.
threshold : TYPE
    Description of `threshold`.

Returns
-------
None


@_log<EOL><INDENT>def filter_gradient_threshold(self, analyte, win, threshold, recalc=True):<DEDENT>

params = locals()<EOL>del(params['<STR_LIT>'])<EOL>if recalc or not self.grads_calced:<EOL><INDENT>self.grads = calc_grads(self.Time, self.focus,<EOL>[analyte], win)<EOL>self.grads_calced = True<EOL><DEDENT>below, above = filters.threshold(abs(self.grads[analyte]), threshold)<EOL>setn = self.filt.maxset + <NUM_LIT:1><EOL>self.filt.add(analyte + '<STR_LIT>',<EOL>below,<EOL>'<STR_LIT>'.format(threshold) + analyte,<EOL>params, setn=setn)<EOL>self.filt.add(analyte + '<STR_LIT>',<EOL>above,<EOL>'<STR_LIT>'.format(threshold) + analyte,<EOL>params, setn=setn)<EOL>

Apply gradient threshold filter.

Generates threshold filters for the given analytes above and below
the specified threshold.

Two filters are created with prefixes '_above' and '_below'.
    '_above' keeps all the data above the threshold.
    '_below' keeps all the data below the threshold.

i.e. to select data below the threshold value, you should turn the
'_above' filter off.

Parameters
----------
analyte : str
    Description of `analyte`.
threshold : float
    Description of `threshold`.
win : int
    Window used to calculate gradients (n points)
recalc : bool
    Whether or not to re-calculate the gradients.

Returns
-------
None


@_log<EOL><INDENT>def filter_clustering(self, analytes, filt=False, normalise=True,<EOL>method='<STR_LIT>', include_time=False,<EOL>sort=None, min_data=<NUM_LIT:10>, **kwargs):<DEDENT>

params = locals()<EOL>del(params['<STR_LIT>'])<EOL>if isinstance(analytes, str):<EOL><INDENT>analytes = [analytes]<EOL><DEDENT>setn = self.filt.maxset + <NUM_LIT:1><EOL>vals = np.vstack(nominal_values(list(self.focus.values())))<EOL>if filt is not None:<EOL><INDENT>ind = (self.filt.grab_filt(filt, analytes) &<EOL>np.apply_along_axis(all, <NUM_LIT:0>, ~np.isnan(vals)))<EOL><DEDENT>else:<EOL><INDENT>ind = np.apply_along_axis(all, <NUM_LIT:0>, ~np.isnan(vals))<EOL><DEDENT>if sum(ind) > min_data:<EOL><INDENT>sampled = np.arange(self.Time.size)[ind]<EOL>if include_time:<EOL><INDENT>extra = self.Time<EOL><DEDENT>else:<EOL><INDENT>extra = None<EOL><DEDENT>ds = stack_keys(self.focus, analytes, extra)<EOL>ds = nominal_values(ds[ind, :])<EOL>if normalise | (len(analytes) > <NUM_LIT:1>):<EOL><INDENT>ds = preprocessing.scale(ds)<EOL><DEDENT>method_key = {'<STR_LIT>': clustering.cluster_kmeans,<EOL>'<STR_LIT>': clustering.cluster_meanshift}<EOL>cfun = method_key[method]<EOL>labels, core_samples_mask = cfun(ds, **kwargs)<EOL>labels_unique = np.unique(labels)<EOL>if (sort is not None) & (sort is not False):<EOL><INDENT>if isinstance(sort, str):<EOL><INDENT>sort = [sort]<EOL><DEDENT>sanalytes = analytes<EOL>if sort is True:<EOL><INDENT>sortk = np.array([True] * len(sanalytes))<EOL><DEDENT>else:<EOL><INDENT>sortk = np.array([s in sort for s in sanalytes])<EOL><DEDENT>sd = np.apply_along_axis(sum, <NUM_LIT:1>, ds[:, sortk])<EOL>avs = [np.nanmean(sd[labels == lab]) for lab in labels_unique]<EOL>order = [x[<NUM_LIT:0>] for x in sorted(enumerate(avs), key=lambda x:x[<NUM_LIT:1>])]<EOL>sdict = dict(zip(order, labels_unique))<EOL><DEDENT>else:<EOL><INDENT>sdict = dict(zip(labels_unique, labels_unique))<EOL><DEDENT>filts = {}<EOL>for ind, lab in sdict.items():<EOL><INDENT>filts[lab] = labels == ind<EOL><DEDENT>if not all(np.isnan(core_samples_mask)):<EOL><INDENT>filts['<STR_LIT>'] = core_samples_mask<EOL><DEDENT>resized = {}<EOL>for k, v in filts.items():<EOL><INDENT>resized[k] = np.zeros(self.Time.size, dtype=bool)<EOL>resized[k][sampled] = v<EOL><DEDENT>namebase = '<STR_LIT:->'.join(analytes) + '<STR_LIT>' + method<EOL>info = '<STR_LIT:->'.join(analytes) + '<STR_LIT>'<EOL>if method == '<STR_LIT>':<EOL><INDENT>for k, v in resized.items():<EOL><INDENT>if isinstance(k, str):<EOL><INDENT>name = namebase + '<STR_LIT>'<EOL><DEDENT>elif k < <NUM_LIT:0>:<EOL><INDENT>name = namebase + '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>name = namebase + '<STR_LIT>'.format(k)<EOL><DEDENT>self.filt.add(name, v, info=info, params=params, setn=setn)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for k, v in resized.items():<EOL><INDENT>name = namebase + '<STR_LIT>'.format(k)<EOL>self.filt.add(name, v, info=info, params=params, setn=setn)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>name = '<STR_LIT:->'.join(analytes) + '<STR_LIT>' + method + '<STR_LIT>'<EOL>info = '<STR_LIT:->'.join(analytes) + '<STR_LIT>'<EOL>self.filt.add(name, np.zeros(self.Time.size, dtype=bool),<EOL>info=info, params=params, setn=setn)<EOL><DEDENT>return<EOL>

Applies an n - dimensional clustering filter to the data.

Available Clustering Algorithms

* 'meanshift': The `sklearn.cluster.MeanShift` algorithm.
  Automatically determines number of clusters
  in data based on the `bandwidth` of expected
  variation.
* 'kmeans': The `sklearn.cluster.KMeans` algorithm. Determines
  the characteristics of a known number of clusters
  within the data. Must provide `n_clusters` to specify
  the expected number of clusters.
* 'DBSCAN': The `sklearn.cluster.DBSCAN` algorithm. Automatically
  determines the number and characteristics of clusters
  within the data based on the 'connectivity' of the
  data (i.e. how far apart each data point is in a
  multi - dimensional parameter space). Requires you to
  set `eps`, the minimum distance point must be from
  another point to be considered in the same cluster,
  and `min_samples`, the minimum number of points that
  must be within the minimum distance for it to be
  considered a cluster. It may also be run in automatic
  mode by specifying `n_clusters` alongside
  `min_samples`, where eps is decreased until the
  desired number of clusters is obtained.

For more information on these algorithms, refer to the
documentation.

Parameters
----------
analytes : str
    The analyte(s) that the filter applies to.
filt : bool
    Whether or not to apply existing filters to the data before
    calculating this filter.
normalise : bool
    Whether or not to normalise the data to zero mean and unit
    variance. Reccomended if clustering based on more than 1 analyte.
    Uses `sklearn.preprocessing.scale`.
method : str
    Which clustering algorithm to use (see above).
include_time : bool
    Whether or not to include the Time variable in the clustering
    analysis. Useful if you're looking for spatially continuous
    clusters in your data, i.e. this will identify each spot in your
    analysis as an individual cluster.
sort : bool, str or array-like
    Whether or not to label the resulting clusters according to their
    contents. If used, the cluster with the lowest values will be
    labelled from 0, in order of increasing cluster mean value.analytes.
    The sorting rules depend on the value of 'sort', which can be the name
    of a single analyte (str), a list of several analyte names (array-like)
    or True (bool), to specify all analytes used to calcualte the cluster.
min_data : int
    The minimum number of data points that should be considered by
    the filter. Default = 10.
**kwargs
    Parameters passed to the clustering algorithm specified by
    `method`.

Meanshift Parameters
--------------------
bandwidth : str or float
    The bandwith (float) or bandwidth method ('scott' or 'silverman')
    used to estimate the data bandwidth.
bin_seeding : bool
    Modifies the behaviour of the meanshift algorithm. Refer to
    sklearn.cluster.meanshift documentation.

K - Means Parameters
------------------
n_clusters : int
    The number of clusters expected in the data.

DBSCAN Parameters
-----------------
eps : float
    The minimum 'distance' points must be apart for them to be in the
    same cluster. Defaults to 0.3. Note: If the data are normalised
    (they should be for DBSCAN) this is in terms of total sample
    variance. Normalised data have a mean of 0 and a variance of 1.
min_samples : int
    The minimum number of samples within distance `eps` required
    to be considered as an independent cluster.
n_clusters : int
    The number of clusters expected. If specified, `eps` will be
    incrementally reduced until the expected number of clusters is
    found.
maxiter : int
    The maximum number of iterations DBSCAN will run.

Returns
-------
None


@_log<EOL><INDENT>def calc_correlation(self, x_analyte, y_analyte, window=<NUM_LIT:15>, filt=True, recalc=True):<DEDENT>

label = '<STR_LIT>'.format(x_analyte, y_analyte, window)<EOL>if label in self.correlations and not recalc:<EOL><INDENT>return<EOL><DEDENT>if window % <NUM_LIT:2> != <NUM_LIT:1>:<EOL><INDENT>window += <NUM_LIT:1><EOL><DEDENT>ind = self.filt.grab_filt(filt, [x_analyte, y_analyte])<EOL>x = nominal_values(self.focus[x_analyte])<EOL>x[~ind] = np.nan<EOL>xr = rolling_window(x, window, pad=np.nan)<EOL>y = nominal_values(self.focus[y_analyte])<EOL>y[~ind] = np.nan<EOL>yr = rolling_window(y, window, pad=np.nan)<EOL>r, p = zip(*map(nan_pearsonr, xr, yr))<EOL>r = np.array(r)<EOL>p = np.array(p)<EOL>self.correlations[label] = r, p<EOL>return<EOL>

Calculate local correlation between two analytes.

Parameters
----------
x_analyte, y_analyte : str
    The names of the x and y analytes to correlate.
window : int, None
    The rolling window used when calculating the correlation.
filt : bool
    Whether or not to apply existing filters to the data before
    calculating this filter.
recalc : bool
    If True, the correlation is re-calculated, even if it is already present.

Returns
-------
None


@_log<EOL><INDENT>def filter_correlation(self, x_analyte, y_analyte, window=<NUM_LIT:15>,<EOL>r_threshold=<NUM_LIT>, p_threshold=<NUM_LIT>, filt=True, recalc=False):<DEDENT>

<EOL>if window % <NUM_LIT:2> != <NUM_LIT:1>:<EOL><INDENT>window += <NUM_LIT:1><EOL><DEDENT>params = locals()<EOL>del(params['<STR_LIT>'])<EOL>setn = self.filt.maxset + <NUM_LIT:1><EOL>label = '<STR_LIT>'.format(x_analyte, y_analyte, window)<EOL>self.calc_correlation(x_analyte, y_analyte, window, filt, recalc)<EOL>r, p = self.correlations[label]<EOL>cfilt = (abs(r) > r_threshold) & (p < p_threshold)<EOL>cfilt = ~cfilt<EOL>name = x_analyte + '<STR_LIT:_>' + y_analyte + '<STR_LIT>'<EOL>self.filt.add(name=name,<EOL>filt=cfilt,<EOL>info=(x_analyte + '<STR_LIT>' + y_analyte +<EOL>'<STR_LIT>'),<EOL>params=params, setn=setn)<EOL>self.filt.off(filt=name)<EOL>self.filt.on(analyte=y_analyte, filt=name)<EOL>return<EOL>

Calculate correlation filter.

Parameters
----------
x_analyte, y_analyte : str
    The names of the x and y analytes to correlate.
window : int, None
    The rolling window used when calculating the correlation.
r_threshold : float
    The correlation index above which to exclude data.
    Note: the absolute pearson R value is considered, so
    negative correlations below -`r_threshold` will also
    be excluded.
p_threshold : float
    The significant level below which data are excluded.
filt : bool
    Whether or not to apply existing filters to the data before
    calculating this filter.
recalc : bool
    If True, the correlation is re-calculated, even if it is already present.

Returns
-------
None


@_log<EOL><INDENT>def correlation_plot(self, x_analyte, y_analyte, window=<NUM_LIT:15>, filt=True, recalc=False):<DEDENT>

label = '<STR_LIT>'.format(x_analyte, y_analyte, window)<EOL>self.calc_correlation(x_analyte, y_analyte, window, filt, recalc)<EOL>r, p = self.correlations[label]<EOL>fig, axs = plt.subplots(<NUM_LIT:3>, <NUM_LIT:1>, figsize=[<NUM_LIT:7>, <NUM_LIT:5>], sharex=True)<EOL>ax = axs[<NUM_LIT:0>]<EOL>ax.plot(self.Time, nominal_values(self.focus[x_analyte]), color=self.cmap[x_analyte], label=x_analyte)<EOL>ax.plot(self.Time, nominal_values(self.focus[y_analyte]), color=self.cmap[y_analyte], label=y_analyte)<EOL>ax.set_yscale('<STR_LIT>')<EOL>ax.legend()<EOL>ax.set_ylabel('<STR_LIT>')<EOL>ax = axs[<NUM_LIT:1>]<EOL>ax.plot(self.Time, r)<EOL>ax.set_ylabel('<STR_LIT>')<EOL>ax = axs[<NUM_LIT:2>]<EOL>ax.plot(self.Time, p)<EOL>ax.set_ylabel('<STR_LIT>')<EOL>fig.tight_layout()<EOL>return fig, axs<EOL>

Plot the local correlation between two analytes.

Parameters
----------
x_analyte, y_analyte : str
    The names of the x and y analytes to correlate.
window : int, None
    The rolling window used when calculating the correlation.
filt : bool
    Whether or not to apply existing filters to the data before
    calculating this filter.
recalc : bool
    If True, the correlation is re-calculated, even if it is already present.

Returns
-------
fig, axs : figure and axes objects


@_log<EOL><INDENT>def filter_new(self, name, filt_str):<DEDENT>

filt = self.filt.grab_filt(filt=filt_str)<EOL>self.filt.add(name, filt, info=filt_str)<EOL>return<EOL>

Make new filter from combination of other filters.

Parameters
----------
name : str
    The name of the new filter. Should be unique.
filt_str : str
    A logical combination of partial strings which will create
    the new filter. For example, 'Albelow & Mnbelow' will combine
    all filters that partially match 'Albelow' with those that
    partially match 'Mnbelow' using the 'AND' logical operator.

Returns
-------
None


@_log<EOL><INDENT>def filter_trim(self, start=<NUM_LIT:1>, end=<NUM_LIT:1>, filt=True):<DEDENT>

params = locals()<EOL>del(params['<STR_LIT>'])<EOL>f = self.filt.grab_filt(filt)<EOL>nf = filters.trim(f, start, end)<EOL>self.filt.add('<STR_LIT>',<EOL>nf,<EOL>'<STR_LIT>'.format(start, end),<EOL>params, setn=self.filt.maxset + <NUM_LIT:1>)<EOL>

Remove points from the start and end of filter regions.

Parameters
----------
start, end : int
    The number of points to remove from the start and end of
    the specified filter.
filt : valid filter string or bool
    Which filter to trim. If True, applies to currently active
    filters.


@_log<EOL><INDENT>def filter_exclude_downhole(self, threshold, filt=True):<DEDENT>

f = self.filt.grab_filt(filt)<EOL>if self.n == <NUM_LIT:1>:<EOL><INDENT>nfilt = filters.exclude_downhole(f, threshold)<EOL><DEDENT>else:<EOL><INDENT>nfilt = []<EOL>for i in range(self.n):<EOL><INDENT>nf = self.ns == i + <NUM_LIT:1><EOL>nfilt.append(filters.exclude_downhole(f & nf, threshold))<EOL><DEDENT>nfilt = np.apply_along_axis(any, <NUM_LIT:0>, nfilt)<EOL><DEDENT>self.filt.add(name='<STR_LIT>'.format(threshold),<EOL>filt=nfilt,<EOL>info='<STR_LIT>'.format(threshold),<EOL>params=(threshold, filt))<EOL>

Exclude all points down-hole (after) the first excluded data.

Parameters
----------
threhold : int
    The minimum number of contiguous excluded data points
    that must exist before downhole exclusion occurs.
file : valid filter string or bool
    Which filter to consider. If True, applies to currently active
    filters.


@_log<EOL><INDENT>def signal_optimiser(self, analytes, min_points=<NUM_LIT:5>,<EOL>threshold_mode='<STR_LIT>',<EOL>threshold_mult=<NUM_LIT:1.>, x_bias=<NUM_LIT:0>,<EOL>weights=None, filt=True, mode='<STR_LIT>'):<DEDENT>

params = locals()<EOL>del(params['<STR_LIT>'])<EOL>setn = self.filt.maxset + <NUM_LIT:1><EOL>if isinstance(analytes, str):<EOL><INDENT>analytes = [analytes]<EOL><DEDENT>if filt is not False:<EOL><INDENT>ind = (self.filt.grab_filt(filt, analytes))<EOL><DEDENT>else:<EOL><INDENT>ind = np.full(self.Time.shape, True)<EOL><DEDENT>errmsg = []<EOL>ofilt = []<EOL>self.opt = {}<EOL>for i in range(self.n):<EOL><INDENT>nind = ind & (self.ns == i + <NUM_LIT:1>)<EOL>self.opt[i + <NUM_LIT:1>], err = signal_optimiser(self, analytes=analytes,<EOL>min_points=min_points, <EOL>threshold_mode=threshold_mode,<EOL>threshold_mult=threshold_mult,<EOL>weights=weights,<EOL>ind=nind, x_bias=x_bias,<EOL>mode=mode)<EOL>if err == '<STR_LIT>':<EOL><INDENT>ofilt.append(self.opt[i + <NUM_LIT:1>].filt)<EOL><DEDENT>else:<EOL><INDENT>errmsg.append(self.sample + '<STR_LIT>'.format(i + <NUM_LIT:1>) + err)<EOL><DEDENT><DEDENT>if len(ofilt) > <NUM_LIT:0>:<EOL><INDENT>ofilt = np.apply_along_axis(any, <NUM_LIT:0>, ofilt)<EOL>name = '<STR_LIT>' + '<STR_LIT:_>'.join(analytes)<EOL>self.filt.add(name=name,<EOL>filt=ofilt,<EOL>info="<STR_LIT>" + '<STR_LIT:U+002CU+0020>'.join(analytes),<EOL>params=params, setn=setn)            <EOL><DEDENT>if len(errmsg) > <NUM_LIT:0>:<EOL><INDENT>return '<STR_LIT:\n>'.join(errmsg)<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>

Optimise data selection based on specified analytes.

Identifies the longest possible contiguous data region in
the signal where the relative standard deviation (std) and
concentration of all analytes is minimised.

Optimisation is performed via a grid search of all possible
contiguous data regions. For each region, the mean std and
mean scaled analyte concentration ('amplitude') are calculated.

The size and position of the optimal data region are identified
using threshold std and amplitude values. Thresholds are derived
from all calculated stds and amplitudes using the method specified
by `threshold_mode`. For example, using the 'kde_max' method, a
probability density function (PDF) is calculated for std and
amplitude values, and the threshold is set as the maximum of the
PDF. These thresholds are then used to identify the size and position
of the longest contiguous region where the std is below the threshold,
and the amplitude is either below the threshold.

All possible regions of the data that have at least
`min_points` are considered.

For a graphical demonstration of the action of signal_optimiser,
use `optimisation_plot`.

Parameters
----------
d : latools.D object
    An latools data object.
analytes : str or array-like
    Which analytes to consider.
min_points : int
    The minimum number of contiguous points to
    consider.
threshold_mode : str
    The method used to calculate the optimisation
    thresholds. Can be 'mean', 'median', 'kde_max'
    or 'bayes_mvs', or a custom function. If a
    function, must take a 1D array, and return a
    single, real number.
weights : array-like of length len(analytes)
    An array of numbers specifying the importance of
    each analyte considered. Larger number makes the
    analyte have a greater effect on the optimisation.
    Default is None.


@_log<EOL><INDENT>def optimisation_plot(self, overlay_alpha=<NUM_LIT:0.5>, **kwargs):<DEDENT>

return optimisation_plot(self, overlay_alpha=<NUM_LIT:0.5>, **kwargs)<EOL>

Plot the result of signal_optimise.

`signal_optimiser` must be run first, and the output
stored in the `opt` attribute of the latools.D object.

Parameters
----------
d : latools.D object
    A latools data object.
overlay_alpha : float
    The opacity of the threshold overlays. Between 0 and 1.
**kwargs
    Passed to `tplot`


@_log<EOL><INDENT>def tplot(self, analytes=None, figsize=[<NUM_LIT:10>, <NUM_LIT:4>], scale='<STR_LIT>', filt=None,<EOL>ranges=False, stats=False, stat='<STR_LIT>', err='<STR_LIT>',<EOL>focus_stage=None, err_envelope=False, ax=None):<DEDENT>

return plot.tplot(self=self, analytes=analytes, figsize=figsize, scale=scale, filt=filt,<EOL>ranges=ranges, stats=stats, stat=stat, err=err,<EOL>focus_stage=focus_stage, err_envelope=err_envelope, ax=ax)<EOL>

Plot analytes as a function of Time.

Parameters
----------
analytes : array_like
    list of strings containing names of analytes to plot.
    None = all analytes.
figsize : tuple
    size of final figure.
scale : str or None
   'log' = plot data on log scale
filt : bool, str or dict
    False: plot unfiltered data.
    True: plot filtered data over unfiltered data.
    str: apply filter key to all analytes
    dict: apply key to each analyte in dict. Must contain all
    analytes plotted. Can use self.filt.keydict.
ranges : bool
    show signal/background regions.
stats : bool
    plot average and error of each trace, as specified by `stat` and
    `err`.
stat : str
    average statistic to plot.
err : str
    error statistic to plot.

Returns
-------
figure, axis


@_log<EOL><INDENT>def gplot(self, analytes=None, win=<NUM_LIT:5>, figsize=[<NUM_LIT:10>, <NUM_LIT:4>],<EOL>ranges=False, focus_stage=None, ax=None):<DEDENT>

return plot.gplot(self=self, analytes=analytes, win=win, figsize=figsize,<EOL>ranges=ranges, focus_stage=focus_stage, ax=ax)<EOL>

Plot analytes gradients as a function of Time.

Parameters
----------
analytes : array_like
    list of strings containing names of analytes to plot.
    None = all analytes.
win : int
    The window over which to calculate the rolling gradient.
figsize : tuple
    size of final figure.
ranges : bool
    show signal/background regions.

Returns
-------
figure, axis


@_log<EOL><INDENT>def crossplot(self, analytes=None, bins=<NUM_LIT>, lognorm=True, filt=True, colourful=True, figsize=(<NUM_LIT:12>, <NUM_LIT:12>)):<DEDENT>

if analytes is None:<EOL><INDENT>analytes = self.analytes<EOL><DEDENT>if self.focus_stage in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>analytes = [a for a in analytes if self.internal_standard not in a]<EOL><DEDENT>if figsize[<NUM_LIT:0>] < <NUM_LIT> * len(analytes):<EOL><INDENT>figsize = [<NUM_LIT> * len(analytes)] * <NUM_LIT:2><EOL><DEDENT>numvars = len(analytes)<EOL>fig, axes = plt.subplots(nrows=numvars, ncols=numvars,<EOL>figsize=(<NUM_LIT:12>, <NUM_LIT:12>))<EOL>fig.subplots_adjust(hspace=<NUM_LIT>, wspace=<NUM_LIT>)<EOL>for ax in axes.flat:<EOL><INDENT>ax.xaxis.set_visible(False)<EOL>ax.yaxis.set_visible(False)<EOL>if ax.is_first_col():<EOL><INDENT>ax.yaxis.set_ticks_position('<STR_LIT:left>')<EOL><DEDENT>if ax.is_last_col():<EOL><INDENT>ax.yaxis.set_ticks_position('<STR_LIT:right>')<EOL><DEDENT>if ax.is_first_row():<EOL><INDENT>ax.xaxis.set_ticks_position('<STR_LIT>')<EOL><DEDENT>if ax.is_last_row():<EOL><INDENT>ax.xaxis.set_ticks_position('<STR_LIT>')<EOL><DEDENT><DEDENT>if colourful:<EOL><INDENT>cmlist = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>cmlist = ['<STR_LIT>']<EOL><DEDENT>while len(cmlist) < len(analytes):<EOL><INDENT>cmlist *= <NUM_LIT:2><EOL><DEDENT>udict = {}<EOL>for i, j in zip(*np.triu_indices_from(axes, k=<NUM_LIT:1>)):<EOL><INDENT>for x, y in [(i, j), (j, i)]:<EOL><INDENT>mx, ux = unitpicker(np.nanmean(self.focus[analytes[x]]),<EOL>denominator=self.internal_standard,<EOL>focus_stage=self.focus_stage)<EOL>my, uy = unitpicker(np.nanmean(self.focus[analytes[y]]),<EOL>denominator=self.internal_standard,<EOL>focus_stage=self.focus_stage)<EOL>udict[analytes[x]] = (x, ux)<EOL>xd = nominal_values(self.focus[analytes[x]])<EOL>yd = nominal_values(self.focus[analytes[y]])<EOL>ind = (self.filt.grab_filt(filt, analytes[x]) &<EOL>self.filt.grab_filt(filt, analytes[y]) &<EOL>~np.isnan(xd) &<EOL>~np.isnan(yd))<EOL>pi = xd[ind] * mx<EOL>pj = yd[ind] * my<EOL>if lognorm:<EOL><INDENT>norm = mpl.colors.LogNorm()<EOL><DEDENT>else:<EOL><INDENT>norm = None<EOL><DEDENT>axes[i, j].hist2d(pj, pi, bins,<EOL>norm=norm,<EOL>cmap=plt.get_cmap(cmlist[i]))<EOL>axes[j, i].hist2d(pi, pj, bins,<EOL>norm=norm,<EOL>cmap=plt.get_cmap(cmlist[j]))<EOL>axes[x, y].set_ylim([pi.min(), pi.max()])<EOL>axes[x, y].set_xlim([pj.min(), pj.max()])<EOL><DEDENT><DEDENT>for a, (i, u) in udict.items():<EOL><INDENT>axes[i, i].annotate(a + '<STR_LIT:\n>' + u, (<NUM_LIT:0.5>, <NUM_LIT:0.5>),<EOL>xycoords='<STR_LIT>',<EOL>ha='<STR_LIT>', va='<STR_LIT>')<EOL><DEDENT>for i, j in zip(range(numvars), itertools.cycle((-<NUM_LIT:1>, <NUM_LIT:0>))):<EOL><INDENT>axes[j, i].xaxis.set_visible(True)<EOL>for label in axes[j, i].get_xticklabels():<EOL><INDENT>label.set_rotation(<NUM_LIT>)<EOL><DEDENT>axes[i, j].yaxis.set_visible(True)<EOL><DEDENT>axes[<NUM_LIT:0>, <NUM_LIT:0>].set_title(self.sample, weight='<STR_LIT>', x=<NUM_LIT>, ha='<STR_LIT:left>')<EOL>return fig, axes<EOL>

Plot analytes against each other.

Parameters
----------
analytes : optional, array_like or str
    The analyte(s) to plot. Defaults to all analytes.
lognorm : bool
    Whether or not to log normalise the colour scale
    of the 2D histogram.
bins : int
    The number of bins in the 2D histogram.
filt : str, dict or bool
    Either logical filter expression contained in a str,
    a dict of expressions specifying the filter string to
    use for each analyte or a boolean. Passed to `grab_filt`.

Returns
-------
(fig, axes)


def crossplot_filters(self, filter_string, analytes=None):

if analytes is None:<EOL><INDENT>analytes = [a for a in self.analytes if '<STR_LIT>' not in a]<EOL><DEDENT>filts = self.filt.components.keys()<EOL>cfilts = [f for f in filts if filter_string in f]<EOL>flab = re.compile('<STR_LIT>')  <EOL>numvars = len(analytes)<EOL>fig, axes = plt.subplots(nrows=numvars, ncols=numvars,<EOL>figsize=(<NUM_LIT:12>, <NUM_LIT:12>))<EOL>fig.subplots_adjust(hspace=<NUM_LIT>, wspace=<NUM_LIT>)<EOL>for ax in axes.flat:<EOL><INDENT>ax.xaxis.set_visible(False)<EOL>ax.yaxis.set_visible(False)<EOL>if ax.is_first_col():<EOL><INDENT>ax.yaxis.set_ticks_position('<STR_LIT:left>')<EOL><DEDENT>if ax.is_last_col():<EOL><INDENT>ax.yaxis.set_ticks_position('<STR_LIT:right>')<EOL><DEDENT>if ax.is_first_row():<EOL><INDENT>ax.xaxis.set_ticks_position('<STR_LIT>')<EOL><DEDENT>if ax.is_last_row():<EOL><INDENT>ax.xaxis.set_ticks_position('<STR_LIT>')<EOL><DEDENT><DEDENT>focus = {k: nominal_values(v) for k, v in self.focus.items()}<EOL>udict = {a: unitpicker(np.nanmean(focus[a]),<EOL>denominator=self.internal_standard,<EOL>focus_stage=self.focus_stage) for a in analytes}<EOL>rdict = {a: (np.nanmin(focus[a] * udict[a][<NUM_LIT:0>]),<EOL>np.nanmax(focus[a] * udict[a][<NUM_LIT:0>])) for a in analytes}<EOL>for f in cfilts:<EOL><INDENT>ind = self.filt.grab_filt(f)<EOL>focus = {k: nominal_values(v[ind]) for k, v in self.focus.items()}<EOL>lab = flab.match(f).groups()[<NUM_LIT:0>]<EOL>axes[<NUM_LIT:0>, <NUM_LIT:0>].scatter([], [], s=<NUM_LIT:10>, label=lab)<EOL>for i, j in zip(*np.triu_indices_from(axes, k=<NUM_LIT:1>)):<EOL><INDENT>ai = analytes[i]<EOL>aj = analytes[j]<EOL>pi = focus[ai][~np.isnan(focus[ai])] * udict[ai][<NUM_LIT:0>]<EOL>pj = focus[aj][~np.isnan(focus[aj])] * udict[aj][<NUM_LIT:0>]<EOL>axes[i, j].scatter(pj, pi, alpha=<NUM_LIT>, s=<NUM_LIT:10>, lw=<NUM_LIT:0>)<EOL>axes[j, i].scatter(pi, pj, alpha=<NUM_LIT>, s=<NUM_LIT:10>, lw=<NUM_LIT:0>)<EOL>axes[i, j].set_ylim(*rdict[ai])<EOL>axes[i, j].set_xlim(*rdict[aj])<EOL>axes[j, i].set_ylim(*rdict[aj])<EOL>axes[j, i].set_xlim(*rdict[ai])<EOL><DEDENT><DEDENT>for a, n in zip(analytes, np.arange(len(analytes))):<EOL><INDENT>axes[n, n].annotate(a + '<STR_LIT:\n>' + udict[a][<NUM_LIT:1>], (<NUM_LIT:0.5>, <NUM_LIT:0.5>),<EOL>xycoords='<STR_LIT>',<EOL>ha='<STR_LIT>', va='<STR_LIT>')<EOL>axes[n, n].set_xlim(*rdict[a])<EOL>axes[n, n].set_ylim(*rdict[a])<EOL><DEDENT>axes[<NUM_LIT:0>, <NUM_LIT:0>].legend(loc='<STR_LIT>', title=filter_string, fontsize=<NUM_LIT:8>)<EOL>for i, j in zip(range(numvars), itertools.cycle((-<NUM_LIT:1>, <NUM_LIT:0>))):<EOL><INDENT>axes[j, i].xaxis.set_visible(True)<EOL>for label in axes[j, i].get_xticklabels():<EOL><INDENT>label.set_rotation(<NUM_LIT>)<EOL><DEDENT>axes[i, j].yaxis.set_visible(True)<EOL><DEDENT>return fig, axes<EOL>

Plot the results of a group of filters in a crossplot.

Parameters
----------
filter_string : str
    A string that identifies a group of filters.
    e.g. 'test' would plot all filters with 'test' in the
    name.
analytes : optional, array_like or str
    The analyte(s) to plot. Defaults to all analytes.

Returns
-------
fig, axes objects


@_log<EOL><INDENT>def filter_report(self, filt=None, analytes=None, savedir=None, nbin=<NUM_LIT:5>):<DEDENT>

return plot.filter_report(self, filt, analytes, savedir, nbin)<EOL>

Visualise effect of data filters.

Parameters
----------
filt : str
    Exact or partial name of filter to plot. Supports
    partial matching. i.e. if 'cluster' is specified, all
    filters with 'cluster' in the name will be plotted.
    Defaults to all filters.
analyte : str
    Name of analyte to plot.
save : str
    file path to save the plot

Returns
-------
(fig, axes)


outputs = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']<EOL>out = {}<EOL>for o in outputs:<EOL><INDENT>out[o] = getattr(self, o)<EOL><DEDENT>out['<STR_LIT>'] = self.filt.params<EOL>out['<STR_LIT>'] = self.filt.sequence<EOL>out['<STR_LIT>'] = self.filt.make_keydict()<EOL>return out<EOL>

Returns paramters used to process data.

Returns
-------
dict
    dict of analysis parameters


def reproduce(past_analysis, plotting=False, data_folder=None,<EOL>srm_table=None, custom_stat_functions=None):

if '<STR_LIT>' in past_analysis:<EOL><INDENT>dirpath = utils.extract_zipdir(past_analysis)<EOL>logpath = os.path.join(dirpath, '<STR_LIT>')<EOL><DEDENT>elif os.path.isdir(past_analysis):<EOL><INDENT>if os.path.exists(os.path.join(past_analysis, '<STR_LIT>')):<EOL><INDENT>logpath = os.path.join(past_analysis, '<STR_LIT>')<EOL><DEDENT><DEDENT>elif '<STR_LIT>' in past_analysis:<EOL><INDENT>logpath = past_analysis<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(('<STR_LIT>' + <EOL>'<STR_LIT>' +<EOL>'<STR_LIT>' + <EOL>'<STR_LIT>' +<EOL>'<STR_LIT>'))<EOL><DEDENT>runargs, paths = logging.read_logfile(logpath)<EOL>csfs = Bunch()<EOL>if custom_stat_functions is None and '<STR_LIT>' in paths.keys():<EOL><INDENT>with open(paths['<STR_LIT>'], '<STR_LIT:r>') as f:<EOL><INDENT>csf = f.read()<EOL><DEDENT>fname = re.compile('<STR_LIT>')<EOL>for c in csf.split('<STR_LIT>'):<EOL><INDENT>if fname.match(c):<EOL><INDENT>csfs[fname.match(c).groups()[<NUM_LIT:0>]] = c<EOL><DEDENT><DEDENT><DEDENT>rep = analyse(*runargs[<NUM_LIT:0>][-<NUM_LIT:1>]['<STR_LIT:args>'], **runargs[<NUM_LIT:0>][-<NUM_LIT:1>]['<STR_LIT>'])<EOL>for fname, arg in runargs:<EOL><INDENT>if fname != '<STR_LIT>':<EOL><INDENT>if '<STR_LIT>' in fname.lower() and plotting:<EOL><INDENT>getattr(rep, fname)(*arg['<STR_LIT:args>'], **arg['<STR_LIT>'])<EOL><DEDENT>elif '<STR_LIT>' in fname.lower():<EOL><INDENT>rep.sample_stats(*arg['<STR_LIT:args>'], csf_dict=csfs, **arg['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>getattr(rep, fname)(*arg['<STR_LIT:args>'], **arg['<STR_LIT>'])<EOL><DEDENT><DEDENT><DEDENT>return rep<EOL>

Reproduce a previous analysis exported with :func:`latools.analyse.minimal_export`

For normal use, supplying `log_file` and specifying a plotting option should be
enough to reproduce an analysis. All requisites (raw data, SRM table and any
custom stat functions) will then be imported from the minimal_export folder.

You may also specify your own raw_data, srm_table and custom_stat_functions,
if you wish.

Parameters
----------
log_file : str
    The path to the log file produced by :func:`~latools.analyse.minimal_export`.
plotting : bool
    Whether or not to output plots.
data_folder : str
    Optional. Specify a different data folder. Data folder
    should normally be in the same folder as the log file.
srm_table : str
    Optional. Specify a different SRM table. SRM table
    should normally be in the same folder as the log file.
custom_stat_functions : str
    Optional. Specify a python file containing custom
    stat functions for use by reproduce. Any custom
    stat functions should normally be included in the
    same folder as the log file.


def __init__(self, data_folder, errorhunt=False, config='<STR_LIT>',<EOL>dataformat=None, extension='<STR_LIT>', srm_identifier='<STR_LIT>',<EOL>cmap=None, time_format=None, internal_standard='<STR_LIT>',<EOL>names='<STR_LIT>', srm_file=None, pbar=None):

<EOL>params = {k: v for k, v in locals().items() if k not in ['<STR_LIT>', '<STR_LIT>']}<EOL>self.log = ['<STR_LIT>'.format(str(params))]<EOL>self.folder = os.path.realpath(data_folder)<EOL>self.parent_folder = os.path.dirname(self.folder)<EOL>self.files = np.array([f for f in os.listdir(self.folder)<EOL>if extension in f])<EOL>self.report_dir = re.sub('<STR_LIT>', '<STR_LIT:/>',<EOL>self.parent_folder + '<STR_LIT:/>' +<EOL>os.path.basename(self.folder) + '<STR_LIT>')<EOL>if not os.path.isdir(self.report_dir):<EOL><INDENT>os.mkdir(self.report_dir)<EOL><DEDENT>self.export_dir = re.sub('<STR_LIT>', '<STR_LIT:/>',<EOL>self.parent_folder + '<STR_LIT:/>' +<EOL>os.path.basename(self.folder) + '<STR_LIT>')<EOL>if not os.path.isdir(self.export_dir):<EOL><INDENT>os.mkdir(self.export_dir)<EOL><DEDENT>self.config = read_configuration(config)<EOL>print('<STR_LIT>' + self.config['<STR_LIT>'] + '<STR_LIT>')<EOL>if srm_file is not None:<EOL><INDENT>if os.path.exists(srm_file):<EOL><INDENT>self.srmfile = srm_file<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(('<STR_LIT>' +<EOL>srm_file +<EOL>'<STR_LIT>'))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if os.path.exists(self.config['<STR_LIT>']):<EOL><INDENT>self.srmfile = self.config['<STR_LIT>']<EOL><DEDENT>elif os.path.exists(pkgrs.resource_filename('<STR_LIT>',<EOL>self.config['<STR_LIT>'])):<EOL><INDENT>self.srmfile = pkgrs.resource_filename('<STR_LIT>',<EOL>self.config['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(('<STR_LIT>' + config +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' + config + '<STR_LIT>' + self.config['<STR_LIT>']))<EOL><DEDENT><DEDENT>if dataformat is None:<EOL><INDENT>if os.path.exists(self.config['<STR_LIT>']):<EOL><INDENT>dataformat = self.config['<STR_LIT>']<EOL><DEDENT>elif os.path.exists(pkgrs.resource_filename('<STR_LIT>',<EOL>self.config['<STR_LIT>'])):<EOL><INDENT>dataformat = pkgrs.resource_filename('<STR_LIT>',<EOL>self.config['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(('<STR_LIT>' +<EOL>config + '<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' +<EOL>config + '<STR_LIT>' + dataformat))<EOL><DEDENT>self.dataformat_file = dataformat<EOL><DEDENT>else:<EOL><INDENT>self.dataformat_file = '<STR_LIT>'<EOL><DEDENT>if isinstance(dataformat, str):<EOL><INDENT>if os.path.exists(dataformat):<EOL><INDENT>self.dataformat = json.load(open(dataformat))<EOL><DEDENT>else:<EOL><INDENT>warnings.warn(("<STR_LIT>" + dataformat +<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" + dataformat))<EOL><DEDENT><DEDENT>elif isinstance(dataformat, dict):<EOL><INDENT>self.dataformat = dataformat<EOL><DEDENT>if pbar is None:<EOL><INDENT>self.pbar = progressbar()<EOL><DEDENT>else:<EOL><INDENT>self.pbar = pbar<EOL><DEDENT>with self.pbar.set(total=len(self.files), desc='<STR_LIT>') as prog:<EOL><INDENT>data = []<EOL>for f in self.files:<EOL><INDENT>data.append(D(self.folder + '<STR_LIT:/>' + f,<EOL>dataformat=self.dataformat,<EOL>errorhunt=errorhunt,<EOL>cmap=cmap,<EOL>internal_standard=internal_standard,<EOL>name=names))<EOL>prog.update()<EOL><DEDENT><DEDENT>if '<STR_LIT:date>' in data[<NUM_LIT:0>].meta.keys():<EOL><INDENT>if (time_format is None) and ('<STR_LIT>' in self.dataformat.keys()):<EOL><INDENT>time_format = self.dataformat['<STR_LIT>']<EOL><DEDENT>start_times = []<EOL>for d in data:<EOL><INDENT>start_times.append(get_date(d.meta['<STR_LIT:date>'], time_format))<EOL><DEDENT>min_time = min(start_times)<EOL>for d, st in zip(data, start_times):<EOL><INDENT>d.uTime = d.Time + (st - min_time).seconds<EOL><DEDENT><DEDENT>else:<EOL><INDENT>ts = <NUM_LIT:0><EOL>for d in data:<EOL><INDENT>d.uTime = d.Time + ts<EOL>ts += d.Time[-<NUM_LIT:1>]<EOL><DEDENT>warnings.warn("<STR_LIT>" +<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>")<EOL><DEDENT>self.max_time = max([d.uTime.max() for d in data])<EOL>data.sort(key=lambda d: d.uTime[<NUM_LIT:0>])<EOL>if (names == '<STR_LIT>') | (names == '<STR_LIT>'):<EOL><INDENT>samples = np.array([s.sample for s in data], dtype=object)  <EOL>usamples, ucounts = np.unique(samples, return_counts=True)<EOL>if usamples.size != samples.size:<EOL><INDENT>dups = usamples[ucounts > <NUM_LIT:1>]  <EOL>nreps = ucounts[ucounts > <NUM_LIT:1>]  <EOL>for d, n in zip(dups, nreps):  <EOL><INDENT>new = [d + '<STR_LIT>'.format(i) for i in range(n)]  <EOL>ind = samples == d<EOL>samples[ind] = new  <EOL>for s, ns in zip([data[i] for i in np.where(ind)[<NUM_LIT:0>]], new):<EOL><INDENT>s.sample = ns  <EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>samples = np.arange(len(data))  <EOL>for i, s in enumerate(samples):<EOL><INDENT>data[i].sample = s<EOL><DEDENT><DEDENT>self.samples = samples<EOL>self.cmaps = data[<NUM_LIT:0>].cmap<EOL>self.analytes = np.array(data[<NUM_LIT:0>].analytes)<EOL>if np.asanyarray(self.analytes).size == <NUM_LIT:0>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if internal_standard in self.analytes:<EOL><INDENT>self.internal_standard = internal_standard<EOL><DEDENT>else:<EOL><INDENT>ValueError('<STR_LIT>'.format(internal_standard) +<EOL>'<STR_LIT>')<EOL><DEDENT>self.minimal_analytes = set([internal_standard])<EOL>self.stages_complete = set(['<STR_LIT>'])<EOL>self.data = Bunch(zip(self.samples, data))<EOL>self.srm_identifier = srm_identifier<EOL>self.stds = []  <EOL>_ = [self.stds.append(s) for s in self.data.values()<EOL>if self.srm_identifier in s.sample]<EOL>self.srms_ided = False<EOL>self.focus_stage = '<STR_LIT>'<EOL>self.focus = Bunch()<EOL>self._has_subsets = False<EOL>self._subset_names = []<EOL>self.subsets = Bunch()<EOL>self.subsets['<STR_LIT>'] = self.samples<EOL>self.subsets[self.srm_identifier] = [s for s in self.samples if self.srm_identifier in s]<EOL>self.subsets['<STR_LIT>'] = [s for s in self.samples if self.srm_identifier not in s]<EOL>self.subsets['<STR_LIT>'] = self.subsets['<STR_LIT>'].copy()<EOL>self.classifiers = Bunch()<EOL>print(('<STR_LIT>'<EOL>'<STR_LIT>').format(len(self.data),<EOL>len(self.stds),<EOL>len(self.data) - len(self.stds)))<EOL>print('<STR_LIT>' + '<STR_LIT:U+0020>'.join(self.analytes))<EOL>print('<STR_LIT>'.format(self.internal_standard))<EOL>

For processing and analysing whole LA - ICPMS datasets.


if subset is None:<EOL><INDENT>samples = self.subsets['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>samples = self.subsets[subset]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise KeyError(("<STR_LIT>".format(subset) +<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>"))<EOL><DEDENT><DEDENT>return samples<EOL>

Helper function to get sample names from subset.

Parameters
----------
subset : str
    Subset name. If None, returns all samples.

Returns
-------
List of sample names


@_log<EOL><INDENT>def autorange(self, analyte='<STR_LIT>', gwin=<NUM_LIT:5>, swin=<NUM_LIT:3>, win=<NUM_LIT:20>,<EOL>on_mult=[<NUM_LIT:1.>, <NUM_LIT>], off_mult=[<NUM_LIT>, <NUM_LIT:1>],<EOL>transform='<STR_LIT>', ploterrs=True, focus_stage='<STR_LIT>'):<DEDENT>

if focus_stage == '<STR_LIT>':<EOL><INDENT>if '<STR_LIT>' not in self.stages_complete:<EOL><INDENT>focus_stage = '<STR_LIT>'<EOL><DEDENT><DEDENT>if analyte is None:<EOL><INDENT>analyte = self.internal_standard<EOL><DEDENT>elif analyte in self.analytes:<EOL><INDENT>self.minimal_analytes.update([analyte])<EOL><DEDENT>fails = {}  <EOL>with self.pbar.set(total=len(self.data), desc='<STR_LIT>') as prog:<EOL><INDENT>for s, d in self.data.items():<EOL><INDENT>f = d.autorange(analyte=analyte, gwin=gwin, swin=swin, win=win,<EOL>on_mult=on_mult, off_mult=off_mult,<EOL>ploterrs=ploterrs, transform=transform)<EOL>if f is not None:<EOL><INDENT>fails[s] = f<EOL><DEDENT>prog.update()  <EOL><DEDENT><DEDENT>if len(fails) > <NUM_LIT:0>:<EOL><INDENT>wstr = ('<STR_LIT>' + '<STR_LIT:*>' * <NUM_LIT> + '<STR_LIT:\n>' +<EOL>'<STR_LIT>' + '<STR_LIT:*>' * <NUM_LIT> + '<STR_LIT:\n>' +<EOL>'<STR_LIT>')<EOL>kwidth = max([len(k) for k in fails.keys()]) + <NUM_LIT:1><EOL>fstr = '<STR_LIT>' + '<STR_LIT:{}>'.format(kwidth) + '<STR_LIT>'<EOL>for k in sorted(fails.keys()):<EOL><INDENT>wstr += fstr.format(k) + '<STR_LIT:U+002CU+0020>'.join(['<STR_LIT>'.format(f) for f in fails[k][-<NUM_LIT:1>]]) + '<STR_LIT:\n>'<EOL><DEDENT>wstr += ('<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>"<STR_LIT>" +<EOL>'<STR_LIT:*>' * <NUM_LIT> + '<STR_LIT:\n>')<EOL>warnings.warn(wstr)<EOL><DEDENT>self.stages_complete.update(['<STR_LIT>'])<EOL>return<EOL>

Automatically separates signal and background data regions.

Automatically detect signal and background regions in the laser
data, based on the behaviour of a single analyte. The analyte used
should be abundant and homogenous in the sample.

**Step 1: Thresholding.**
The background signal is determined using a gaussian kernel density
estimator (kde) of all the data. Under normal circumstances, this
kde should find two distinct data distributions, corresponding to
'signal' and 'background'. The minima between these two distributions
is taken as a rough threshold to identify signal and background
regions. Any point where the trace crosses this thrshold is identified
as a 'transition'.

**Step 2: Transition Removal.**
The width of the transition regions between signal and background are
then determined, and the transitions are excluded from analysis. The
width of the transitions is determined by fitting a gaussian to the
smoothed first derivative of the analyte trace, and determining its
width at a point where the gaussian intensity is at at `conf` time the
gaussian maximum. These gaussians are fit to subsets of the data
centered around the transitions regions determined in Step 1, +/- `win`
data points. The peak is further isolated by finding the minima and
maxima of a second derivative within this window, and the gaussian is
fit to the isolated peak.

Parameters
----------
analyte : str
    The analyte that autorange should consider. For best results,
    choose an analyte that is present homogeneously in high
    concentrations.
    This can also be 'total_counts' to use the sum of all analytes.
gwin : int
    The smoothing window used for calculating the first derivative.
    Must be odd.
win : int
    Determines the width (c +/- win) of the transition data subsets.
smwin : int
    The smoothing window used for calculating the second derivative.
    Must be odd.
conf : float
    The proportional intensity of the fitted gaussian tails that
    determines the transition width cutoff (lower = wider transition
    regions excluded).
trans_mult : array_like, len=2
    Multiples of the peak FWHM to add to the transition cutoffs, e.g.
    if the transitions consistently leave some bad data proceeding the
    transition, set trans_mult to [0, 0.5] to ad 0.5 * the FWHM to the
    right hand side of the limit.
focus_stage : str
    Which stage of analysis to apply processing to.
    Defaults to 'despiked', or rawdata' if not despiked. Can be one of:
    * 'rawdata': raw data, loaded from csv file.
    * 'despiked': despiked data.
    * 'signal'/'background': isolated signal and background data.
      Created by self.separate, after signal and background
      regions have been identified by self.autorange.
    * 'bkgsub': background subtracted data, created by
      self.bkg_correct
    * 'ratios': element ratio data, created by self.ratio.
    * 'calibrated': ratio data calibrated to standards, created by self.calibrate.

Returns
-------
Outputs added as instance attributes. Returns None.
bkg, sig, trn : iterable, bool
    Boolean arrays identifying background, signal and transision
    regions
bkgrng, sigrng and trnrng : iterable
    (min, max) pairs identifying the boundaries of contiguous
    True regions in the boolean arrays.


def find_expcoef(self, nsd_below=<NUM_LIT:0.>, plot=False,<EOL>trimlim=None, autorange_kwargs={}):

print('<STR_LIT>')<EOL>def findtrim(tr, lim=None):<EOL><INDENT>trr = np.roll(tr, -<NUM_LIT:1>)<EOL>trr[-<NUM_LIT:1>] = <NUM_LIT:0><EOL>if lim is None:<EOL><INDENT>lim = <NUM_LIT:0.5> * np.nanmax(tr - trr)<EOL><DEDENT>ind = (tr - trr) >= lim<EOL>return np.arange(len(ind))[ind ^ np.roll(ind, -<NUM_LIT:1>)][<NUM_LIT:0>]<EOL><DEDENT>if not hasattr(self.stds[<NUM_LIT:0>], '<STR_LIT>'):<EOL><INDENT>for s in self.stds:<EOL><INDENT>s.autorange(**autorange_kwargs, ploterrs=False)<EOL><DEDENT><DEDENT>trans = []<EOL>times = []<EOL>for v in self.stds:<EOL><INDENT>for trnrng in v.trnrng[-<NUM_LIT:1>::-<NUM_LIT:2>]:<EOL><INDENT>tr = minmax_scale(v.data['<STR_LIT>'][(v.Time > trnrng[<NUM_LIT:0>]) & (v.Time < trnrng[<NUM_LIT:1>])])<EOL>sm = np.apply_along_axis(np.nanmean, <NUM_LIT:1>,<EOL>rolling_window(tr, <NUM_LIT:3>, pad=<NUM_LIT:0>))<EOL>sm[<NUM_LIT:0>] = sm[<NUM_LIT:1>]<EOL>trim = findtrim(sm, trimlim) + <NUM_LIT:2><EOL>trans.append(minmax_scale(tr[trim:]))<EOL>times.append(np.arange(tr[trim:].size) *<EOL>np.diff(v.Time[<NUM_LIT:1>:<NUM_LIT:3>]))<EOL><DEDENT><DEDENT>times = np.concatenate(times)<EOL>times = np.round(times, <NUM_LIT:2>)<EOL>trans = np.concatenate(trans)<EOL>ti = []<EOL>tr = []<EOL>for t in np.unique(times):<EOL><INDENT>ti.append(t)<EOL>tr.append(np.nanmin(trans[times == t]))<EOL><DEDENT>def expfit(x, e):<EOL><INDENT>"""<STR_LIT>"""<EOL>return np.exp(e * x)<EOL><DEDENT>ep, ecov = curve_fit(expfit, ti, tr, p0=(-<NUM_LIT:1.>))<EOL>eeR2 = R2calc(trans, expfit(times, ep))<EOL>if plot:<EOL><INDENT>fig, ax = plt.subplots(<NUM_LIT:1>, <NUM_LIT:1>, figsize=[<NUM_LIT:6>, <NUM_LIT:4>])<EOL>ax.scatter(times, trans, alpha=<NUM_LIT>, color='<STR_LIT:k>', marker='<STR_LIT:x>', zorder=-<NUM_LIT:2>)<EOL>ax.scatter(ti, tr, alpha=<NUM_LIT:1>, color='<STR_LIT:k>', marker='<STR_LIT:o>')<EOL>fitx = np.linspace(<NUM_LIT:0>, max(ti))<EOL>ax.plot(fitx, expfit(fitx, ep), color='<STR_LIT:r>', label='<STR_LIT>')<EOL>ax.plot(fitx, expfit(fitx, ep - nsd_below * np.diag(ecov)**<NUM_LIT>, ),<EOL>color='<STR_LIT:b>', label='<STR_LIT>')<EOL>ax.text(<NUM_LIT>, <NUM_LIT>,<EOL>('<STR_LIT>'<EOL>'<STR_LIT>') % (ep,<EOL>np.diag(ecov)**<NUM_LIT>,<EOL>eeR2,<EOL>ep - nsd_below * np.diag(ecov)**<NUM_LIT>),<EOL>transform=ax.transAxes, ha='<STR_LIT:right>', va='<STR_LIT>', size=<NUM_LIT:12>)<EOL>ax.set_xlim(<NUM_LIT:0>, ax.get_xlim()[-<NUM_LIT:1>])<EOL>ax.set_xlabel('<STR_LIT>')<EOL>ax.set_ylim(-<NUM_LIT>, <NUM_LIT>)<EOL>ax.set_ylabel('<STR_LIT>')<EOL>plt.legend()<EOL>if isinstance(plot, str):<EOL><INDENT>fig.savefig(plot)<EOL><DEDENT><DEDENT>self.expdecay_coef = ep - nsd_below * np.diag(ecov)**<NUM_LIT><EOL>print('<STR_LIT>'.format(self.expdecay_coef[<NUM_LIT:0>]))<EOL>return<EOL>

Determines exponential decay coefficient for despike filter.

Fits an exponential decay function to the washout phase of standards
to determine the washout time of your laser cell. The exponential
coefficient reported is `nsd_below` standard deviations below the
fitted exponent, to ensure that no real data is removed.

Total counts are used in fitting, rather than a specific analyte.

Parameters
----------
nsd_below : float
    The number of standard deviations to subtract from the fitted
    coefficient when calculating the filter exponent.
plot : bool or str
    If True, creates a plot of the fit, if str the plot is to the
    location specified in str.
trimlim : float
    A threshold limit used in determining the start of the
    exponential decay region of the washout. Defaults to half
    the increase in signal over background. If the data in
    the plot don't fall on an exponential decay line, change
    this number. Normally you'll need to increase it.

Returns
-------
None


@_log<EOL><INDENT>def despike(self, expdecay_despiker=False, exponent=None,<EOL>noise_despiker=True, win=<NUM_LIT:3>, nlim=<NUM_LIT>, exponentplot=False,<EOL>maxiter=<NUM_LIT:4>, autorange_kwargs={}, focus_stage='<STR_LIT>'):<DEDENT>

if focus_stage != self.focus_stage:<EOL><INDENT>self.set_focus(focus_stage)<EOL><DEDENT>if expdecay_despiker and exponent is None:<EOL><INDENT>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self.find_expcoef(plot=exponentplot,<EOL>autorange_kwargs=autorange_kwargs)<EOL><DEDENT>exponent = self.expdecay_coef<EOL>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT>with self.pbar.set(total=len(self.data), desc='<STR_LIT>') as prog:<EOL><INDENT>for d in self.data.values():<EOL><INDENT>d.despike(expdecay_despiker, exponent,<EOL>noise_despiker, win, nlim, maxiter)<EOL>prog.update()<EOL><DEDENT><DEDENT>self.stages_complete.update(['<STR_LIT>'])<EOL>self.focus_stage = '<STR_LIT>'<EOL>return<EOL>

Despikes data with exponential decay and noise filters.

Parameters
----------
expdecay_despiker : bool
    Whether or not to apply the exponential decay filter.
exponent : None or float
    The exponent for the exponential decay filter. If None,
    it is determined automatically using `find_expocoef`.
tstep : None or float
    The timeinterval between measurements. If None, it is
    determined automatically from the Time variable.
noise_despiker : bool
    Whether or not to apply the standard deviation spike filter.
win : int
    The rolling window over which the spike filter calculates
    the trace statistics.
nlim : float
    The number of standard deviations above the rolling mean
    that data are excluded.
exponentplot : bool
    Whether or not to show a plot of the automatically determined
    exponential decay exponent.
maxiter : int
    The max number of times that the fitler is applied.
focus_stage : str
    Which stage of analysis to apply processing to.
    Defaults to 'rawdata'. Can be one of:
    * 'rawdata': raw data, loaded from csv file.
    * 'despiked': despiked data.
    * 'signal'/'background': isolated signal and background data.
      Created by self.separate, after signal and background
      regions have been identified by self.autorange.
    * 'bkgsub': background subtracted data, created by
      self.bkg_correct
    * 'ratios': element ratio data, created by self.ratio.
    * 'calibrated': ratio data calibrated to standards, created by self.calibrate.

Returns
-------
None