def populate_subtree(basepath, treemap, verbose=False):

file_nodes = OrderedDict()<EOL>if isinstance(treemap, tuple):<EOL><INDENT>try:<EOL><INDENT>file_nodes = process_tuple_node(basepath, treemap)<EOL><DEDENT>except:<EOL><INDENT>raise FileTreeMapError('<STR_LIT>'<EOL>'<STR_LIT>'.format(basepath, treemap))<EOL><DEDENT><DEDENT>if isinstance(treemap, list):<EOL><INDENT>for node in treemap:<EOL><INDENT>try:<EOL><INDENT>file_nodes.update(process_tuple_node(basepath, node))<EOL><DEDENT>except:<EOL><INDENT>raise FileTreeMapError('<STR_LIT>'<EOL>'<STR_LIT>'.format(basepath, node))<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(treemap, dict):<EOL><INDENT>for k in treemap.keys():<EOL><INDENT>cname = k<EOL>child_map = treemap[k]<EOL>if isinstance(child_map, tuple) or isinstance(child_map, dict):<EOL><INDENT>try:<EOL><INDENT>file_nodes[cname] = populate_subtree(basepath, child_map)<EOL><DEDENT>except:<EOL><INDENT>raise FileTreeMapError('<STR_LIT>'<EOL>'<STR_LIT>'.format(basepath,<EOL>child_map))<EOL><DEDENT><DEDENT>elif isinstance(child_map, str):<EOL><INDENT>if child_map[<NUM_LIT:0>] == os.sep:<EOL><INDENT>raise FileTreeMapError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(str(child_map),<EOL>os.sep))<EOL><DEDENT>subpaths = get_possible_paths(basepath, child_map)<EOL>if subpaths:<EOL><INDENT>file_nodes[cname] = subpaths<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if verbose:<EOL><INDENT>log.info('<STR_LIT>'.format(basepath, file_nodes.keys()))<EOL><DEDENT>return file_nodes<EOL>

:param path: str

:param treemap: dict

:return: dict


self._filetree = {}<EOL>self._treemap = {}<EOL>self._basepath = '<STR_LIT>'<EOL>self._ignore_regexes = []<EOL>


return '<STR_LIT>'.format(self._basepath,<EOL>self._filetree)<EOL>


def from_config_file(self, config_file, verbose=False):

assert(op.isfile(config_file))<EOL>self.__init__()<EOL>try:<EOL><INDENT>self._basepath, self._treemap = self._import_config(config_file)<EOL>self.update(verbose)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>raise EnvironmentError('<STR_LIT>'.format(config_file)) from exc<EOL><DEDENT>

:param config_file: str
 Path to a configuration file.
 This file must declare a root_path and a filetree regex tree.

:param verbose: bool


def from_dict(self, root_path, filetree, verbose=False):

self.__init__()<EOL>self._basepath = root_path<EOL>self._treemap = filetree<EOL>self.update(verbose)<EOL>

:param root_path: string
:param filetree: dict


if not overwrite:<EOL><INDENT>while op.exists(dirpath):<EOL><INDENT>dirpath += '<STR_LIT:+>'<EOL><DEDENT><DEDENT>os.makedirs(dirpath, exist_ok=overwrite)<EOL>return dirpath<EOL>

Will create dirpath folder. If dirpath already exists and overwrite is False,
        will append a '+' suffix to dirpath until dirpath does not exist.


if not op.isfile(filepath):<EOL><INDENT>raise IOError('<STR_LIT>'<EOL>'<STR_LIT>'.format(filepath))<EOL><DEDENT>cfg = import_pyfile(filepath)<EOL>if not hasattr(cfg, '<STR_LIT>'):<EOL><INDENT>raise KeyError('<STR_LIT>')<EOL><DEDENT>if not hasattr(cfg, '<STR_LIT>'):<EOL><INDENT>raise KeyError('<STR_LIT>')<EOL><DEDENT>return cfg.root_path, cfg.filetree<EOL>

Imports filetree and root_path variable values from the filepath.

:param filepath:
:return: root_path and filetree


return self._filetree.keys()<EOL>

Return a list of the names of the root nodes.


files = self.get_node(nodepath)<EOL>return get_dict_leaves(files)<EOL>


return commonprefix(self.get_node_filepaths(nodepath))<EOL>

Returns the common filepath between all leaves in the filetree.


mydict = self._filetree if adict is None else adict<EOL>if isinstance(mydict, dict):<EOL><INDENT>for nom in mydict.keys():<EOL><INDENT>if isinstance(mydict[nom], dict):<EOL><INDENT>matchs = filter_list(mydict[nom], pattern)<EOL>for nom in matchs:<EOL><INDENT>mydict = self.remove_nodes(pattern, mydict[nom])<EOL>mydict.pop(nom)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>mydict[nom] = filter_list(mydict[nom], pattern)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>matchs = set(filter_list(mydict, pattern))<EOL>mydict = set(mydict) - matchs<EOL><DEDENT>return mydict<EOL>

Remove the nodes that match the pattern.


def count_node_match(self, pattern, adict=None):

mydict = self._filetree if adict is None else adict<EOL>k = <NUM_LIT:0><EOL>if isinstance(mydict, dict):<EOL><INDENT>names = mydict.keys()<EOL>k += len(filter_list(names, pattern))<EOL>for nom in names:<EOL><INDENT>k += self.count_node_match(pattern, mydict[nom])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>k = len(filter_list(mydict, pattern))<EOL><DEDENT>return k<EOL>

Return the number of nodes that match the pattern.

:param pattern:

:param adict:
:return: int


for key in config_map:<EOL><INDENT>if config_map[key] == '<STR_LIT:None>':<EOL><INDENT>config_map[key] = None<EOL><DEDENT>setattr(self, key, config_map[key])<EOL><DEDENT>


def copy_w_ext(srcfile, destdir, basename):

ext = get_extension(op.basename(srcfile))<EOL>dstpath = op.join(destdir, basename + ext)<EOL>return copy_w_plus(srcfile, dstpath)<EOL>

Copy `srcfile` in `destdir` with name `basename + get_extension(srcfile)`.
    Add pluses to the destination path basename if a file with the same name already
    exists in `destdir`.

    Parameters
    ----------
    srcfile: str

    destdir: str

    basename:str

    Returns
    -------
    dstpath: str


dst_ext = get_extension(dst)<EOL>dst_pre = remove_ext   (dst)<EOL>while op.exists(dst_pre + dst_ext):<EOL><INDENT>dst_pre += '<STR_LIT:+>'<EOL><DEDENT>shutil.copy(src, dst_pre + dst_ext)<EOL>return dst_pre + dst_ext<EOL>

Copy file from `src` path to `dst` path. If `dst` already exists, will add '+' characters
    to the end of the basename without extension.

    Parameters
    ----------
    src: str

    dst: str

    Returns
    -------
    dstpath: str


if not op.exists(folderpath):<EOL><INDENT>raise FolderNotFound(folderpath)<EOL><DEDENT>return op.abspath(folderpath)<EOL>

Returns the absolute path of folderpath.
    If the path does not exist, will raise IOError.


if not op.exists(dirpath):<EOL><INDENT>os.mkdir(dirpath)<EOL><DEDENT>

Call os.mkdir only if `dirpath` does not exist.


def get_extension(filepath, check_if_exists=False, allowed_exts=ALLOWED_EXTS):

if check_if_exists:<EOL><INDENT>if not op.exists(filepath):<EOL><INDENT>raise IOError('<STR_LIT>' + filepath)<EOL><DEDENT><DEDENT>rest, ext = op.splitext(filepath)<EOL>if ext in allowed_exts:<EOL><INDENT>alloweds = allowed_exts[ext]<EOL>_, ext2 = op.splitext(rest)<EOL>if ext2 in alloweds:<EOL><INDENT>ext = ext2 + ext<EOL><DEDENT><DEDENT>return ext<EOL>

Return the extension of fpath.

    Parameters
    ----------
    fpath: string
    File name or path

    check_if_exists: bool

    allowed_exts: dict
    Dictionary of strings, where the key if the last part of a complex ('.' separated) extension
    and the value is the previous part.
    For example: for the '.nii.gz' extension I would have a dict as {'.gz': ['.nii',]}

    Returns
    -------
    str
    The extension of the file name or path


def add_extension_if_needed(filepath, ext, check_if_exists=False):

if not filepath.endswith(ext):<EOL><INDENT>filepath += ext<EOL><DEDENT>if check_if_exists:<EOL><INDENT>if not op.exists(filepath):<EOL><INDENT>raise IOError('<STR_LIT>' + filepath)<EOL><DEDENT><DEDENT>return filepath<EOL>

Add the extension ext to fpath if it doesn't have it.

    Parameters
    ----------
    filepath: str
    File name or path

    ext: str
    File extension

    check_if_exists: bool

    Returns
    -------
    File name or path with extension added, if needed.


return filepath[:filepath.rindex(get_extension(filepath))]<EOL>

Removes the extension of the file.

    Parameters
    ----------
    filepath: str
    File path or name

    Returns
    -------
    str
    File path or name without extension


with open(filepath, '<STR_LIT:w>') as f:<EOL><INDENT>f.writelines(lines)<EOL><DEDENT>

Write the given lines to the file in filepath

    Parameters
    ----------
    filepath: str

    lines: list of str


for line in open(filepath):<EOL><INDENT>if srch_str in line:<EOL><INDENT>return line<EOL><DEDENT><DEDENT>return None<EOL>

Return the first line in file defined by filepath
    that contains srch_str

    Parameters
    ----------
    srch_str: str

    filepath: str

    Returns
    ----------
    str


def parse_subjects_list(filepath, datadir='<STR_LIT>', split='<STR_LIT::>', labelsf=None):

labels = []<EOL>subjs  = []<EOL>if datadir:<EOL><INDENT>datadir += op.sep<EOL><DEDENT>with open(filepath, '<STR_LIT:r>') as f:<EOL><INDENT>for s in f:<EOL><INDENT>line = s.strip().split(split)<EOL>if len(line) == <NUM_LIT:2>:<EOL><INDENT>labels.append(np.float(line[<NUM_LIT:1>]))<EOL>subjf = line[<NUM_LIT:0>].strip()<EOL><DEDENT>else:<EOL><INDENT>subjf = line.strip()<EOL><DEDENT>if not op.isabs(subjf):<EOL><INDENT>subjs.append(datadir + subjf)<EOL><DEDENT>else:<EOL><INDENT>subjs.append(subjf)<EOL><DEDENT><DEDENT><DEDENT>if labelsf is not None:<EOL><INDENT>labels = np.loadtxt(labelsf)<EOL><DEDENT>return [labels, subjs]<EOL>

Parses a file with a list of: <subject_file>:<subject_class_label>.

    Parameters
    ----------
    filepath: str
    Path to file with a list of: <subject_file>:<subject_class_label>.
    Where ':' can be any split character

    datadir: str
    String to be path prefix of each line of the fname content,
    only in case the lines are relative file paths.

    split: str
    Split character for each line

    labelsf: str
    Path to file with a list of the labels if it is not included in
    fname. It will overwrite the labels from fname.

    Returns
    -------
    [labels, subjs] where labels is a list of labels and subjs a list of
    filepaths


def create_subjects_file(filelist, labels, output_file, split='<STR_LIT::>'):

if len(filelist) != len(labels):<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'.format(len(filelist), len(labels)))<EOL><DEDENT>lines = []<EOL>for i, subj in enumerate(filelist):<EOL><INDENT>lab  = labels[i]<EOL>line = subj + split + str(lab)<EOL>lines.append(line)<EOL><DEDENT>lines = np.array(lines)<EOL>np.savetxt(output_file, lines, fmt='<STR_LIT:%s>')<EOL>

Creates a file where each line is <subject_file>:<subject_class_label>.

    Parameters
    ----------
    filelist: list of str
    List of filepaths

    labels: list of int, str or labels that can be transformed with str()
    List of labels

    output_file: str
    Output file path

    split: str
    Split character for each line


return [op.join(path, str(item)) for item in filelist]<EOL>

Joins path to each line in filelist

    Parameters
    ----------
    path: str

    filelist: list of str

    Returns
    -------
    list of filepaths


if not folder:<EOL><INDENT>for f in filelist:<EOL><INDENT>os.remove(f)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for f in filelist:<EOL><INDENT>os.remove(op.join(folder, f))<EOL><DEDENT><DEDENT>

Deletes all files in filelist

    Parameters
    ----------
    filelist: list of str
        List of the file paths to be removed

    folder: str
        Path to be used as common directory for all file paths in filelist


if path[<NUM_LIT:0>] == op.sep:<EOL><INDENT>folder_depth += <NUM_LIT:1><EOL><DEDENT>return op.sep.join(path.split(op.sep)[<NUM_LIT:0>:folder_depth])<EOL>

Returns a folder path of path with depth given by folder_dept:

Parameters
----------
path: str

folder_depth: int > 0

Returns
-------
A folder path

Example
-------
>>> get_folder_subpath('/home/user/mydoc/work/notes.txt', 3)
>>> '/home/user/mydoc'


return tempfile.NamedTemporaryFile(dir=dirpath, suffix=suffix)<EOL>

Uses tempfile to create a NamedTemporaryFile using
the default arguments.

Parameters
----------
dirpath: str
Directory where it must be created.
If dir is specified, the file will be created
in that directory, otherwise, a default directory is used.
The default directory is chosen from a platform-dependent
list, but the user of the application can control the
directory location by setting the TMPDIR, TEMP or TMP
environment variables.

suffix: str
File name suffix.
It does not put a dot between the file name and the
suffix; if you need one, put it at the beginning of suffix.

Returns
-------
file object

Note
----
Please, close it once you have used the file.


if basepath is None:<EOL><INDENT>return tempfile.TemporaryDirectory(dir=basepath)<EOL><DEDENT>else:<EOL><INDENT>return tempfile.TemporaryDirectory(prefix=prefix, dir=basepath)<EOL><DEDENT>

Uses tempfile to create a TemporaryDirectory using
the default arguments.
The folder is created using tempfile.mkdtemp() function.

Parameters
----------
prefix: str
Name prefix for the temporary folder.

basepath: str
Directory where the new folder must be created.
The default directory is chosen from a platform-dependent
list, but the user of the application can control the
directory location by setting the TMPDIR, TEMP or TMP
environment variables.

Returns
-------
folder object


p = subprocess.Popen(['<STR_LIT>', '<STR_LIT>', filepath], stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>result, err = p.communicate()<EOL>if p.returncode != <NUM_LIT:0>:<EOL><INDENT>raise IOError(err)<EOL><DEDENT>l = result.strip()<EOL>l = int(l.split()[<NUM_LIT:0>])<EOL>return l<EOL>

Returns the length of the file using the 'wc' GNU command

    Parameters
    ----------
    filepath: str

    Returns
    -------
    float


statinfo = os.stat(filepath)<EOL>return statinfo.st_size<EOL>

Return the number of lines in file in filepath

    Parameters
    ----------
    filepath: str

    Returns
    -------
    int


return op.getsize(filepath)<EOL>

Returns the length of the file

    Parameters
    ----------
    filepath: str

    Returns
    -------
    float


file_obj.seek(<NUM_LIT:0>, os.SEEK_END)<EOL>return file_obj.tell()<EOL>

Returns the length of the size of the file

    Parameters
    ----------
    file_obj: file-like object

    Returns
    -------
    float


return dict((s, getattr(a_named_tuple, s)) for s in a_named_tuple._fields)<EOL>

Transform a named tuple into a dictionary


def merge_dict_of_lists(adict, indices, pop_later=True, copy=True):

def check_indices(idxs, x):<EOL><INDENT>for i in chain(*idxs):<EOL><INDENT>if i < <NUM_LIT:0> or i >= x:<EOL><INDENT>raise IndexError("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>check_indices(indices, len(adict))<EOL>rdict = adict.copy() if copy else adict<EOL>dict_keys = list(rdict.keys())<EOL>for i, j in zip(*indices):<EOL><INDENT>rdict[dict_keys[i]].extend(rdict[dict_keys[j]])<EOL><DEDENT>if pop_later:<EOL><INDENT>for i, j in zip(*indices):<EOL><INDENT>rdict.pop(dict_keys[j], '<STR_LIT>')<EOL><DEDENT><DEDENT>return rdict<EOL>

Extend the within a dict of lists. The indices will indicate which
    list have to be extended by which other list.

    Parameters
    ----------
    adict: OrderedDict
        An ordered dictionary of lists

    indices: list or tuple of 2 iterables of int, bot having the same length
        The indices of the lists that have to be merged, both iterables items
         will be read pair by pair, the first is the index to the list that
         will be extended with the list of the second index.
         The indices can be constructed with Numpy e.g.,
         indices = np.where(square_matrix)

    pop_later: bool
        If True will oop out the lists that are indicated in the second
         list of indices.

    copy: bool
        If True will perform a deep copy of the input adict before
         modifying it, hence not changing the original input.

    Returns
    -------
    Dictionary of lists

    Raises
    ------
    IndexError
        If the indices are out of range


if keys is None:<EOL><INDENT>keys = list(list_of_dicts[<NUM_LIT:0>].keys())<EOL><DEDENT>dict_of_lists = DefaultOrderedDict(list)<EOL>for d in list_of_dicts:<EOL><INDENT>for k in keys:<EOL><INDENT>dict_of_lists[k].append(d[k])<EOL><DEDENT><DEDENT>return dict_of_lists<EOL>

Return a dict of lists from a list of dicts with the same keys.
For each dict in list_of_dicts with look for the values of the
given keys and append it to the output dict.

Parameters
----------
list_of_dicts: list of dicts

keys: list of str
    List of keys to create in the output dict
    If None will use all keys in the first element of list_of_dicts
Returns
-------
DefaultOrderedDict of lists


def __init__(self, dir=None, filepath=None, hdf_basepath='<STR_LIT:/>',<EOL>overwrite_if_exist=False, remove_on_destroy=False):

if dir is None:<EOL><INDENT>dir = tempfile.gettempdir()<EOL><DEDENT>if filepath is None:<EOL><INDENT>self._fname = self.get_temp_file(dir)<EOL><DEDENT>else:<EOL><INDENT>self._fname = filepath<EOL><DEDENT>self._fname = self.get_temp_file(dir).name<EOL>self._remove_on_destroy = remove_on_destroy<EOL>self._overwrite = overwrite_if_exist<EOL>self._hdf_basepath = hdf_basepath<EOL>self._hdf_file = None<EOL>self._group = None<EOL>self._datasets = {}<EOL>self.create_hdf_file()<EOL>

:param dir: string

:param filepath: string

:param hdf_basepath: string

:param overwrite_if_exist: bool

:param remove_on_destroy: bool


return tempfile.NamedTemporaryFile(dir=dir, suffix=suffix)<EOL>

Uses tempfile to create a NamedTemporaryFile using
the default arguments.

@param dir: string
Directory where it must be created.
If dir is specified, the file will be created
in that directory, otherwise, a default directory is used.
The default directory is chosen from a platform-dependent
list, but the user of the application can control the
directory location by setting the TMPDIR, TEMP or TMP
environment variables.

@param suffix: string
File name suffix.
It does not put a dot between the file name and the
suffix; if you need one, put it at the beginning of suffix.

@return: file object

@note:
Close it once you have used the file.


self._hdf_file.close()<EOL>if self._remove_on_destroy:<EOL><INDENT>os.remove(self._fname)<EOL><DEDENT>


mode = '<STR_LIT:w>'<EOL>if not self._overwrite and os.path.exists(self._fname):<EOL><INDENT>mode = '<STR_LIT:a>'<EOL><DEDENT>self._hdf_file = h5py.File(self._fname, mode)<EOL>if self._hdf_basepath == '<STR_LIT:/>':<EOL><INDENT>self._group = self._hdf_file['<STR_LIT:/>']<EOL><DEDENT>else:<EOL><INDENT>self._group = self._hdf_file.create_group(self._hdf_basepath)<EOL><DEDENT>


def get_dataset(self, ds_name, mode='<STR_LIT:r>'):

if ds_name in self._datasets:<EOL><INDENT>return self._datasets[ds_name]<EOL><DEDENT>else:<EOL><INDENT>return self.create_empty_dataset(ds_name)<EOL><DEDENT>

Returns a h5py dataset given its registered name.

:param ds_name: string
Name of the dataset to be returned.

:return:


def create_empty_dataset(self, ds_name, dtype=np.float32):

if ds_name in self._datasets:<EOL><INDENT>return self._datasets[ds_name]<EOL><DEDENT>ds = self._group.create_dataset(ds_name, (<NUM_LIT:1>, <NUM_LIT:1>), maxshape=None,<EOL>dtype=dtype)<EOL>self._datasets[ds_name] = ds<EOL>return ds<EOL>

Creates a Dataset with unknown size.
Resize it before using.

:param ds_name: string

:param dtype: dtype
Datatype of the dataset

:return: h5py DataSet


def create_dataset(self, ds_name, data, attrs=None, dtype=None):

if ds_name in self._datasets:<EOL><INDENT>ds = self._datasets[ds_name]<EOL>if ds.dtype != data.dtype:<EOL><INDENT>warnings.warn('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if dtype is None:<EOL><INDENT>dtype = data.dtype<EOL><DEDENT>ds = self._group.create_dataset(ds_name, data.shape,<EOL>dtype=dtype)<EOL>if attrs is not None:<EOL><INDENT>for key in attrs:<EOL><INDENT>setattr(ds.attrs, key, attrs[key])<EOL><DEDENT><DEDENT><DEDENT>ds.read_direct(data)<EOL>self._datasets[ds_name] = ds<EOL>return ds<EOL>

Saves a Numpy array in a dataset in the HDF file, registers it as
ds_name and returns the h5py dataset.

:param ds_name: string
Registration name of the dataset to be registered.

:param data: Numpy ndarray

:param dtype: dtype
Datatype of the dataset

:return: h5py dataset


def save(self, ds_name, data, dtype=None):

return self.create_dataset(ds_name, data, dtype)<EOL>


@staticmethod<EOL><INDENT>def _fill_missing_values(df, range_values, fill_value=<NUM_LIT:0>, fill_method=None):<DEDENT>

idx_colnames  = df.index.names<EOL>idx_colranges = [range_values[x] for x in idx_colnames]<EOL>fullindex = pd.Index([p for p in product(*idx_colranges)],<EOL>name=tuple(idx_colnames))<EOL>fulldf = df.reindex(index=fullindex, fill_value=fill_value,<EOL>method=fill_method)<EOL>fulldf.index.names = idx_colnames<EOL>return fulldf, idx_colranges<EOL>

Will get the names of the index colums of df, obtain their ranges from
range_values dict and return a reindexed version of df with the given
range values.

:param df: pandas DataFrame

:param range_values: dict or array-like
Must contain for each index column of df an entry with all the values
within the range of the column.

:param fill_value: scalar or 'nearest', default 0
Value to use for missing values. Defaults to 0, but can be any
"compatible" value, e.g., NaN.
The 'nearest' mode will fill the missing value with the nearest value in
 the column.

:param fill_method:  {'backfill', 'bfill', 'pad', 'ffill', None}, default None
Method to use for filling holes in reindexed DataFrame
'pad' / 'ffill': propagate last valid observation forward to next valid
'backfill' / 'bfill': use NEXT valid observation to fill gap

:return: pandas Dataframe and used column ranges
reindexed DataFrame and dict with index column ranges


node = self.get_node(key)<EOL>if node is None:<EOL><INDENT>raise KeyError('<STR_LIT>' % key)<EOL><DEDENT>if hasattr(node, '<STR_LIT>'):<EOL><INDENT>if '<STR_LIT>' in node.attrs:<EOL><INDENT>return self._read_group(node)<EOL><DEDENT><DEDENT>return self._read_array(node)<EOL>

Retrieve pandas object or group of Numpy ndarrays
stored in file

Parameters
----------
key : object

Returns
-------
obj : type of object stored in file


def put(self, key, value, attrs=None, format=None, append=False, **kwargs):

if not isinstance(value, np.ndarray):<EOL><INDENT>super(NumpyHDFStore, self).put(key, value, format, append, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>group = self.get_node(key)<EOL>if group is not None and not append:<EOL><INDENT>self._handle.removeNode(group, recursive=True)<EOL>group = None<EOL><DEDENT>if group is None:<EOL><INDENT>paths = key.split('<STR_LIT:/>')<EOL>path = '<STR_LIT:/>'<EOL>for p in paths:<EOL><INDENT>if not len(p):<EOL><INDENT>continue<EOL><DEDENT>new_path = path<EOL>if not path.endswith('<STR_LIT:/>'):<EOL><INDENT>new_path += '<STR_LIT:/>'<EOL><DEDENT>new_path += p<EOL>group = self.get_node(new_path)<EOL>if group is None:<EOL><INDENT>group = self._handle.createGroup(path, p)<EOL><DEDENT>path = new_path<EOL><DEDENT><DEDENT>ds_name = kwargs.get('<STR_LIT>', self._array_dsname)<EOL>ds = self._handle.createArray(group, ds_name, value)<EOL>if attrs is not None:<EOL><INDENT>for key in attrs:<EOL><INDENT>setattr(ds.attrs, key, attrs[key])<EOL><DEDENT><DEDENT>self._handle.flush()<EOL>return ds<EOL><DEDENT>

Store object in HDFStore

Parameters
----------
key : str

value : {Series, DataFrame, Panel, Numpy ndarray}

format : 'fixed(f)|table(t)', default is 'fixed'
    fixed(f) : Fixed format
        Fast writing/reading. Not-appendable, nor searchable

    table(t) : Table format
        Write as a PyTables Table structure which may perform worse but allow more flexible operations
        like searching/selecting subsets of the data

append : boolean, default False
    This will force Table format, append the input data to the
    existing.

encoding : default None, provide an encoding for strings


def _push_dfblock(self, key, df, ds_name, range_values):

<EOL>vals_colranges = [range_values[x] for x in df.index.names]<EOL>nu_shape = [len(x) for x in vals_colranges]<EOL>return self.put(key, np.reshape(df.values, tuple(nu_shape)),<EOL>attrs={'<STR_LIT>': df.index.names},<EOL>ds_name=ds_name, append=True)<EOL>

:param key: string
:param df: pandas Dataframe
:param ds_name: string


def put_df_as_ndarray(self, key, df, range_values, loop_multiindex=False,<EOL>unstack=False, fill_value=<NUM_LIT:0>, fill_method=None):

idx_colnames = df.index.names<EOL>if key is None:<EOL><INDENT>key = idx_colnames[<NUM_LIT:0>]<EOL><DEDENT>if loop_multiindex:<EOL><INDENT>idx_values = df.index.get_level_values(<NUM_LIT:0>).unique()<EOL>for idx in idx_values:<EOL><INDENT>vals, _ = self._fill_missing_values(df.xs((idx,), level=idx_colnames[<NUM_LIT:0>]),<EOL>range_values,<EOL>fill_value=fill_value,<EOL>fill_method=fill_method)<EOL>ds_name = str(idx) + '<STR_LIT:_>' + '<STR_LIT:_>'.join(vals.columns)<EOL>self._push_dfblock(key, vals, ds_name, range_values)<EOL><DEDENT>return self._handle.get_node('<STR_LIT:/>' + str(key))<EOL><DEDENT>else:<EOL><INDENT>if unstack:<EOL><INDENT>df = df.unstack(idx_colnames[<NUM_LIT:0>])<EOL>for idx in df:<EOL><INDENT>vals, _ = self._fill_missing_values(df[idx], range_values,<EOL>fill_value=fill_value,<EOL>fill_method=fill_method)<EOL>vals = np.nan_to_num(vals)<EOL>ds_name = '<STR_LIT:_>'.join([str(x) for x in vals.name])<EOL>self._push_dfblock(key, vals, ds_name, range_values)<EOL><DEDENT>return self._handle.get_node('<STR_LIT:/>' + str(key))<EOL><DEDENT><DEDENT>vals, _ = self._fill_missing_values(df, range_values,<EOL>fill_value=fill_value,<EOL>fill_method=fill_method)<EOL>ds_name = self._array_dsname<EOL>return self._push_dfblock(key, vals, ds_name, range_values)<EOL>

Returns a PyTables HDF Array from df in the shape given by its index columns range values.

        :param key: string object

        :param df: pandas DataFrame

        :param range_values: dict or array-like
        Must contain for each index column of df an entry with all the values
        within the range of the column.

        :param loop_multiindex: bool
        Will loop through the first index in a multiindex dataframe, extract a
        dataframe only for one value, complete and fill the missing values and
        store in the HDF.
        If this is True, it will not use unstack.
        This is as fast as unstacking.

        :param unstack: bool
        Unstack means that this will use the first index name to
        unfold the DataFrame, and will create a group with as many datasets
        as valus has this first index.
        Use this if you think the filled dataframe won't fit in your RAM memory.
        If set to False, this will transform the dataframe in memory first
        and only then save it.

        :param fill_value: scalar or 'nearest', default 0
        Value to use for missing values. Defaults to 0, but can be any
        "compatible" value, e.g., NaN.
        The 'nearest' mode will fill the missing value with the nearest value in
         the column.

        :param fill_method:  {'backfill', 'bfill', 'pad', 'ffill', None}, default None
        Method to use for filling holes in reindexed DataFrame
        'pad' / 'ffill': propagate last valid observation forward to next valid
        'backfill' / 'bfill': use NEXT valid observation to fill gap

        :return: PyTables data node


import pandas.rpy.common as com<EOL>w = com.robj.r('<STR_LIT>' % input_file)<EOL>return com.convert_robj(w)<EOL>

SPSS .sav files to Pandas DataFrame through Rpy2

:param input_file: string

:return:


from savReaderWriter import SavReader<EOL>lines = []<EOL>with SavReader(input_file, returnHeader=True) as reader:<EOL><INDENT>header = next(reader)<EOL>for line in reader:<EOL><INDENT>lines.append(line)<EOL><DEDENT><DEDENT>return pd.DataFrame(data=lines, columns=header)<EOL>

SPSS .sav files to Pandas DataFrame through savreader module

:param input_file: string

:return:


mashelf = shelve.open(file_path, '<STR_LIT:n>')<EOL>for vn in variables.keys():<EOL><INDENT>try:<EOL><INDENT>mashelf[vn] = variables[vn]<EOL><DEDENT>except KeyError as ke:<EOL><INDENT>raise Exception('<STR_LIT>'.format(vn)) from ke<EOL><DEDENT><DEDENT>mashelf.close()<EOL>

Parameters
----------
file_path: str

variables: dict
    Dictionary with objects. Object name -> object

Notes
-----
    Before calling this function, create a varlist this way:

    shelfvars = []
    for v in varnames:
        shelfvars.append(eval(v))

    #to_restore variables from shelf
    my_shelf = shelve.open(filename)
    for key in my_shelf:
       globals()[key]=my_shelf[key]
    my_shelf.close()


def save_variables_to_mat(file_path, variables, format='<STR_LIT:5>'):

try:<EOL><INDENT>sio.savemat(file_path, variables, format=format)<EOL><DEDENT>except IOError as ioe:<EOL><INDENT>raise IOError('<STR_LIT>'.format(file_path)) from ioe<EOL><DEDENT>

Parameters
---------
file_path: str

variables: dict
    Dictionary with objects. Object name -> object

format : {'5', '4'}, string, optional
    '5' (the default) for MATLAB 5 and up (to 7.2),
    '4' for MATLAB 4 .mat files
    See scipy.io.savemat dostrings.


ext = get_extension(filename).lower()<EOL>out_exts = {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'}<EOL>output_file = filename<EOL>if not ext in out_exts:<EOL><INDENT>output_file = add_extension_if_needed(filename, '<STR_LIT>')<EOL>ext = get_extension(filename)<EOL><DEDENT>if ext == '<STR_LIT>' or ext == '<STR_LIT>':<EOL><INDENT>save_variables_to_shelve(output_file, variables)<EOL><DEDENT>elif ext == '<STR_LIT>':<EOL><INDENT>save_variables_to_mat(output_file, variables)<EOL><DEDENT>elif ext == '<STR_LIT>' or ext == '<STR_LIT>':<EOL><INDENT>from .hdf5 import save_variables_to_hdf5<EOL>save_variables_to_hdf5(output_file, variables)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(ext))<EOL><DEDENT>

Save given variables in a file.
        Valid extensions: '.pyshelf' or '.shelf' (Python shelve)
                          '.mat' (Matlab archive),
                          '.hdf5' or '.h5' (HDF5 file)

        Parameters
        ----------
        filename: str
            Output file path.

        variables: dict
            Dictionary varname -> variable

        Raises
        ------
        ValueError: if the extension of the filesname is not recognized.


@staticmethod<EOL><INDENT>def save_varlist(filename, varnames, varlist):<DEDENT>

variables = {}<EOL>for i, vn in enumerate(varnames):<EOL><INDENT>variables[vn] = varlist[i]<EOL><DEDENT>ExportData.save_variables(filename, variables)<EOL>

Valid extensions '.pyshelf', '.mat', '.hdf5' or '.h5'

@param filename: string

@param varnames: list of strings
Names of the variables

@param varlist: list of objects
The objects to be saved


def save_variables_to_hdf5(file_path, variables, mode='<STR_LIT:w>', h5path='<STR_LIT:/>'):

if not isinstance(variables, dict):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(type(variables)))<EOL><DEDENT>if not variables:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>h5file  = h5py.File(file_path, mode=mode)<EOL>h5group = h5file.require_group(h5path)<EOL>for vn in variables:<EOL><INDENT>data = variables[vn]<EOL>if hasattr(data, '<STR_LIT>') and (data.dtype.type is np.string_ or data.dtype.type is np.unicode_):<EOL><INDENT>dt   = h5py.special_dtype(vlen=str)<EOL>data = data.astype(dt)<EOL><DEDENT>if isinstance(data, dict):<EOL><INDENT>for key in data:<EOL><INDENT>h5group[str(key)] = data[key]<EOL><DEDENT><DEDENT>elif isinstance(data, list):<EOL><INDENT>for idx, item in enumerate(data):<EOL><INDENT>h5group[str(idx)] = item<EOL><DEDENT><DEDENT>else:<EOL><INDENT>h5group[vn] = data<EOL><DEDENT><DEDENT>h5file.close()<EOL>

Parameters
----------
file_path: str

variables: dict
    Dictionary with objects. Object name -> object

mode: str
    HDF5 file access mode
    See h5py documentation for details.
    r   Readonly, file must exist
    r+  Read/write, file must exist
    w   Create file, truncate if exists
    w-  Create file, fail if exists
    a   Read/write if exists, create otherwise (default)

Notes
-----
It is recommended to use numpy arrays as objects.
List or tuples of strings won't work, convert them into numpy.arrays before.


if not op.exists(file_path):<EOL><INDENT>raise IOError('<STR_LIT>'.format(file_path))<EOL><DEDENT>try:<EOL><INDENT>h5file = h5py.File(file_path, mode=mode)<EOL><DEDENT>except:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>return h5file<EOL><DEDENT>

Return the h5py.File given its file path.

    Parameters
    ----------
    file_path: string
        HDF5 file path

    mode: string
        r   Readonly, file must exist
        r+  Read/write, file must exist
        w   Create file, truncate if exists
        w-  Create file, fail if exists
        a   Read/write if exists, create otherwise (default)

    Returns
    -------
    h5file: h5py.File


return _get_node_names(h5file, h5path, node_type=h5py.Group)<EOL>

Return the groups names within h5file/h5path

    Parameters
    ----------
    h5file: h5py.File or path to hdf5 file
        HDF5 file object

    h5path: str
        HDF5 group path to get the group names from

    Returns
    -------
    groupnames: list of str
        List of group names


return _get_node_names(h5file, h5path, node_type=h5py.Dataset)<EOL>

Return all dataset names from h5path group in h5file.

    Parameters
    ----------
    h5file: h5py.File
        HDF5 file object

    h5path: str
        HDF5 group path to read datasets from

    Returns
    -------
    dsnames: list of str
        List of dataset names contained in h5file/h5path


return _get_nodes(h5file, h5path, node_type=h5py.Dataset)<EOL>

Return all datasets from h5path group in file_path.

    Parameters
    ----------
    h5file: h5py.File
        HDF5 file object

    h5path: str
        HDF5 group path to read datasets from

    Returns
    -------
    datasets: dict
        Dict with all h5py.Dataset contained in file_path/h5path


if isinstance(h5file, str):<EOL><INDENT>_h5file = h5py.File(h5file, mode='<STR_LIT:r>')<EOL><DEDENT>else:<EOL><INDENT>_h5file = h5file<EOL><DEDENT>_datasets = get_datasets(_h5file, h5path)<EOL>datasets  = OrderedDict()<EOL>try:<EOL><INDENT>for ds in _datasets:<EOL><INDENT>datasets[ds.name.split('<STR_LIT:/>')[-<NUM_LIT:1>]] = ds[:]<EOL><DEDENT><DEDENT>except:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(_h5file.filename, h5path))<EOL><DEDENT>finally:<EOL><INDENT>if isinstance(h5file, str):<EOL><INDENT>_h5file.close()<EOL><DEDENT><DEDENT>return datasets<EOL>

Return all dataset contents from h5path group in h5file in an OrderedDict.

    Parameters
    ----------
    h5file: h5py.File
        HDF5 file object

    h5path: str
        HDF5 group path to read datasets from

    Returns
    -------
    datasets: OrderedDict
        Dict with variables contained in file_path/h5path


def _get_node_names(h5file, h5path='<STR_LIT:/>', node_type=h5py.Dataset):

if isinstance(h5file, str):<EOL><INDENT>_h5file = get_h5file(h5file, mode='<STR_LIT:r>')<EOL><DEDENT>else:<EOL><INDENT>_h5file = h5file<EOL><DEDENT>if not h5path.startswith('<STR_LIT:/>'):<EOL><INDENT>h5path = '<STR_LIT:/>' + h5path<EOL><DEDENT>names = []<EOL>try:<EOL><INDENT>h5group = _h5file.require_group(h5path)<EOL>for node in _hdf5_walk(h5group, node_type=node_type):<EOL><INDENT>names.append(node.name)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(_h5file.filename, h5path))<EOL><DEDENT>finally:<EOL><INDENT>if isinstance(h5file, str):<EOL><INDENT>_h5file.close()<EOL><DEDENT><DEDENT>return names<EOL>

Return the node of type node_type names within h5path of h5file.

    Parameters
    ----------
    h5file: h5py.File
        HDF5 file object

    h5path: str
        HDF5 group path to get the group names from

    node_type: h5py object type
        HDF5 object type

    Returns
    -------
    names: list of str
        List of names


def _get_nodes(h5file, h5path='<STR_LIT:/>', node_type=h5py.Dataset):

if isinstance(h5file, str):<EOL><INDENT>_h5file = get_h5file(h5file, mode='<STR_LIT:r>')<EOL><DEDENT>else:<EOL><INDENT>_h5file = h5file<EOL><DEDENT>if not h5path.startswith('<STR_LIT:/>'):<EOL><INDENT>h5path = '<STR_LIT:/>' + h5path<EOL><DEDENT>names = []<EOL>try:<EOL><INDENT>h5group = _h5file.require_group(h5path)<EOL>for node in _hdf5_walk(h5group, node_type=node_type):<EOL><INDENT>names.append(node)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(str(node_type), _h5file.filename, h5path))<EOL><DEDENT>finally:<EOL><INDENT>if isinstance(h5file, str):<EOL><INDENT>_h5file.close()<EOL><DEDENT><DEDENT>return names<EOL>

Return the nodes within h5path of the h5file.

    Parameters
    ----------
    h5file: h5py.File
        HDF5 file object

    h5path: str
        HDF5 group path to get the nodes from

    node_type: h5py object type
        The type of the nodes that you want to get

    Returns
    -------
    nodes: list of node_type objects


import shutil<EOL>if not os.path.exists(out_path):<EOL><INDENT>os.mkdir(out_path)<EOL><DEDENT>new_groups = defaultdict(list)<EOL>for group in dicom_sets:<EOL><INDENT>group_path = os.path.join(out_path, str(group))<EOL>os.mkdir(group_path)<EOL>group_dicoms = dicom_sets[group]<EOL>for idx, dcm in enumerate(group_dicoms):<EOL><INDENT>num = str(idx).zfill(<NUM_LIT:5>)<EOL>new_dcm = os.path.join(group_path, num + DICOM_FILE_EXTENSIONS[<NUM_LIT:0>].lower())<EOL>log.info('<STR_LIT>'.format(dcm, new_dcm))<EOL>shutil.copyfile(dcm, new_dcm)<EOL>new_groups[group].append(new_dcm)<EOL><DEDENT><DEDENT>return new_groups<EOL>

:param out_path: str
 Path to the output directory

:param dicom_sets: dict of {str: list of strs}
 Groups of dicom files


file_lst.sort()<EOL>c = <NUM_LIT:1><EOL>for f in file_lst:<EOL><INDENT>dirname = get_abspath(f.dirname())<EOL>fdest = f.joinpath(dirname, "<STR_LIT>".format(c) +<EOL>OUTPUT_DICOM_EXTENSION)<EOL>log.info('<STR_LIT>'.format(f, fdest))<EOL>f.rename(fdest)<EOL>c += <NUM_LIT:1><EOL><DEDENT>

Will rename all files in file_lst to a padded serial
    number plus its extension

    :param file_lst: list of path.py paths


self.items = []<EOL>if folders is not None:<EOL><INDENT>self._store_dicom_paths(folders)<EOL><DEDENT>

:param folders: str or list of strs
    Path or paths to folders to be searched for Dicom files


if isinstance(folders, str):<EOL><INDENT>folders = [folders]<EOL><DEDENT>for folder in folders:<EOL><INDENT>if not os.path.exists(folder):<EOL><INDENT>raise FolderNotFound(folder)<EOL><DEDENT>self.items.extend(list(find_all_dicom_files(folder)))<EOL><DEDENT>

Search for dicoms in folders and save file paths into
        self.dicom_paths set.

        :param folders: str or list of str


self.items = []<EOL>self._store_dicom_paths(folders)<EOL>

Restart the self.items and stores all dicom file paths found
within folders

Parameters
----------
folders: str or list of str


def from_set(self, fileset, check_if_dicoms=True):

if check_if_dicoms:<EOL><INDENT>self.items = []<EOL>for f in fileset:<EOL><INDENT>if is_dicom_file(f):<EOL><INDENT>self.items.append(f)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self.items = fileset<EOL><DEDENT>

Overwrites self.items with the given set of files.
        Will filter the fileset and keep only Dicom files.

        Parameters
        ----------
        fileset: iterable of str
        Paths to files

        check_if_dicoms: bool
        Whether to check if the items in fileset are dicom file paths


if not isinstance(dicomset, DicomFileSet):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>self.items = list(set(self.items).update(dicomset))<EOL>

Update this set with the union of itself and dicomset.

        Parameters
        ----------
        dicomset: DicomFileSet


def copy_files_to_other_folder(self, output_folder, rename_files=True,<EOL>mkdir=True, verbose=False):

import shutil<EOL>if not os.path.exists(output_folder):<EOL><INDENT>os.mkdir(output_folder)<EOL><DEDENT>if not rename_files:<EOL><INDENT>for dcmf in self.items:<EOL><INDENT>outf = os.path.join(output_folder, os.path.basename(dcmf))<EOL>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(dcmf, outf))<EOL><DEDENT>shutil.copyfile(dcmf, outf)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>n_pad = len(self.items)+<NUM_LIT:2><EOL>for idx, dcmf in enumerate(self.items):<EOL><INDENT>outf = '<STR_LIT>'.format(width=n_pad, number=idx)<EOL>outf = os.path.join(output_folder, outf)<EOL>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(dcmf, outf))<EOL><DEDENT>shutil.copyfile(dcmf, outf)<EOL><DEDENT><DEDENT>

Copies all files within this set to the output_folder

Parameters
----------
output_folder: str
Path of the destination folder of the files

rename_files: bool
Whether or not rename the files to a sequential format

mkdir: bool
Whether to make the folder if it does not exist

verbose: bool
Whether to print to stdout the files that are beind copied


def __init__(self, folders, read_metadata=True, header_fields=None):

DicomFileSet.__init__(self,  folders)<EOL>self.read_dcm = self.get_dcm_reader(read_metadata, header_fields)<EOL>

:param folders: str or list of strs
Path or paths to folders to be searched for Dicom files

:param read_metadata: bool
If True, will make a list of DicomFiles, otherwise will store
a simple DICOM header (namedtuples) with the fields specified
in header_fields.

:param header_fields: set of strings
Set of header fields to be read from each DICOM file in a DicomHeader.
If store_metadata is False, this won't be used. Else and if this is
None, will store the whole DicomFile.


if not store_metadata:<EOL><INDENT>return lambda fpath: fpath<EOL><DEDENT>if header_fields is None:<EOL><INDENT>build_dcm = lambda fpath: DicomFile(fpath)<EOL><DEDENT>else:<EOL><INDENT>dicom_header = namedtuple('<STR_LIT>', header_fields)<EOL>build_dcm = lambda fpath: dicom_header._make(DicomFile(fpath).get_attributes(header_fields))<EOL><DEDENT>return build_dcm<EOL>

Creates a lambda function to read DICOM files.
If store_store_metadata is False, will only return the file path.
Else if you give header_fields, will return only the set of of
header_fields within a DicomFile object or the whole DICOM file if
None.

:return: function
This function has only one parameter: file_path


try:<EOL><INDENT>for dcmf in self.items:<EOL><INDENT>yield self.read_dcm(dcmf)<EOL><DEDENT><DEDENT>except IOError as ioe:<EOL><INDENT>raise IOError('<STR_LIT>'.format(dcmf)) from ioe<EOL><DEDENT>

Generator that yields one by one the return value for self.read_dcm
for each file within this set


dist = SimpleDicomFileDistance(field_weights=header_fields)<EOL>path_list = dicom_file_paths.copy()<EOL>path_groups = DefaultOrderedDict(DicomFileSet)<EOL>while len(path_list) > <NUM_LIT:0>:<EOL><INDENT>file_path1 = path_list.pop()<EOL>file_subgroup = [file_path1]<EOL>dist.set_dicom_file1(file_path1)<EOL>j = len(path_list)-<NUM_LIT:1><EOL>while j >= <NUM_LIT:0>:<EOL><INDENT>file_path2 = path_list[j]<EOL>dist.set_dicom_file2(file_path2)<EOL>if dist.transform():<EOL><INDENT>file_subgroup.append(file_path2)<EOL>path_list.pop(j)<EOL><DEDENT>j -= <NUM_LIT:1><EOL><DEDENT>path_groups[file_path1].from_set(file_subgroup, check_if_dicoms=False)<EOL><DEDENT>return path_groups<EOL>

Gets a list of DICOM file absolute paths and returns a list of lists of
DICOM file paths. Each group contains a set of DICOM files that have
exactly the same headers.

Parameters
----------
dicom_file_paths: list of str
    List or set of DICOM file paths

header_fields: list of str
    List of header field names to check on the comparisons of the DICOM files.

Returns
-------
dict of DicomFileSets
    The key is one filepath representing the group (the first found).


def copy_groups_to_folder(dicom_groups, folder_path, groupby_field_name):

if dicom_groups is None or not dicom_groups:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if not os.path.exists(folder_path):<EOL><INDENT>os.makedirs(folder_path, exist_ok=False)<EOL><DEDENT>for dcmg in dicom_groups:<EOL><INDENT>if groupby_field_name is not None and len(groupby_field_name) > <NUM_LIT:0>:<EOL><INDENT>dfile = DicomFile(dcmg)<EOL>dir_name = '<STR_LIT>'<EOL>for att in groupby_field_name:<EOL><INDENT>dir_name = os.path.join(dir_name, dfile.get_attributes(att))<EOL><DEDENT>dir_name = str(dir_name)<EOL><DEDENT>else:<EOL><INDENT>dir_name = os.path.basename(dcmg)<EOL><DEDENT>group_folder = os.path.join(folder_path, dir_name)<EOL>os.makedirs(group_folder, exist_ok=False)<EOL>log.debug('<STR_LIT>'.format(group_folder))<EOL>import shutil<EOL>dcm_files = dicom_groups[dcmg]<EOL>for srcf in dcm_files:<EOL><INDENT>destf = os.path.join(group_folder, os.path.basename(srcf))<EOL>while os.path.exists(destf):<EOL><INDENT>destf += '<STR_LIT:+>'<EOL><DEDENT>shutil.copy2(srcf, destf)<EOL><DEDENT><DEDENT>

Copy the DICOM file groups to folder_path. Each group will be copied into
    a subfolder with named given by groupby_field.

    Parameters
    ----------
    dicom_groups: boyle.dicom.sets.DicomFileSet

    folder_path: str
     Path to where copy the DICOM files.

    groupby_field_name: str
     DICOM field name. Will get the value of this field to name the group
     folder.


def calculate_file_distances(dicom_files, field_weights=None,<EOL>dist_method_cls=None, **kwargs):

if dist_method_cls is None:<EOL><INDENT>dist_method = LevenshteinDicomFileDistance(field_weights)<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>dist_method = dist_method_cls(field_weights=field_weights, **kwargs)<EOL><DEDENT>except:<EOL><INDENT>log.exception('<STR_LIT>'<EOL>'<STR_LIT>'.format(dist_method_cls, kwargs))<EOL><DEDENT><DEDENT>dist_dtype = np.float16<EOL>n_files = len(dicom_files)<EOL>try:<EOL><INDENT>file_dists = np.zeros((n_files, n_files), dtype=dist_dtype)<EOL><DEDENT>except MemoryError as mee:<EOL><INDENT>import scipy.sparse<EOL>file_dists = scipy.sparse.lil_matrix((n_files, n_files),<EOL>dtype=dist_dtype)<EOL><DEDENT>for idxi in range(n_files):<EOL><INDENT>dist_method.set_dicom_file1(dicom_files[idxi])<EOL>for idxj in range(idxi+<NUM_LIT:1>, n_files):<EOL><INDENT>dist_method.set_dicom_file2(dicom_files[idxj])<EOL>if idxi != idxj:<EOL><INDENT>file_dists[idxi, idxj] = dist_method.transform()<EOL><DEDENT><DEDENT><DEDENT>return file_dists<EOL>

Calculates the DicomFileDistance between all files in dicom_files, using an
weighted Levenshtein measure between all field names in field_weights and
their corresponding weights.

Parameters
----------
dicom_files: iterable of str
    Dicom file paths

field_weights: dict of str to float
    A dict with header field names to float scalar values, that
    indicate a distance measure ratio for the levenshtein distance
    averaging of all the header field names in it. e.g., {'PatientID': 1}

dist_method_cls: DicomFileDistance class
    Distance method object to compare the files.
    If None, the default DicomFileDistance method using Levenshtein
    distance between the field_wieghts will be used.

kwargs: DicomFileDistance instantiation named arguments
    Apart from the field_weitghts argument.

Returns
-------
file_dists: np.ndarray or scipy.sparse.lil_matrix of shape NxN
    Levenshtein distances between each of the N items in dicom_files.


samples, targets = check_X_y(samples, targets, ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>if not callable(self.score_func):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>% (self.score_func, type(self.score_func)))<EOL><DEDENT>self._check_params(samples, targets)<EOL>self.scores_ = np.asarray(self.score_func(samples, targets))<EOL>return self<EOL>

Parameters
----------
samples: array-like, shape = [n_samples, n_features]
    The training input samples.

targets: array-like, shape = [n_samples]
    The target values (class labels in classification, real numbers in
    regression).

Returns
-------
self : object
    Returns self.


self.set_dicom_file1(dcm_file1)<EOL>self.set_dicom_file2(dcm_file2)<EOL>

Parameters
----------
dcm_file1: str (path to file) or DicomFile or namedtuple

dcm_file2: str (path to file) or DicomFile or namedtuple


self.dcmf1 = self._read_dcmfile(dcm_file)<EOL>

Parameters
----------
dcm_file: str (path to file) or DicomFile or namedtuple


self.dcmf2 = self._read_dcmfile(dcm_file)<EOL>


if isinstance(dcm_file, str):<EOL><INDENT>return DicomFile(dcm_file)<EOL><DEDENT>else:<EOL><INDENT>return dcm_file<EOL><DEDENT>

Parameters
----------
dcm_file: str or DicomFile
    File path or DicomFile

Returns
-------
DicomFile


if self.dcmf1 is None or self.dcmf2 is None:<EOL><INDENT>return np.inf<EOL><DEDENT>for field_name in self.field_weights:<EOL><INDENT>if (str(getattr(self.dcmf1, field_name, '<STR_LIT>'))<EOL>!= str(getattr(self.dcmf2, field_name, '<STR_LIT>'))):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Check the field values in self.dcmf1 and self.dcmf2 and returns True
        if all the field values are the same, False otherwise.

        Returns
        -------
        bool


if field_weights is None:<EOL><INDENT>if not isinstance(self.field_weights, dict):<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'.format(type(self.field_weights)))<EOL><DEDENT><DEDENT>key_dicoms = list(self.dicom_groups.keys())<EOL>file_dists = calculate_file_distances(key_dicoms, field_weights, self._dist_method_cls)<EOL>return file_dists<EOL>

Updates the status of the file clusters comparing the cluster
key files with a levenshtein weighted measure using either the
header_fields or self.header_fields.

Parameters
----------
field_weights: dict of strings with floats
    A dict with header field names to float scalar values, that indicate a distance measure
    ratio for the levenshtein distance averaging of all the header field names in it.
    e.g., {'PatientID': 1}


@staticmethod<EOL><INDENT>def dist_percentile_threshold(dist_matrix, perc_thr=<NUM_LIT>, k=<NUM_LIT:1>):<DEDENT>

triu_idx = np.triu_indices(dist_matrix.shape[<NUM_LIT:0>], k=k)<EOL>upper = np.zeros_like(dist_matrix)<EOL>upper[triu_idx] = dist_matrix[triu_idx] < np.percentile(dist_matrix[triu_idx], perc_thr)<EOL>return upper<EOL>

Thresholds a distance matrix and returns the result.

        Parameters
        ----------

        dist_matrix: array_like
        Input array or object that can be converted to an array.

        perc_thr: float in range of [0,100]
        Percentile to compute which must be between 0 and 100 inclusive.

        k: int, optional
        Diagonal above which to zero elements.
        k = 0 (the default) is the main diagonal,
        k < 0 is below it and k > 0 is above.

        Returns
        -------
        array_like


group_pairs = []<EOL>key_dicoms = list(self.dicom_groups.keys())<EOL>idx = len(key_dicoms)<EOL>while idx > <NUM_LIT:0>:<EOL><INDENT>group1 = key_dicoms.pop()<EOL>dir_group1 = get_folder_subpath(group1, folder_depth)<EOL>for group in key_dicoms:<EOL><INDENT>if group.startswith(dir_group1):<EOL><INDENT>group_pairs.append((group1, group))<EOL><DEDENT><DEDENT>idx -= <NUM_LIT:1><EOL><DEDENT>return group_pairs<EOL>

Returns a list of 2-tuples with pairs of dicom groups that
are in the same folder within given depth.

Parameters
----------
folder_depth: int
Path depth to check for folder equality.

Returns
-------
list of tuples of str


import matplotlib.pyplot as plt<EOL>fig = plt.figure()<EOL>ax = fig.add_subplot(<NUM_LIT>)<EOL>ax.matshow(dist_matrix, interpolation='<STR_LIT>',<EOL>cmap=plt.cm.get_cmap('<STR_LIT>'))<EOL>

Plots dist_matrix

Parameters
----------
dist_matrix: np.ndarray


try:<EOL><INDENT>merged = merge_dict_of_lists(self.dicom_groups, indices,<EOL>pop_later=True, copy=True)<EOL>self.dicom_groups = merged<EOL><DEDENT>except IndexError:<EOL><INDENT>raise IndexError('<STR_LIT>')<EOL><DEDENT>

Extend the lists within the DICOM groups dictionary.
        The indices will indicate which list have to be extended by which
        other list.

        Parameters
        ----------
        indices: list or tuple of 2 iterables of int, bot having the same len
             The indices of the lists that have to be merged, both iterables
             items will be read pair by pair, the first is the index to the
             list that will be extended with the list of the second index.
             The indices can be constructed with Numpy e.g.,
             indices = np.where(square_matrix)


def move_to_folder(self, folder_path, groupby_field_name=None):

try:<EOL><INDENT>copy_groups_to_folder(self.dicom_groups, folder_path, groupby_field_name)<EOL><DEDENT>except IOError as ioe:<EOL><INDENT>raise IOError('<STR_LIT>'.format(folder_path)) from ioe<EOL><DEDENT>

Copy the file groups to folder_path. Each group will be copied into
        a subfolder with named given by groupby_field.

        Parameters
        ----------
        folder_path: str
         Path to where copy the DICOM files.

        groupby_field_name: str
         DICOM field name. Will get the value of this field to name the group
         folder. If empty or None will use the basename of the group key file.


unique_vals = DefaultOrderedDict(set)<EOL>for dcmg in self.dicom_groups:<EOL><INDENT>for f in self.dicom_groups[dcmg]:<EOL><INDENT>field_val = DicomFile(f).get_attributes(field_name)<EOL>key_val = dcmg<EOL>if field_to_use_as_key is not None:<EOL><INDENT>try:<EOL><INDENT>key_val = str(DicomFile(dcmg).get_attributes(field_to_use_as_key))<EOL><DEDENT>except KeyError as ke:<EOL><INDENT>raise KeyError('<STR_LIT>'<EOL>'<STR_LIT>'.format(field_to_use_as_key,<EOL>dcmg)) from ke<EOL><DEDENT><DEDENT>unique_vals[key_val].add(field_val)<EOL><DEDENT><DEDENT>return unique_vals<EOL>

Return a dictionary where the key is the group key file path and
        the values are sets of unique values of the field name of all DICOM
        files in the group.

        Parameters
        ----------
        field_name: str
         Name of the field to read from all files

        field_to_use_as_key: str
         Name of the field to get the value and use as key.
         If None, will use the same key as the dicom_groups.

        Returns
        -------
        Dict of sets


if not op.isdir(output_directory):<EOL><INDENT>os.makedirs(output_directory)<EOL><DEDENT>config_file = op.join(output_directory, "<STR_LIT>")<EOL>open_file = open(config_file, "<STR_LIT:w>")<EOL>open_file.write("<STR_LIT>")<EOL>open_file.close()<EOL>return config_file<EOL>

Generate a dcm2nii configuration file that disable the interactive
    mode.


def add_meta_to_nii(nii_file, dicom_file, dcm_tags='<STR_LIT>'):

<EOL>dcmimage = dicom.read_file(dicom_file)<EOL>image = nibabel.load(nii_file)<EOL>if not isinstance(image, nibabel.nifti1.Nifti1Image):<EOL><INDENT>raise Exception(<EOL>"<STR_LIT>".format(<EOL>type(image)))<EOL><DEDENT>if isinstance(dcm_tags, str):<EOL><INDENT>dcm_tags = [dcm_tags]<EOL><DEDENT>header = image.get_header()<EOL>repetition_time = float(dcmimage[("<STR_LIT>", "<STR_LIT>")].value)<EOL>header.set_dim_info(slice=<NUM_LIT:2>)<EOL>nb_slices = header.get_n_slices()<EOL>slice_duration = round(repetition_time / nb_slices, <NUM_LIT:0>)<EOL>header.set_slice_duration(slice_duration)<EOL>if dcm_tags:<EOL><INDENT>content = ["<STR_LIT>".format(name, dcmimage[tag].value)<EOL>for name, tag in dcm_tags]<EOL>free_field = numpy.array("<STR_LIT:;>".join(content),<EOL>dtype=header["<STR_LIT>"].dtype)<EOL>image.get_header()["<STR_LIT>"] = free_field<EOL><DEDENT>image.update_header()<EOL>nibabel.save(image, nii_file)<EOL>

Add slice duration and acquisition times to the headers of the nifit1 files in `nii_file`.
    It will add the repetition time of the DICOM file (field: {0x0018, 0x0080, DS, Repetition Time})
    to the NifTI file as well as any other tag in `dcm_tags`.
    All selected DICOM tags values are set in the `descrip` nifti header field.
    Note that this will modify the header content of `nii_file`.

    Parameters
    ----------
    nii_files: str
        Path to the NifTI file to modify.

    dicom_file: str
        Paths to the DICOM file from where to get the meta data.

    dcm_tags: list of str
        List of tags from the DICOM file to read and store in the nifti file.


if not op.exists(work_dir):<EOL><INDENT>raise IOError('<STR_LIT>'.format(work_dir))<EOL><DEDENT>cmd_line = '<STR_LIT>'.format(arguments, work_dir)<EOL>log.info(cmd_line)<EOL>return subprocess.check_call(cmd_line, shell=True)<EOL>

Converts all DICOM files within `work_dir` into one or more
    NifTi files by calling dcm2nii on this folder.

    Parameters
    ----------
    work_dir: str
        Path to the folder that contain the DICOM files

    arguments: str
        String containing all the flag arguments for `dcm2nii` CLI.

    Returns
    -------
    sys_code: int
        dcm2nii execution return code


def convert_dcm2nii(input_dir, output_dir, filename):

<EOL>if not op.exists(input_dir):<EOL><INDENT>raise IOError('<STR_LIT>'.format(input_dir))<EOL><DEDENT>if not op.exists(output_dir):<EOL><INDENT>raise IOError('<STR_LIT>'.format(output_dir))<EOL><DEDENT>tmpdir = tempfile.TemporaryDirectory(prefix='<STR_LIT>')<EOL>arguments = '<STR_LIT>'.format(tmpdir.name)<EOL>try:<EOL><INDENT>call_out = call_dcm2nii(input_dir, arguments)<EOL><DEDENT>except:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>log.info('<STR_LIT>'.format(input_dir))<EOL>filenames  = glob(op.join(tmpdir.name, '<STR_LIT>'))<EOL>cleaned_filenames = remove_dcm2nii_underprocessed(filenames)<EOL>filepaths = []<EOL>for srcpath in cleaned_filenames:<EOL><INDENT>dstpath = op.join(output_dir, filename)<EOL>realpath = copy_w_plus(srcpath, dstpath)<EOL>filepaths.append(realpath)<EOL>basename = op.basename(remove_ext(srcpath))<EOL>aux_files = set(glob(op.join(tmpdir.name, '<STR_LIT>'     .format(basename)))) -set(glob(op.join(tmpdir.name, '<STR_LIT>'.format(basename))))<EOL>for aux_file in aux_files:<EOL><INDENT>aux_dstpath = copy_w_ext(aux_file, output_dir, remove_ext(op.basename(realpath)))<EOL>filepaths.append(aux_dstpath)<EOL><DEDENT><DEDENT>return filepaths<EOL><DEDENT>

Call MRICron's `dcm2nii` to convert the DICOM files inside `input_dir`
    to Nifti and save the Nifti file in `output_dir` with a `filename` prefix.

    Parameters
    ----------
    input_dir: str
        Path to the folder that contains the DICOM files

    output_dir: str
        Path to the folder where to save the NifTI file

    filename: str
        Output file basename

    Returns
    -------
    filepaths: list of str
        List of file paths created in `output_dir`.


cln_flist = []<EOL>len_sorted = sorted(filepaths, key=len)<EOL>for idx, fpath in enumerate(len_sorted):<EOL><INDENT>remove = False<EOL>fname = op.basename(fpath)<EOL>rest  = len_sorted[idx+<NUM_LIT:1>:]<EOL>for rest_fpath in rest:<EOL><INDENT>rest_file = op.basename(rest_fpath)<EOL>if rest_file.endswith(fname):<EOL><INDENT>remove = True<EOL>break<EOL><DEDENT><DEDENT>if not remove:<EOL><INDENT>cln_flist.append(fpath)<EOL><DEDENT><DEDENT>return cln_flist<EOL>

Return a subset of `filepaths`. Keep only the files that have a basename longer than the
    others with same suffix.
    This works based on that dcm2nii appends a preffix character for each processing
    step it does automatically in the DICOM to NifTI conversion.

    Parameters
    ----------
    filepaths: iterable of str

    Returns
    -------
    cleaned_paths: iterable of str


num_elems = len(iterable)<EOL>for i in range(num_elems, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>for c in combinations(iterable, i):<EOL><INDENT>yield c<EOL><DEDENT><DEDENT>

Generate all combinations of the elements of iterable and its subsets.

Parameters
----------
iterable: list, set or dict or any iterable object

Returns
-------
A generator of all possible combinations of the iterable.

Example:
-------
>>> for i in treefall([1, 2, 3, 4, 5]): print(i)
>>> (1, 2, 3)
>>> (1, 2)
>>> (1, 3)
>>> (2, 3)
>>> (1,)
>>> (2,)
>>> (3,)
>>> ()


field_values = set()<EOL>for dcm in dcm_file_list:<EOL><INDENT>field_values.add(str(DicomFile(dcm).get_attributes(field_name)))<EOL><DEDENT>return field_values<EOL>

Return a set of unique field values from a list of DICOM files

    Parameters
    ----------
    dcm_file_list: iterable of DICOM file paths

    field_name: str
     Name of the field from where to get each value

    Returns
    -------
    Set of field values


dicoms = set()<EOL>try:<EOL><INDENT>for fpath in get_all_files(root_path):<EOL><INDENT>if is_dicom_file(fpath):<EOL><INDENT>dicoms.add(fpath)<EOL><DEDENT><DEDENT><DEDENT>except IOError as ioe:<EOL><INDENT>raise IOError('<STR_LIT>'.format(fpath)) from ioe<EOL><DEDENT>return dicoms<EOL>

Returns a list of the dicom files within root_path

Parameters
----------
root_path: str
Path to the directory to be recursively searched for DICOM files.

Returns
-------
dicoms: set
Set of DICOM absolute file paths


if not os.path.exists(filepath):<EOL><INDENT>raise IOError('<STR_LIT>'.format(filepath))<EOL><DEDENT>filename = os.path.basename(filepath)<EOL>if filename == '<STR_LIT>':<EOL><INDENT>return False<EOL><DEDENT>try:<EOL><INDENT>_ = dicom.read_file(filepath)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>log.debug('<STR_LIT>'<EOL>'<STR_LIT>'.format(filepath))<EOL>return False<EOL><DEDENT>return True<EOL>

Tries to read the file using dicom.read_file,
if the file exists and dicom.read_file does not raise
and Exception returns True. False otherwise.

:param filepath: str
 Path to DICOM file

:return: bool


dicom_groups = defaultdict(list)<EOL>try:<EOL><INDENT>for dcm in dicom_paths:<EOL><INDENT>hdr = dicom.read_file(dcm)<EOL>group_key = getattr(hdr, hdr_field)<EOL>dicom_groups[group_key].append(dcm)<EOL><DEDENT><DEDENT>except KeyError as ke:<EOL><INDENT>raise KeyError('<STR_LIT>'.format(hdr_field, dcm)) from ke<EOL><DEDENT>return dicom_groups<EOL>

Group in a dictionary all the DICOM files in dicom_paths
    separated by the given `hdr_field` tag value.

    Parameters
    ----------
    dicom_paths: str
        Iterable of DICOM file paths.

    hdr_field: str
        Name of the DICOM tag whose values will be used as key for the group.

    Returns
    -------
    dicom_groups: dict of dicom_paths


dcmfiles = sorted(recursive_glob(input_dir, dcm_pattern))<EOL>for dcm in dcmfiles:<EOL><INDENT>cmd = '<STR_LIT>'.format(dcm)<EOL>log.debug('<STR_LIT>'.format(cmd))<EOL>subprocess.check_call(cmd, shell=True)<EOL><DEDENT>

Decompress all *.dcm files recursively found in DICOM_DIR.
    This uses 'gdcmconv --raw'.
    It works when 'dcm2nii' shows the `Unsupported Transfer Syntax` error. This error is
    usually caused by lack of JPEG2000 support in dcm2nii compilation.

    Read more:
    http://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#Transfer_Syntaxes_and_Compressed_Images

    Parameters
    ----------
    input_dir: str
        Folder path

    dcm_patther: str
        Pattern of the DICOM file names in `input_dir`.

    Notes
    -----
    The *.dcm files in `input_folder` will be overwritten.


def get_attributes(self, attributes, default='<STR_LIT>'):

if isinstance(attributes, str):<EOL><INDENT>attributes = [attributes]<EOL><DEDENT>attrs = [getattr(self, attr, default) for attr in attributes]<EOL>if len(attrs) == <NUM_LIT:1>:<EOL><INDENT>return attrs[<NUM_LIT:0>]<EOL><DEDENT>return tuple(attrs)<EOL>

Return the attributes values from this DicomFile

        Parameters
        ----------
        attributes: str or list of str
         DICOM field names

        default: str
         Default value if the attribute does not exist.

        Returns
        -------
        Value of the field or list of values.


def as_ndarray(arr, copy=False, dtype=None, order='<STR_LIT>'):

if order not in ('<STR_LIT:C>', '<STR_LIT:F>', '<STR_LIT:A>', '<STR_LIT>', None):<EOL><INDENT>raise ValueError("<STR_LIT>".format(str(order)))<EOL><DEDENT>if isinstance(arr, np.memmap):<EOL><INDENT>if dtype is None:<EOL><INDENT>if order in ('<STR_LIT>', '<STR_LIT:A>', None):<EOL><INDENT>ret = np.array(np.asarray(arr), copy=True)<EOL><DEDENT>else:<EOL><INDENT>ret = np.array(np.asarray(arr), copy=True, order=order)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if order in ('<STR_LIT>', '<STR_LIT:A>', None):<EOL><INDENT>ret = np.asarray(arr).astype(dtype)<EOL><DEDENT>else:<EOL><INDENT>ret = _asarray(np.array(arr, copy=True), dtype=dtype, order=order)<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(arr, np.ndarray):<EOL><INDENT>ret = _asarray(arr, dtype=dtype, order=order)<EOL>if np.may_share_memory(ret, arr) and copy:<EOL><INDENT>ret = ret.T.copy().T if ret.flags['<STR_LIT>'] else ret.copy()<EOL><DEDENT><DEDENT>elif isinstance(arr, (list, tuple)):<EOL><INDENT>if order in ("<STR_LIT:A>", "<STR_LIT>"):<EOL><INDENT>ret = np.asarray(arr, dtype=dtype)<EOL><DEDENT>else:<EOL><INDENT>ret = np.asarray(arr, dtype=dtype, order=order)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>".format(arr.__class__))<EOL><DEDENT>return ret<EOL>

Convert an arbitrary array to numpy.ndarray.

    In the case of a memmap array, a copy is automatically made to break the
    link with the underlying file (whatever the value of the "copy" keyword).

    The purpose of this function is mainly to get rid of memmap objects, but
    it can be used for other purposes. In particular, combining copying and
    casting can lead to performance improvements in some cases, by avoiding
    unnecessary copies.

    If not specified, input array order is preserved, in all cases, even when
    a copy is requested.

    Caveat: this function does not copy during bool to/from 1-byte dtype
    conversions. This can lead to some surprising results in some rare cases.
    Example:

        a = numpy.asarray([0, 1, 2], dtype=numpy.int8)
        b = as_ndarray(a, dtype=bool)  # array([False, True, True], dtype=bool)
        c = as_ndarray(b, dtype=numpy.int8)  # array([0, 1, 2], dtype=numpy.int8)

    The usually expected result for the last line would be array([0, 1, 1])
    because True evaluates to 1. Since there is no copy made here, the original
    array is recovered.

    Parameters
    ----------
    arr: array-like
        input array. Any value accepted by numpy.asarray is valid.

    copy: bool
        if True, force a copy of the array. Always True when arr is a memmap.

    dtype: any numpy dtype
        dtype of the returned array. Performing copy and type conversion at the
        same time can in some cases avoid an additional copy.

    order: string
        gives the order of the returned array.
        Valid values are: "C", "F", "A", "K", None.
        default is "K". See ndarray.copy() for more information.

    Returns
    -------
    ret: np.ndarray
        Numpy array containing the same data as arr, always of class
        numpy.ndarray, and with no link to any underlying file.


import sys<EOL>if sys.version_info.major == <NUM_LIT:3>:<EOL><INDENT>import importlib.machinery<EOL>loader = importlib.machinery.SourceFileLoader('<STR_LIT>', filepath)<EOL>mod = loader.load_module(mod_name)<EOL><DEDENT>else:<EOL><INDENT>import imp<EOL>mod = imp.load_source(mod_name, filepath)<EOL><DEDENT>return mod<EOL>

Imports the contents of filepath as a Python module.

:param filepath: string

:param mod_name: string
Name of the module when imported

:return: module
Imported module


X = np.asanyarray(X)<EOL>if (X.dtype.char in np.typecodes['<STR_LIT>'] and not np.isfinite(X.sum())<EOL>and not np.isfinite(X).all()):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>" % X.dtype)<EOL><DEDENT>

Like assert_all_finite, but only for ndarray.


<EOL>_assert_all_finite(X.data if sp.issparse(X) else X)<EOL>

Throw a ValueError if X contains NaN or infinity.

    Input MUST be an np.ndarray instance or a scipy.sparse matrix.


def as_float_array(X, copy=True, force_all_finite=True):

if isinstance(X, np.matrix) or (not isinstance(X, np.ndarray)<EOL>and not sp.issparse(X)):<EOL><INDENT>return check_array(X, ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'], dtype=np.float64,<EOL>copy=copy, force_all_finite=force_all_finite,<EOL>ensure_2d=False)<EOL><DEDENT>elif sp.issparse(X) and X.dtype in [np.float32, np.float64]:<EOL><INDENT>return X.copy() if copy else X<EOL><DEDENT>elif X.dtype in [np.float32, np.float64]:  <EOL><INDENT>return X.copy('<STR_LIT:F>' if X.flags['<STR_LIT>'] else '<STR_LIT:C>') if copy else X<EOL><DEDENT>else:<EOL><INDENT>return X.astype(np.float32 if X.dtype == np.int32 else np.float64)<EOL><DEDENT>

Converts an array-like to an array of floats

    The new dtype will be np.float32 or np.float64, depending on the original
    type. The function can create a copy or modify the argument depending
    on the argument copy.

    Parameters
    ----------
    X : {array-like, sparse matrix}

    copy : bool, optional
        If True, a copy of X will be created. If False, a copy may still be
        returned if X's dtype is not a floating point type.

    Returns
    -------
    XT : {array, sparse matrix}
        An array of type np.float


if not hasattr(x, '<STR_LIT>') and not hasattr(x, '<STR_LIT>'):<EOL><INDENT>if hasattr(x, '<STR_LIT>'):<EOL><INDENT>x = np.asarray(x)<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % x)<EOL><DEDENT><DEDENT>return x.shape[<NUM_LIT:0>] if hasattr(x, '<STR_LIT>') else len(x)<EOL>

Return number of samples in array-like x.


uniques = np.unique([_num_samples(X) for X in arrays if X is not None])<EOL>if len(uniques) > <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>% str(uniques))<EOL><DEDENT>

Check that all arrays have consistent first dimensions.

    Checks whether all objects in arrays have the same shape or length.

    Parameters
    ----------
    arrays : list or tuple of input objects.
        Objects that will be checked for consistent length.


result = []<EOL>for X in iterables:<EOL><INDENT>if sp.issparse(X):<EOL><INDENT>result.append(X.tocsr())<EOL><DEDENT>elif hasattr(X, "<STR_LIT>") or hasattr(X, "<STR_LIT>"):<EOL><INDENT>result.append(X)<EOL><DEDENT>elif X is None:<EOL><INDENT>result.append(X)<EOL><DEDENT>else:<EOL><INDENT>result.append(np.array(X))<EOL><DEDENT><DEDENT>check_consistent_length(*result)<EOL>return result<EOL>

Make arrays indexable for cross-validation.

    Checks consistent length, passes through None, and ensures that everything
    can be indexed by converting sparse matrices to csr and converting
    non-interable objects to arrays.

    Parameters
    ----------
    iterables : lists, dataframes, arrays, sparse matrices
        List of objects to ensure sliceability.


def _ensure_sparse_format(spmatrix, accept_sparse, dtype, order, copy,<EOL>force_all_finite):

if accept_sparse is None:<EOL><INDENT>raise TypeError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>sparse_type = spmatrix.format<EOL>if dtype is None:<EOL><INDENT>dtype = spmatrix.dtype<EOL><DEDENT>if sparse_type in accept_sparse:<EOL><INDENT>if dtype == spmatrix.dtype:<EOL><INDENT>if copy:<EOL><INDENT>spmatrix = spmatrix.copy()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>spmatrix = spmatrix.astype(dtype)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>spmatrix = spmatrix.asformat(accept_sparse[<NUM_LIT:0>]).astype(dtype)<EOL><DEDENT>if force_all_finite:<EOL><INDENT>if not hasattr(spmatrix, "<STR_LIT:data>"):<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>% spmatrix.format)<EOL><DEDENT>else:<EOL><INDENT>_assert_all_finite(spmatrix.data)<EOL><DEDENT><DEDENT>if hasattr(spmatrix, "<STR_LIT:data>"):<EOL><INDENT>spmatrix.data = np.array(spmatrix.data, copy=False, order=order)<EOL><DEDENT>return spmatrix<EOL>

Convert a sparse matrix to a given format.

    Checks the sparse format of spmatrix and converts if necessary.

    Parameters
    ----------
    spmatrix : scipy sparse matrix
        Input to validate and convert.

    accept_sparse : string, list of string or None (default=None)
        String[s] representing allowed sparse matrix formats ('csc',
        'csr', 'coo', 'dok', 'bsr', 'lil', 'dia'). None means that sparse
        matrix input will raise an error.  If the input is sparse but not in
        the allowed format, it will be converted to the first listed format.

    order : 'F', 'C' or None (default=None)
        Whether an array will be forced to be fortran or c-style.

    copy : boolean (default=False)
        Whether a forced copy will be triggered. If copy=False, a copy might
        be triggered by a conversion.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X.

    Returns
    -------
    spmatrix_converted : scipy sparse matrix.
        Matrix that is ensured to have an allowed type.


def check_array(array, accept_sparse=None, dtype=None, order=None, copy=False,<EOL>force_all_finite=True, ensure_2d=True, allow_nd=False):

if isinstance(accept_sparse, str):<EOL><INDENT>accept_sparse = [accept_sparse]<EOL><DEDENT>if sp.issparse(array):<EOL><INDENT>array = _ensure_sparse_format(array, accept_sparse, dtype, order,<EOL>copy, force_all_finite)<EOL><DEDENT>else:<EOL><INDENT>if ensure_2d:<EOL><INDENT>array = np.atleast_2d(array)<EOL><DEDENT>array = np.array(array, dtype=dtype, order=order, copy=copy)<EOL>if not allow_nd and array.ndim >= <NUM_LIT:3>:<EOL><INDENT>raise ValueError("<STR_LIT>" %<EOL>array.ndim)<EOL><DEDENT>if force_all_finite:<EOL><INDENT>_assert_all_finite(array)<EOL><DEDENT><DEDENT>return array<EOL>

Input validation on an array, list, sparse matrix or similar.

    By default, the input is converted to an at least 2nd numpy array.

    Parameters
    ----------
    array : object
        Input object to check / convert.

    accept_sparse : string, list of string or None (default=None)
        String[s] representing allowed sparse matrix formats, such as 'csc',
        'csr', etc.  None means that sparse matrix input will raise an error.
        If the input is sparse but not in the allowed format, it will be
        converted to the first listed format.

    order : 'F', 'C' or None (default=None)
        Whether an array will be forced to be fortran or c-style.

    copy : boolean (default=False)
        Whether a forced copy will be triggered. If copy=False, a copy might
        be triggered by a conversion.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X.

    ensure_2d : boolean (default=True)
        Whether to make X at least 2d.

    allow_nd : boolean (default=False)
        Whether to allow X.ndim > 2.

    Returns
    -------
    X_converted : object
        The converted and validated X.


def check_X_y(X, y, accept_sparse=None, dtype=None, order=None, copy=False,<EOL>force_all_finite=True, ensure_2d=True, allow_nd=False,<EOL>multi_output=False):

X = check_array(X, accept_sparse, dtype, order, copy, force_all_finite,<EOL>ensure_2d, allow_nd)<EOL>if multi_output:<EOL><INDENT>y = check_array(y, '<STR_LIT>', force_all_finite=True, ensure_2d=False)<EOL><DEDENT>else:<EOL><INDENT>y = column_or_1d(y, warn=True)<EOL>_assert_all_finite(y)<EOL><DEDENT>check_consistent_length(X, y)<EOL>return X, y<EOL>

Input validation for standard estimators.

    Checks X and y for consistent length, enforces X 2d and y 1d.
    Standard input checks are only applied to y. For multi-label y,
    set multi_ouput=True to allow 2d and sparse y.

    Parameters
    ----------
    X : nd-array, list or sparse matrix
        Input data.

    y : nd-array, list or sparse matrix
        Labels.

    accept_sparse : string, list of string or None (default=None)
        String[s] representing allowed sparse matrix formats, such as 'csc',
        'csr', etc.  None means that sparse matrix input will raise an error.
        If the input is sparse but not in the allowed format, it will be
        converted to the first listed format.

    order : 'F', 'C' or None (default=None)
        Whether an array will be forced to be fortran or c-style.

    copy : boolean (default=False)
        Whether a forced copy will be triggered. If copy=False, a copy might
        be triggered by a conversion.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X.

    ensure_2d : boolean (default=True)
        Whether to make X at least 2d.

    allow_nd : boolean (default=False)
        Whether to allow X.ndim > 2.

    Returns
    -------
    X_converted : object
        The converted and validated X.


shape = np.shape(y)<EOL>if len(shape) == <NUM_LIT:1>:<EOL><INDENT>return np.ravel(y)<EOL><DEDENT>if len(shape) == <NUM_LIT:2> and shape[<NUM_LIT:1>] == <NUM_LIT:1>:<EOL><INDENT>if warn:<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>DataConversionWarning, stacklevel=<NUM_LIT:2>)<EOL><DEDENT>return np.ravel(y)<EOL><DEDENT>raise ValueError("<STR_LIT>".format(shape))<EOL>

Ravel column or 1d numpy array, else raises an error

    Parameters
    ----------
    y : array-like

    Returns
    -------
    y : array


if not isinstance(estimator, str):<EOL><INDENT>estimator = estimator.__class__.__name__<EOL><DEDENT>if X.dtype.kind != '<STR_LIT:f>':<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>" % (estimator, X.dtype))<EOL>return True<EOL><DEDENT>return False<EOL>

Warning utility function to check that data type is floating point.

    Returns True if a warning was raised (i.e. the input is not float) and
    False otherwise, for easier input validation.


if seed is None or seed is np.random:<EOL><INDENT>return np.random.mtrand._rand<EOL><DEDENT>if isinstance(seed, (numbers.Integral, np.integer)):<EOL><INDENT>return np.random.RandomState(seed)<EOL><DEDENT>if isinstance(seed, np.random.RandomState):<EOL><INDENT>return seed<EOL><DEDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' % seed)<EOL>

Turn seed into a np.random.RandomState instance

    If seed is None, return the RandomState singleton used by np.random.
    If seed is an int, return a new RandomState instance seeded with seed.
    If seed is already a RandomState instance, return it.
    Otherwise raise ValueError.


def filter_objlist(olist, fieldname, fieldval):

return [x for x in olist if getattr(x, fieldname) == fieldval]<EOL>

Returns a list with of the objects in olist that have a fieldname valued as fieldval

Parameters
----------
olist: list of objects

fieldname: string

fieldval: anything

Returns
-------
list of objets


return [m for s in lst for m in (filt(s),) if m]<EOL>

Parameters
----------
lst: list

filter: function
    Unary string filter function

Returns
-------
list
    List of items that passed the filter

Example
-------
>>> l    = ['12123123', 'N123213']
>>> filt = re.compile('\d*').match
>>> nu_l = list_filter(l, filt)


def match_list(lst, pattern, group_names=[]):

filtfn = re.compile(pattern).match<EOL>filtlst = filter_list(lst, filtfn)<EOL>if not group_names:<EOL><INDENT>return [m.string for m in filtlst]<EOL><DEDENT>else:<EOL><INDENT>return [m.group(group_names) for m in filtlst]<EOL><DEDENT>

Parameters
----------
lst: list of str

regex: string

group_names: list of strings
    See re.MatchObject group docstring

Returns
-------
list of strings
    Filtered list, with the strings that match the pattern


filt = re.compile(pattern).search<EOL>return filter_list(lst, filt)<EOL>

Parameters
----------
pattern: string

lst: list of strings

Returns
-------
filtered_list: list of str
    Filtered lists with the strings in which the pattern is found.


return {preffix + str(key): (value if isinstance(value, dict) else value)<EOL>for key, value in list(adict.items())}<EOL>

Parameters
----------
adict:
preffix:

Returns
-------


return [preffix + str(item) for item in lst]<EOL>

Parameters
----------
lst:
preffix:

Returns
-------


try:<EOL><INDENT>re.compile(string)<EOL>is_valid = True<EOL><DEDENT>except re.error:<EOL><INDENT>is_valid = False<EOL><DEDENT>return is_valid<EOL>

Checks whether the re module can compile the given regular expression.

Parameters
----------
string: str

Returns
-------
boolean


is_regex = False<EOL>regex_chars = ['<STR_LIT:\\>', '<STR_LIT:(>', '<STR_LIT:+>', '<STR_LIT>', '<STR_LIT:$>']<EOL>for c in regex_chars:<EOL><INDENT>if string.find(c) > -<NUM_LIT:1>:<EOL><INDENT>return is_valid_regex(string)<EOL><DEDENT><DEDENT>return is_regex<EOL>

TODO: improve this!

Returns True if the given string is considered a regular expression,
False otherwise.
It will be considered a regex if starts with a non alphabetic character
and then correctly compiled by re.compile

:param string: str


is_regex = False<EOL>regex_chars = ['<STR_LIT:!>', '<STR_LIT:*>', '<STR_LIT:$>']<EOL>for c in regex_chars:<EOL><INDENT>if string.find(c) > -<NUM_LIT:1>:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return is_regex<EOL>

Returns True if the given string is considered a fnmatch
regular expression, False otherwise.
It will look for

:param string: str


for v in values:<EOL><INDENT>string = string.replace(v, '<STR_LIT>')<EOL><DEDENT>return string<EOL>

Parameters
----------
string:
values:

Returns
-------


def where_is(strings, pattern, n=<NUM_LIT:1>, lookup_func=re.match):

count = <NUM_LIT:0><EOL>for idx, item in enumerate(strings):<EOL><INDENT>if lookup_func(pattern, item):<EOL><INDENT>count += <NUM_LIT:1><EOL>if count == n:<EOL><INDENT>return idx<EOL><DEDENT><DEDENT><DEDENT>return -<NUM_LIT:1><EOL>

Return index of the nth match found of pattern in strings

    Parameters
    ----------
    strings: list of str
        List of strings

    pattern: str
        Pattern to be matched

    nth: int
        Number of times the match must happen to return the item index.

    lookup_func: callable
        Function to match each item in strings to the pattern, e.g., re.match or re.search.

    Returns
    -------
    index: int
        Index of the nth item that matches the pattern.
        If there are no n matches will return -1


return dict((str(key), dict_1.get(key) or dict_2.get(key))<EOL>for key in set(dict_2) | set(dict_1))<EOL>

Merge two dictionaries.

    Values that evaluate to true take priority over falsy values.
    `dict_1` takes priority over `dict_2`.


def get_sys_path(rcpath, app_name, section_name=None):

<EOL>if op.exists(rcpath):<EOL><INDENT>return op.realpath(op.expanduser(rcpath))<EOL><DEDENT>try:<EOL><INDENT>settings = rcfile(app_name, section_name)<EOL><DEDENT>except:<EOL><INDENT>raise<EOL><DEDENT>try:<EOL><INDENT>sys_path = op.expanduser(settings[rcpath])<EOL><DEDENT>except KeyError:<EOL><INDENT>raise IOError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(rcpath,<EOL>section_name,<EOL>app_name))<EOL><DEDENT>else:<EOL><INDENT>if not op.exists(sys_path):<EOL><INDENT>raise IOError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(rcpath, section_name, app_name,<EOL>sys_path))<EOL><DEDENT>return op.realpath(op.expanduser(sys_path))<EOL><DEDENT>

Return a folder path if it exists.

    First will check if it is an existing system path, if it is, will return it
    expanded and absoluted.

    If this fails will look for the rcpath variable in the app_name rcfiles or
    exclusively within the given section_name, if given.

    Parameters
    ----------
    rcpath: str
        Existing folder path or variable name in app_name rcfile with an
        existing one.

    section_name: str
        Name of a section in the app_name rcfile to look exclusively there for
        variable names.

    app_name: str
        Name of the application to look for rcfile configuration files.

    Returns
    -------
    sys_path: str
        A expanded absolute file or folder path if the path exists.

    Raises
    ------
    IOError if the proposed sys_path does not exist.


def rcfile(appname, section=None, args={}, strip_dashes=True):

if strip_dashes:<EOL><INDENT>for k in args.keys():<EOL><INDENT>args[k.lstrip('<STR_LIT:->')] = args.pop(k)<EOL><DEDENT><DEDENT>environ = get_environment(appname)<EOL>if section is None:<EOL><INDENT>section = appname<EOL><DEDENT>config = get_config(appname,<EOL>section,<EOL>args.get('<STR_LIT>', '<STR_LIT>'),<EOL>args.get('<STR_LIT:path>', '<STR_LIT>'))<EOL>config = merge(merge(args, config), environ)<EOL>if not config:<EOL><INDENT>raise IOError('<STR_LIT>'<EOL>'<STR_LIT>'.format(appname))<EOL><DEDENT>return config<EOL>

Read environment variables and config files and return them merged with
    predefined list of arguments.

    Parameters
    ----------
    appname: str
        Application name, used for config files and environment variable
        names.

    section: str
        Name of the section to be read. If this is not set: appname.

    args:
        arguments from command line (optparse, docopt, etc).

    strip_dashes: bool
        Strip dashes prefixing key names from args dict.

    Returns
    --------
    dict
        containing the merged variables of environment variables, config
        files and args.

    Raises
    ------
    IOError
        In case the return value is empty.

    Notes
    -----
    Environment variables are read if they start with appname in uppercase
    with underscore, for example:

        TEST_VAR=1

    Config files compatible with ConfigParser are read and the section name
    appname is read, example:

        [appname]
        var=1

    We can also have host-dependent configuration values, which have
    priority over the default appname values.

        [appname]
        var=1

        [appname:mylinux]
        var=3

    For boolean flags do not try to use: 'True' or 'False',
                                         'on' or 'off',
                                         '1' or '0'.
    Unless you are willing to parse this values by yourself.
    We recommend commenting the variables out with '#' if you want to set a
    flag to False and check if it is in the rcfile cfg dict, i.e.:

        flag_value = 'flag_variable' in cfg

    Files are read from: /etc/appname/config,
                         /etc/appfilerc,
                         ~/.config/appname/config,
                         ~/.config/appname,
                         ~/.appname/config,
                         ~/.appnamerc,
                         appnamerc,
                         .appnamerc,
                         appnamerc file found in 'path' folder variable in args,
                         .appnamerc file found in 'path' folder variable in args,
                         file provided by 'config' variable in args.

    Example
    -------
        args = rcfile(__name__, docopt(__doc__, version=__version__))


try:<EOL><INDENT>settings = rcfile(app_name, section_name)<EOL><DEDENT>except IOError:<EOL><INDENT>raise<EOL><DEDENT>except:<EOL><INDENT>raise KeyError('<STR_LIT>'<EOL>'<STR_LIT>'.format(section_name, app_name))<EOL><DEDENT>else:<EOL><INDENT>return settings<EOL><DEDENT>

Return the dictionary containing the rcfile section configuration
    variables.

    Parameters
    ----------
    section_name: str
        Name of the section in the rcfiles.

    app_name: str
        Name of the application to look for its rcfiles.

    Returns
    -------
    settings: dict
        Dict with variable values


def get_rcfile_variable_value(var_name, app_name, section_name=None):

cfg = get_rcfile_section(app_name, section_name)<EOL>if var_name in cfg:<EOL><INDENT>raise KeyError('<STR_LIT>'<EOL>'<STR_LIT>'.format(var_name, section_name))<EOL><DEDENT>return cfg[var_name]<EOL>

Return the value of the variable in the section_name section of the
    app_name rc file.

    Parameters
    ----------
    var_name: str
        Name of the variable to be searched for.

    section_name: str
        Name of the section in the rcfiles.

    app_name: str
        Name of the application to look for its rcfiles.

    Returns
    -------
    var_value: str
        The value of the variable with given var_name.


sections = get_sections(app_name)<EOL>if not sections:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(app_name))<EOL><DEDENT>for s in sections:<EOL><INDENT>try:<EOL><INDENT>var_value = get_rcfile_variable_value(var_name, section_name=s,<EOL>app_name=app_name)<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>return s, var_value<EOL><DEDENT><DEDENT>raise KeyError('<STR_LIT>'<EOL>'<STR_LIT>'.format(var_name, app_name))<EOL>

Return the section and the value of the variable where the first
    var_name is found in the app_name rcfiles.

    Parameters
    ----------
    var_name: str
        Name of the variable to be searched for.

    app_name: str
        Name of the application to look for its rcfiles.

    Returns
    -------
    section_name: str
        Name of the section in the rcfiles where var_name was first found.

    var_value: str
        The value of the first variable with given var_name.


def setup_logging(log_config_file=op.join(op.dirname(__file__), '<STR_LIT>'),<EOL>log_default_level=LOG_LEVEL,<EOL>env_key=MODULE_NAME.upper() + '<STR_LIT>'):

path = log_config_file<EOL>value = os.getenv(env_key, None)<EOL>if value:<EOL><INDENT>path = value<EOL><DEDENT>if op.exists(path):<EOL><INDENT>log_cfg = yaml.load(read(path).format(MODULE_NAME))<EOL>logging.config.dictConfig(log_cfg)<EOL><DEDENT>else:<EOL><INDENT>logging.basicConfig(level=log_default_level)<EOL><DEDENT>log = logging.getLogger(__name__)<EOL>log.debug('<STR_LIT>')<EOL>


def _safe_cache(memory, func, **kwargs):

cachedir = memory.cachedir<EOL>if cachedir is None or cachedir in __CACHE_CHECKED:<EOL><INDENT>return memory.cache(func, **kwargs)<EOL><DEDENT>version_file = os.path.join(cachedir, '<STR_LIT>')<EOL>versions = dict()<EOL>if os.path.exists(version_file):<EOL><INDENT>with open(version_file, '<STR_LIT:r>') as _version_file:<EOL><INDENT>versions = json.load(_version_file)<EOL><DEDENT><DEDENT>modules = (nibabel, )<EOL>my_versions = dict((m.__name__, LooseVersion(m.__version__).version[:<NUM_LIT:2>])<EOL>for m in modules)<EOL>commons = set(versions.keys()).intersection(set(my_versions.keys()))<EOL>collisions = [m for m in commons if versions[m] != my_versions[m]]<EOL>if len(collisions) > <NUM_LIT:0>:<EOL><INDENT>if nilearn.CHECK_CACHE_VERSION:<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>% cachedir)<EOL>try:<EOL><INDENT>tmp_dir = (os.path.split(cachedir)[:-<NUM_LIT:1>]<EOL>+ ('<STR_LIT>' % os.getpid(), ))<EOL>tmp_dir = os.path.join(*tmp_dir)<EOL>os.rename(cachedir, tmp_dir)<EOL>shutil.rmtree(tmp_dir)<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>os.makedirs(cachedir)<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>else:<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>" % cachedir)<EOL><DEDENT><DEDENT>if versions != my_versions:<EOL><INDENT>with open(version_file, '<STR_LIT:w>') as _version_file:<EOL><INDENT>json.dump(my_versions, _version_file)<EOL><DEDENT><DEDENT>__CACHE_CHECKED[cachedir] = True<EOL>return memory.cache(func, **kwargs)<EOL>

A wrapper for mem.cache that flushes the cache if the version
        number of nibabel has changed.


def cache(func, memory, func_memory_level=None, memory_level=None,<EOL>**kwargs):

verbose = kwargs.get('<STR_LIT>', <NUM_LIT:0>)<EOL>memory_levels = [memory_level, func_memory_level]<EOL>both_params_integers = all(isinstance(lvl, int) for lvl in memory_levels)<EOL>both_params_none = all(lvl is None for lvl in memory_levels)<EOL>if not (both_params_integers or both_params_none):<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if memory is not None and (func_memory_level is None or<EOL>memory_level >= func_memory_level):<EOL><INDENT>if isinstance(memory, _basestring):<EOL><INDENT>memory = Memory(cachedir=memory, verbose=verbose)<EOL><DEDENT>if not isinstance(memory, MEMORY_CLASSES):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (memory, type(memory)))<EOL><DEDENT>if (memory.cachedir is None and memory_level is not None<EOL>and memory_level > <NUM_LIT:1>):<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(memory_level, func.__name__),<EOL>stacklevel=<NUM_LIT:2>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>memory = Memory(cachedir=None, verbose=verbose)<EOL><DEDENT>return _safe_cache(memory, func, **kwargs)<EOL>

Return a joblib.Memory object.

    The memory_level determines the level above which the wrapped
    function output is cached. By specifying a numeric value for
    this level, the user can to control the amount of cache memory
    used. This function will cache the function call or not
    depending on the cache level.

    Parameters
    ----------
    func: function
        The function which output is to be cached.

    memory: instance of joblib.Memory or string
        Used to cache the function call.

    func_memory_level: int, optional
        The memory_level from which caching must be enabled for the wrapped
        function.

    memory_level: int, optional
        The memory_level used to determine if function call must
        be cached or not (if user_memory_level is equal of greater than
        func_memory_level the function is cached)

    kwargs: keyword arguments
        The keyword arguments passed to memory.cache

    Returns
    -------
    mem: joblib.MemorizedFunc
        object that wraps the function func. This object may be
        a no-op, if the requested level is lower than the value given
        to _cache()). For consistency, a joblib.Memory object is always
        returned.


def _cache(self, func, func_memory_level=<NUM_LIT:1>, **kwargs):

verbose = getattr(self, '<STR_LIT>', <NUM_LIT:0>)<EOL>if not hasattr(self, "<STR_LIT>"):<EOL><INDENT>self.memory_level = <NUM_LIT:0><EOL><DEDENT>if not hasattr(self, "<STR_LIT>"):<EOL><INDENT>self.memory = Memory(cachedir=None, verbose=verbose)<EOL><DEDENT>if isinstance(self.memory, _basestring):<EOL><INDENT>self.memory = Memory(cachedir=self.memory, verbose=verbose)<EOL><DEDENT>if self.memory_level == <NUM_LIT:0>:<EOL><INDENT>if (isinstance(self.memory, _basestring)<EOL>or self.memory.cachedir is not None):<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>self.memory_level = <NUM_LIT:1><EOL><DEDENT><DEDENT>return cache(func, self.memory, func_memory_level=func_memory_level,<EOL>memory_level=self.memory_level, **kwargs)<EOL>

Return a joblib.Memory object.

        The memory_level determines the level above which the wrapped
        function output is cached. By specifying a numeric value for
        this level, the user can to control the amount of cache memory
        used. This function will cache the function call or not
        depending on the cache level.

        Parameters
        ----------
        func: function
            The function the output of which is to be cached.

        memory_level: int
            The memory_level from which caching must be enabled for the wrapped
            function.

        Returns
        -------
        mem: joblib.Memory
            object that wraps the function func. This object may be
            a no-op, if the requested level is lower than the value given
            to _cache()). For consistency, a joblib.Memory object is always
            returned.


if (sys.version_info > (<NUM_LIT:3>, <NUM_LIT:0>)):<EOL><INDENT>return which_py3(program)<EOL><DEDENT>else:<EOL><INDENT>return which_py2(program)<EOL><DEDENT>

Returns the absolute path of the given CLI program name.


return inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>]<EOL>


return inspect.stack()[<NUM_LIT:2>][<NUM_LIT:3>]<EOL>


sys.stderr.write(msg + "<STR_LIT:\n>")<EOL>sys.exit(code)<EOL>

Writes msg to stderr and exits with return code


p = subprocess.Popen(cmd_args, stdout=subprocess.PIPE)<EOL>(output, err) = p.communicate()<EOL>return output<EOL>

Calls the command

Parameters
----------
cmd_args: list of str
    Command name to call and its arguments in a list.

Returns
-------
Command output


if not op.isabs(cmd_name):<EOL><INDENT>cmd_fullpath = which(cmd_name)<EOL><DEDENT>else:<EOL><INDENT>cmd_fullpath = cmd_name<EOL><DEDENT>try:<EOL><INDENT>cmd_line = [cmd_fullpath] + args_strings<EOL>log.info('<STR_LIT>'.format(cmd_line))<EOL>retval = subprocess.check_call(cmd_line)<EOL><DEDENT>except CalledProcessError as ce:<EOL><INDENT>log.exception("<STR_LIT>"<EOL>"<STR_LIT>".format(cmd_name, args_strings,<EOL>ce.returncode))<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>return retval<EOL><DEDENT>

Call CLI command with arguments and returns its return value.

    Parameters
    ----------
    cmd_name: str
        Command name or full path to the binary file.

    arg_strings: list of str
        Argument strings list.

    Returns
    -------
    return_value
        Command return value.


log.info(cmd)<EOL>ret = condor_submit(cmd)<EOL>if ret != <NUM_LIT:0>:<EOL><INDENT>subprocess.call(cmd, shell=shell)<EOL><DEDENT>

Tries to submit cmd to HTCondor, if it does not succeed, it will
be called with subprocess.call.

Parameters
----------
cmd: string
    Command to be submitted

Returns
-------


is_running = subprocess.call('<STR_LIT>', shell=True) == <NUM_LIT:0><EOL>if not is_running:<EOL><INDENT>raise CalledProcessError('<STR_LIT>')<EOL><DEDENT>sub_cmd = '<STR_LIT>'+ cmd.split()[<NUM_LIT:0>] + '<STR_LIT>'<EOL>log.info('<STR_LIT>' + sub_cmd)<EOL>return subprocess.call(sub_cmd + '<STR_LIT:U+0020>' + cmd, shell=True)<EOL>

Submits cmd to HTCondor queue

Parameters
----------
cmd: string
    Command to be submitted

Returns
-------
int
    returncode value from calling the submission command.


if image is None:<EOL><INDENT>self._mask = None<EOL><DEDENT>try:<EOL><INDENT>mask = load_mask(image)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>raise Exception('<STR_LIT>'.format(image)) from exc<EOL><DEDENT>else:<EOL><INDENT>self._mask = mask<EOL><DEDENT>

self.mask setter

        Parameters
        ----------
        image: str or img-like object.
            See NeuroImage constructor docstring.


def check_compatibility(self, one_img, another_img=None):

if another_img is None:<EOL><INDENT>if len(self.items) > <NUM_LIT:0>:<EOL><INDENT>another_img = self.items[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'.format(repr_imgs(one_img)))<EOL><DEDENT><DEDENT>try:<EOL><INDENT>if self.all_compatible:<EOL><INDENT>check_img_compatibility(one_img, another_img)<EOL><DEDENT>if self.mask is not None:<EOL><INDENT>check_img_compatibility(one_img, self.mask, only_check_3d=True)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>raise<EOL><DEDENT>

Parameters
----------
one_img: str or img-like object.
    See NeuroImage constructor docstring.

anoter_img: str or img-like object.
    See NeuroImage constructor docstring.
    If None will use the first image of self.images, if there is any.

Raises
------
NiftiFilesNotCompatible
    If one_img and another_img aren't compatible.

ValueError
    If another_img is None and there are no other images in this set.


if not isinstance(labels, string_types) and len(labels) != self.n_subjs:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'.format(len(labels), self.n_subjs))<EOL><DEDENT>self.labels = labels<EOL>

Parameters
----------
labels: list of int or str
    This list will be checked to have the same size as

Raises
------
ValueError
    if len(labels) != self.n_subjs