def make_git_commit(environment, git_repository, user_message, git_fail):

<EOL>repo = git.Repo(git_repository)<EOL>index = repo.index<EOL>traj = environment.v_trajectory<EOL>if traj.v_comment:<EOL><INDENT>commentstr = '<STR_LIT>' % traj.v_comment<EOL><DEDENT>else:<EOL><INDENT>commentstr = '<STR_LIT>'<EOL><DEDENT>if user_message:<EOL><INDENT>user_message += '<STR_LIT>'<EOL><DEDENT>message = '<STR_LIT>' %(user_message, traj.v_name, traj.v_time, commentstr)<EOL>diff = index.diff(None)<EOL>if diff:<EOL><INDENT>if git_fail:<EOL><INDENT>raise pex.GitDiffError('<STR_LIT>')<EOL><DEDENT>repo.git.add('<STR_LIT>')<EOL>commit = index.commit(message)<EOL>new_commit = True<EOL><DEDENT>else:<EOL><INDENT>commit = repo.commit(None)<EOL>new_commit = False<EOL><DEDENT>add_commit_variables(traj, commit)<EOL>return new_commit, commit.hexsha<EOL>

Makes a commit and returns if a new commit was triggered and the SHA_1 code of the commit.

    If `git_fail` is `True` program fails instead of triggering a new commit given
    not committed changes. Then a `GitDiffError` is raised.


while True:<EOL><INDENT>try:<EOL><INDENT>return next(self._current)<EOL><DEDENT>except StopIteration:<EOL><INDENT>try:<EOL><INDENT>self._current = iter(self._chain.popleft())<EOL><DEDENT>except IndexError:<EOL><INDENT>raise StopIteration('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>

Returns next element from chain.

        More precisely, it returns the next element of the
        foremost iterator. If this iterator is empty it moves iteratively
        along the chain of available iterators to pick the new foremost one.

        Raises StopIteration if there are no elements left.


return self.next()<EOL>


self._ndarray = ndarray<EOL>

Creates a new hashable object encapsulating an ndarray.

            :param ndarray: The wrapped ndarray.


def _create_storage(storage_service, trajectory=None, **kwargs):

kwargs_copy = kwargs.copy()<EOL>kwargs_copy['<STR_LIT>'] = trajectory<EOL>matching_kwargs = get_matching_kwargs(storage_service, kwargs_copy)<EOL>storage_service = storage_service(**matching_kwargs)<EOL>unused_kwargs = set(kwargs.keys()) - set(matching_kwargs.keys())<EOL>return storage_service, unused_kwargs<EOL>

Creates a service from a constructor and checks which kwargs are not used


def storage_factory(storage_service, trajectory=None, **kwargs):

if '<STR_LIT:filename>' in kwargs and storage_service is None:<EOL><INDENT>filename = kwargs['<STR_LIT:filename>']<EOL>_, ext = os.path.splitext(filename)<EOL>if ext in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>storage_service = HDF5StorageService<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' %<EOL>(ext, filename))<EOL><DEDENT><DEDENT>elif isinstance(storage_service, str):<EOL><INDENT>class_name = storage_service.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>storage_service = create_class(class_name, [storage_service, HDF5StorageService])<EOL><DEDENT>if inspect.isclass(storage_service):<EOL><INDENT>return _create_storage(storage_service, trajectory, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>return storage_service, set(kwargs.keys())<EOL><DEDENT>

Creates a storage service, to be extended if new storage services are added

    :param storage_service:

        Storage Service instance of constructor or a string pointing to a file

    :param trajectory:

        A trajectory instance

    :param kwargs:

        Arguments passed to the storage service

    :return:

        A storage service and a set of not used keyword arguments from kwargs


if a.v_is_parameter and b.v_is_parameter:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if a.v_is_parameter or b.v_is_parameter:<EOL><INDENT>return False<EOL><DEDENT>if a.v_full_name != b.v_full_name:<EOL><INDENT>return False<EOL><DEDENT>if hasattr(a, '<STR_LIT>') and not hasattr(b, '<STR_LIT>'):<EOL><INDENT>return False<EOL><DEDENT>if hasattr(a, '<STR_LIT>'):<EOL><INDENT>akeyset = set(a._data.keys())<EOL>bkeyset = set(b._data.keys())<EOL>if akeyset != bkeyset:<EOL><INDENT>return False<EOL><DEDENT>for key in a._data:<EOL><INDENT>val = a._data[key]<EOL>bval = b._data[key]<EOL>if not nested_equal(val, bval):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Compares two result instances

    Checks full name and all data. Does not consider the comment.

    :return: True or False

    :raises: ValueError if both inputs are no result instances


if (not b.v_is_parameter and<EOL>not a.v_is_parameter):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if (not b.v_is_parameter or<EOL>not a.v_is_parameter):<EOL><INDENT>return False<EOL><DEDENT>if a.v_full_name != b.v_full_name:<EOL><INDENT>return False<EOL><DEDENT>if a.f_is_empty() and b.f_is_empty():<EOL><INDENT>return True<EOL><DEDENT>if a.f_is_empty() != b.f_is_empty():<EOL><INDENT>return False<EOL><DEDENT>if not a._values_of_same_type(a.f_get(), b.f_get()):<EOL><INDENT>return False<EOL><DEDENT>if not a._equal_values(a.f_get(), b.f_get()):<EOL><INDENT>return False<EOL><DEDENT>if a.f_has_range() != b.f_has_range():<EOL><INDENT>return False<EOL><DEDENT>if a.f_has_range():<EOL><INDENT>if a.f_get_range_length() != b.f_get_range_length():<EOL><INDENT>return False<EOL><DEDENT>for myitem, bitem in zip(a.f_get_range(copy=False), b.f_get_range(copy=False)):<EOL><INDENT>if not a._values_of_same_type(myitem, bitem):<EOL><INDENT>return False<EOL><DEDENT>if not a._equal_values(myitem, bitem):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Compares two parameter instances

    Checks full name, data, and ranges. Does not consider the comment.

    :return: True or False

    :raises: ValueError if both inputs are no parameter instances


try:<EOL><INDENT>result_dict = instance.__dict__.copy()<EOL><DEDENT>except AttributeError:<EOL><INDENT>result_dict = {}<EOL><DEDENT>if hasattr(instance, '<STR_LIT>'):<EOL><INDENT>all_slots = instance.__all_slots__<EOL><DEDENT>else:<EOL><INDENT>all_slots = slots.get_all_slots(instance.__class__)<EOL><DEDENT>for slot in all_slots:<EOL><INDENT>result_dict[slot] = getattr(instance, slot)<EOL><DEDENT>result_dict.pop('<STR_LIT>', None)<EOL>result_dict.pop('<STR_LIT>', None)<EOL>return result_dict<EOL>

Returns an attribute value dictionary much like `__dict__` but incorporates `__slots__`


if a is b:<EOL><INDENT>return True<EOL><DEDENT>if a is None or b is None:<EOL><INDENT>return False<EOL><DEDENT>a_sparse = spsp.isspmatrix(a)<EOL>b_sparse = spsp.isspmatrix(b)<EOL>if a_sparse != b_sparse:<EOL><INDENT>return False<EOL><DEDENT>if a_sparse:<EOL><INDENT>if a.nnz == <NUM_LIT:0>:<EOL><INDENT>return b.nnz == <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>return not np.any((a != b).data)<EOL><DEDENT><DEDENT>a_series = isinstance(a, pd.Series)<EOL>b_series = isinstance(b, pd.Series)<EOL>if a_series != b_series:<EOL><INDENT>return False<EOL><DEDENT>if a_series:<EOL><INDENT>try:<EOL><INDENT>eq = (a == b).all()<EOL>return eq<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>if not len(a) == len(b):<EOL><INDENT>return False<EOL><DEDENT>for idx, itema in enumerate(a):<EOL><INDENT>itemb = b[idx]<EOL>if not nested_equal(itema, itemb):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL><DEDENT><DEDENT>a_frame = isinstance(a, pd.DataFrame)<EOL>b_frame = isinstance(b, pd.DataFrame)<EOL>if a_frame != b_frame:<EOL><INDENT>return False<EOL><DEDENT>if a_frame:<EOL><INDENT>try:<EOL><INDENT>if a.empty and b.empty:<EOL><INDENT>return True<EOL><DEDENT>new_frame = a == b<EOL>new_frame = new_frame | (pd.isnull(a) & pd.isnull(b))<EOL>if isinstance(new_frame, pd.DataFrame):<EOL><INDENT>return np.all(new_frame.as_matrix())<EOL><DEDENT><DEDENT>except (ValueError, TypeError):<EOL><INDENT>for name in a:<EOL><INDENT>cola = a[name]<EOL>if not name in b:<EOL><INDENT>return False<EOL><DEDENT>colb = b[name]<EOL>if not len(cola) == len(colb):<EOL><INDENT>return False<EOL><DEDENT>for idx, itema in enumerate(cola):<EOL><INDENT>itemb = colb[idx]<EOL>if not nested_equal(itema, itemb):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>return True<EOL><DEDENT><DEDENT>a_array = isinstance(a, np.ndarray)<EOL>b_array = isinstance(b, np.ndarray)<EOL>if a_array != b_array:<EOL><INDENT>return False<EOL><DEDENT>if a_array:<EOL><INDENT>if a.shape != b.shape:<EOL><INDENT>return False<EOL><DEDENT>return np.all(a == b)<EOL><DEDENT>a_list = isinstance(a, (Sequence, list, tuple))<EOL>b_list = isinstance(b, (Sequence, list, tuple))<EOL>if a_list != b_list:<EOL><INDENT>return False<EOL><DEDENT>if a_list:<EOL><INDENT>return all(nested_equal(x, y) for x, y in zip(a, b))<EOL><DEDENT>a_mapping = isinstance(a, (Mapping, dict))<EOL>b_mapping = isinstance(b, (Mapping, dict))<EOL>if a_mapping != b_mapping:<EOL><INDENT>return False<EOL><DEDENT>if a_mapping:<EOL><INDENT>keys_a = a.keys()<EOL>if set(keys_a) != set(b.keys()):<EOL><INDENT>return False<EOL><DEDENT>return all(nested_equal(a[k], b[k]) for k in keys_a)<EOL><DEDENT>equality = NotImplemented<EOL>try:<EOL><INDENT>equality = a.__eq__(b)<EOL><DEDENT>except (AttributeError, NotImplementedError, TypeError, ValueError):<EOL><INDENT>pass<EOL><DEDENT>if equality is NotImplemented:<EOL><INDENT>try:<EOL><INDENT>equality = b.__eq__(a)<EOL><DEDENT>except (AttributeError, NotImplementedError, TypeError, ValueError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if equality is NotImplemented:<EOL><INDENT>try:<EOL><INDENT>cmp = a.__cmp__(b)<EOL>if cmp is not NotImplemented:<EOL><INDENT>equality = cmp == <NUM_LIT:0><EOL><DEDENT><DEDENT>except (AttributeError, NotImplementedError, TypeError, ValueError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if equality is NotImplemented:<EOL><INDENT>try:<EOL><INDENT>cmp = b.__cmp__(a)<EOL>if cmp is not NotImplemented:<EOL><INDENT>equality = cmp == <NUM_LIT:0><EOL><DEDENT><DEDENT>except (AttributeError, NotImplementedError, TypeError, ValueError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if equality is not NotImplemented:<EOL><INDENT>try:<EOL><INDENT>return bool(equality)<EOL><DEDENT>except (AttributeError, NotImplementedError, TypeError, ValueError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>attributes_a = get_all_attributes(a)<EOL>attributes_b = get_all_attributes(b)<EOL>if len(attributes_a) != len(attributes_b):<EOL><INDENT>return False<EOL><DEDENT>if len(attributes_a) > <NUM_LIT:0>:<EOL><INDENT>keys_a = list(attributes_a.keys())<EOL>if set(keys_a) != set(attributes_b.keys()):<EOL><INDENT>return False<EOL><DEDENT>return all(nested_equal(attributes_a[k], attributes_b[k]) for k in keys_a)<EOL><DEDENT>return False<EOL>

Compares two objects recursively by their elements.

    Also handles numpy arrays, pandas data and sparse matrices.

    First checks if the data falls into the above categories.
    If not, it is checked if a or b are some type of sequence or mapping and
    the contained elements are compared.
    If this is not the case, it is checked if a or b do provide a custom `__eq__` that
    evaluates to a single boolean value.
    If this is not the case, the attributes of a and b are compared.
    If this does not help either, normal `==` is used.

    Assumes hashable items are not mutable in a way that affects equality.
    Based on the suggestion from HERE_, thanks again Lauritz V. Thaulow :-)

    .. _HERE: http://stackoverflow.com/questions/18376935/best-practice-for-equality-in-python


if not combined_parameters:<EOL><INDENT>combined_parameters = list(parameter_dict)<EOL><DEDENT>else:<EOL><INDENT>combined_parameters = list(combined_parameters)<EOL><DEDENT>for idx, item in enumerate(combined_parameters):<EOL><INDENT>if isinstance(item, str):<EOL><INDENT>combined_parameters[idx] = (item,)<EOL><DEDENT><DEDENT>iterator_list = []<EOL>for item_tuple in combined_parameters:<EOL><INDENT>inner_iterator_list = [parameter_dict[key] for key in item_tuple]<EOL>zipped_iterator = zip(*inner_iterator_list)<EOL>iterator_list.append(zipped_iterator)<EOL><DEDENT>result_dict = {}<EOL>for key in parameter_dict:<EOL><INDENT>result_dict[key] = []<EOL><DEDENT>cartesian_iterator = itools.product(*iterator_list)<EOL>for cartesian_tuple in cartesian_iterator:<EOL><INDENT>for idx, item_tuple in enumerate(combined_parameters):<EOL><INDENT>for inneridx, key in enumerate(item_tuple):<EOL><INDENT>result_dict[key].append(cartesian_tuple[idx][inneridx])<EOL><DEDENT><DEDENT><DEDENT>return result_dict<EOL>

Generates a Cartesian product of the input parameter dictionary.

    For example:

    >>> print cartesian_product({'param1':[1,2,3], 'param2':[42.0, 52.5]})
    {'param1':[1,1,2,2,3,3],'param2': [42.0,52.5,42.0,52.5,42.0,52.5]}

    :param parameter_dict:

        Dictionary containing parameter names as keys and iterables of data to explore.

    :param combined_parameters:

        Tuple of tuples. Defines the order of the parameters and parameters that are
        linked together.
        If an inner tuple contains only a single item, you can spare the
        inner tuple brackets.

        For example:

        >>> print cartesian_product( {'param1': [42.0, 52.5], 'param2':['a', 'b'], 'param3' : [1,2,3]}, ('param3',('param1', 'param2')))
        {param3':[1,1,2,2,3,3],'param1' : [42.0,52.5,42.0,52.5,42.0,52.5], 'param2':['a','b','a','b','a','b']}

    :returns: Dictionary with cartesian product lists.


ranges = [param.f_get_range(copy=False) for param in explored_parameters]<EOL>zipped_tuples = list(zip(*ranges))<EOL>try:<EOL><INDENT>unique_elements = OrderedDict()<EOL>for idx, val_tuple in enumerate(zipped_tuples):<EOL><INDENT>if val_tuple not in unique_elements:<EOL><INDENT>unique_elements[val_tuple] = []<EOL><DEDENT>unique_elements[val_tuple].append(idx)<EOL><DEDENT>return list(unique_elements.items())<EOL><DEDENT>except TypeError:<EOL><INDENT>logger = logging.getLogger('<STR_LIT>')<EOL>logger.error('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>unique_elements = []<EOL>for idx, val_tuple in enumerate(zipped_tuples):<EOL><INDENT>matches = False<EOL>for added_tuple, pos_list in unique_elements:<EOL><INDENT>matches = True<EOL>for idx2, val in enumerate(added_tuple):<EOL><INDENT>if not explored_parameters[idx2]._equal_values(val_tuple[idx2], val):<EOL><INDENT>matches = False<EOL>break<EOL><DEDENT><DEDENT>if matches:<EOL><INDENT>pos_list.append(idx)<EOL>break<EOL><DEDENT><DEDENT>if not matches:<EOL><INDENT>unique_elements.append((val_tuple, [idx]))<EOL><DEDENT><DEDENT>return unique_elements<EOL><DEDENT>

Takes a list of explored parameters and finds unique parameter combinations.

    If parameter ranges are hashable operates in O(N), otherwise O(N**2).

    :param explored_parameters:

        List of **explored** parameters

    :return:

        List of tuples, first entry being the parameter values, second entry a list
        containing the run position of the unique combination.


@functools.wraps(init_func)<EOL>def new_func(env, *args, **kwargs):<EOL><INDENT>config_interpreter = ConfigInterpreter(kwargs)<EOL>new_kwargs = config_interpreter.interpret()<EOL>init_func(env, *args, **new_kwargs)<EOL>config_interpreter.add_parameters(env.traj)<EOL><DEDENT>return new_func<EOL>

Decorator wrapping the environment to use a config file


kwargs = {}<EOL>try:<EOL><INDENT>if self.parser.has_section(section):<EOL><INDENT>options = self.parser.options(section)<EOL>for option in options:<EOL><INDENT>str_val = self.parser.get(section, option)<EOL>val = ast.literal_eval(str_val)<EOL>kwargs[option] = val<EOL><DEDENT><DEDENT>return kwargs<EOL><DEDENT>except:<EOL><INDENT>raise<EOL><DEDENT>


kwargs = {}<EOL>sections = ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>for section in sections:<EOL><INDENT>kwargs.update(self._collect_section(section))<EOL><DEDENT>return kwargs<EOL>


if self.config_file:<EOL><INDENT>new_kwargs = self._collect_config()<EOL>for key in new_kwargs:<EOL><INDENT>if key not in self.kwargs:<EOL><INDENT>self.kwargs[key] = new_kwargs[key]<EOL><DEDENT><DEDENT>if not use_simple_logging(self.kwargs) and '<STR_LIT>' not in self.kwargs:<EOL><INDENT>self.kwargs['<STR_LIT>'] = self.config_file<EOL><DEDENT><DEDENT>return self.kwargs<EOL>

Copies parsed arguments into the kwargs passed to the environment


if self.config_file:<EOL><INDENT>parameters = self._collect_section('<STR_LIT>')<EOL>for name in parameters:<EOL><INDENT>value = parameters[name]<EOL>if not isinstance(value, tuple):<EOL><INDENT>value = (value,)<EOL><DEDENT>traj.f_add_parameter(name, *value)<EOL><DEDENT>config = self._collect_section('<STR_LIT>')<EOL>for name in config:<EOL><INDENT>value = config[name]<EOL>if not isinstance(value, tuple):<EOL><INDENT>value = (value,)<EOL><DEDENT>traj.f_add_config(name, *value)<EOL><DEDENT><DEDENT>

Adds parameters and config from the `.ini` file to the trajectory


class_data = full_class_string.split("<STR_LIT:.>")<EOL>module_path = "<STR_LIT:.>".join(class_data[:-<NUM_LIT:1>])<EOL>class_str = class_data[-<NUM_LIT:1>]<EOL>module = importlib.import_module(module_path)<EOL>return getattr(module, class_str)<EOL>

Loads a class from a string naming the module and class name.

    For example:
    >>> load_class(full_class_string = 'pypet.brian.parameter.BrianParameter')
    <BrianParameter>


try:<EOL><INDENT>new_class = globals()[class_name]<EOL>if not inspect.isclass(new_class):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>return new_class<EOL><DEDENT>except (KeyError, TypeError):<EOL><INDENT>for dynamic_class in dynamic_imports:<EOL><INDENT>if inspect.isclass(dynamic_class):<EOL><INDENT>if class_name == dynamic_class.__name__:<EOL><INDENT>return dynamic_class<EOL><DEDENT><DEDENT>else:<EOL><INDENT>class_name_to_test = dynamic_class.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>if class_name == class_name_to_test:<EOL><INDENT>new_class = load_class(dynamic_class)<EOL>return new_class<EOL><DEDENT><DEDENT><DEDENT>raise ImportError('<STR_LIT>' % class_name)<EOL><DEDENT>

Dynamically creates a class.

    It is tried if the class can be created by the already given imports.
    If not the list of the dynamically loaded classes is used.


def manual_run(turn_into_run=True, store_meta_data=True, clean_up=True):

def wrapper(func):<EOL><INDENT>@functools.wraps(func)<EOL>def new_func(traj, *args, **kwargs):<EOL><INDENT>do_wrap = not traj._run_by_environment<EOL>if do_wrap:<EOL><INDENT>traj.f_start_run(turn_into_run=turn_into_run)<EOL><DEDENT>result = func(traj, *args, **kwargs)<EOL>if do_wrap:<EOL><INDENT>traj.f_finalize_run(store_meta_data=store_meta_data,<EOL>clean_up=clean_up)<EOL><DEDENT>return result<EOL><DEDENT>return new_func<EOL><DEDENT>return wrapper<EOL>

Can be used to decorate a function as a manual run function.

    This can be helpful if you want the run functionality without using an environment.

    :param turn_into_run:

        If the trajectory should become a `single run` with more specialized functionality
        during a single run.

    :param store_meta_data:

        If meta-data like runtime should be automatically stored

    :param clean_up:

        If all data added during the single run should be removed, only works
        if ``turn_into_run=True``.


def wrapper(func):<EOL><INDENT>@functools.wraps(func)<EOL>def new_func(*args, **kwargs):<EOL><INDENT>warning_string = "<STR_LIT>" % func.__name__<EOL>warning_string = warning_string + '<STR_LIT:U+0020>' + msg<EOL>warnings.warn(<EOL>warning_string,<EOL>category=DeprecationWarning,<EOL>)<EOL>return func(*args, **kwargs)<EOL><DEDENT>return new_func<EOL><DEDENT>return wrapper<EOL>

This is a decorator which can be used to mark functions
    as deprecated. It will result in a warning being emitted
    when the function is used.

    :param msg:

        Additional message added to the warning.


def _decorator(func):<EOL><INDENT>sourcedoc = fromfunc.__doc__<EOL>split_doc = sourcedoc.split('<STR_LIT:\n>')<EOL>split_doc_no_abstract = [line for line in split_doc if not '<STR_LIT>' in line]<EOL>if len(split_doc) != len(split_doc_no_abstract):<EOL><INDENT>sourcedoc = '<STR_LIT:\n>'.join(split_doc_no_abstract[:-<NUM_LIT:1>])<EOL><DEDENT>if func.__doc__ is None:<EOL><INDENT>func.__doc__ = sourcedoc<EOL><DEDENT>else:<EOL><INDENT>func.__doc__ = sep.join([sourcedoc, func.__doc__])<EOL><DEDENT>return func<EOL><DEDENT>return _decorator<EOL>

Decorator: Copy the docstring of `fromfunc`

    If the doc contains a line with the keyword `ABSTRACT`,
    like `ABSTRACT: Needs to be defined in subclass`, this line and the line after are removed.


def kwargs_mutual_exclusive(param1_name, param2_name, map2to1=None):

def wrapper(func):<EOL><INDENT>@functools.wraps(func)<EOL>def new_func(*args, **kwargs):<EOL><INDENT>if param2_name in kwargs:<EOL><INDENT>if param1_name in kwargs:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' % (param1_name, param2_name))<EOL><DEDENT>param2 = kwargs.pop(param2_name)<EOL>if map2to1 is not None:<EOL><INDENT>param1 = map2to1(param2)<EOL><DEDENT>else:<EOL><INDENT>param1 = param2<EOL><DEDENT>kwargs[param1_name] = param1<EOL><DEDENT>return func(*args, **kwargs)<EOL><DEDENT>return new_func<EOL><DEDENT>return wrapper<EOL>

If there exist mutually exclusive parameters checks for them and maps param2 to 1.


def wrapper(func):<EOL><INDENT>@functools.wraps(func)<EOL>def new_func(*args, **kwargs):<EOL><INDENT>if old_name in kwargs:<EOL><INDENT>if new_name is None:<EOL><INDENT>warning_string = '<STR_LIT>''<STR_LIT>''<STR_LIT>' % (old_name, func.__name__)<EOL><DEDENT>else:<EOL><INDENT>warning_string = '<STR_LIT>''<STR_LIT>' %(old_name, func.__name__, new_name)<EOL><DEDENT>warnings.warn(warning_string, category=DeprecationWarning)<EOL>value = kwargs.pop(old_name)<EOL>if new_name is not None:<EOL><INDENT>kwargs[new_name] = value<EOL><DEDENT><DEDENT>return func(*args, **kwargs)<EOL><DEDENT>return new_func<EOL><DEDENT>return wrapper<EOL>

This is a decorator which can be used if a kwarg has changed
    its name over versions to also support the old argument name.

    Issues a warning if the old keyword argument is detected and
    converts call to new API.

    :param old_name:

        Old name of the keyword argument

    :param new_name:

        New name of keyword argument


doc = func.__doc__<EOL>na_string = '''<STR_LIT>'''<EOL>if doc is not None:<EOL><INDENT>func.__doc__ = '<STR_LIT:\n>'.join([doc, na_string])<EOL><DEDENT>func._not_in_run = True<EOL>@functools.wraps(func)<EOL>def new_func(self, *args, **kwargs):<EOL><INDENT>if self._is_run:<EOL><INDENT>raise TypeError('<STR_LIT>' %<EOL>func.__name__)<EOL><DEDENT>return func(self, *args, **kwargs)<EOL><DEDENT>return new_func<EOL>

This is a decorator that signaling that a function is not available during a single run.


doc = func.__doc__<EOL>na_string = '''<STR_LIT>'''<EOL>if doc is not None:<EOL><INDENT>func.__doc__ = '<STR_LIT:\n>'.join([doc, na_string])<EOL><DEDENT>func._with_open_store = True<EOL>@functools.wraps(func)<EOL>def new_func(self, *args, **kwargs):<EOL><INDENT>if not self.traj.v_storage_service.is_open:<EOL><INDENT>raise TypeError('<STR_LIT>' %<EOL>func.__name__)<EOL><DEDENT>return func(self, *args, **kwargs)<EOL><DEDENT>return new_func<EOL>

This is a decorator that signaling that a function is only available if the storage is open.


def retry(n, errors, wait=<NUM_LIT:0.0>, logger_name=None):

def wrapper(func):<EOL><INDENT>@functools.wraps(func)<EOL>def new_func(*args, **kwargs):<EOL><INDENT>retries = <NUM_LIT:0><EOL>while True:<EOL><INDENT>try:<EOL><INDENT>result = func(*args, **kwargs)<EOL>if retries and logger_name:<EOL><INDENT>logger = logging.getLogger(logger_name)<EOL>logger.debug('<STR_LIT>' % func.__name__)<EOL><DEDENT>return result<EOL><DEDENT>except errors:<EOL><INDENT>if retries >= n:<EOL><INDENT>if logger_name:<EOL><INDENT>logger = logging.getLogger(logger_name)<EOL>logger.exception('<STR_LIT>'<EOL>'<STR_LIT>' % (func.__name__,<EOL>str(args),<EOL>str(kwargs)))<EOL><DEDENT>raise<EOL><DEDENT>elif logger_name:<EOL><INDENT>logger = logging.getLogger(logger_name)<EOL>logger.debug('<STR_LIT>'<EOL>'<STR_LIT>' % (func.__name__,<EOL>str(args),<EOL>str(kwargs)))<EOL><DEDENT>retries += <NUM_LIT:1><EOL>if wait:<EOL><INDENT>time.sleep(wait)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return new_func<EOL><DEDENT>return wrapper<EOL>

This is a decorator that retries a function.

    Tries `n` times and catches a given tuple of `errors`.

    If the `n` retries are not enough, the error is reraised.

    If desired `waits` some seconds.

    Optionally takes a 'logger_name' of a given logger to print the caught error.


if item.startswith('<STR_LIT>') or item.startswith('<STR_LIT>'):<EOL><INDENT>return getattr(obj, item[<NUM_LIT:2>:])<EOL><DEDENT>raise AttributeError('<STR_LIT>' % (obj.__class__.__name__, item))<EOL>


if item.startswith('<STR_LIT>'):<EOL><INDENT>return setattr(obj, item[<NUM_LIT:2>:], value)<EOL><DEDENT>else:<EOL><INDENT>return super(obj.__class__, obj).__setattr__(item, value)<EOL><DEDENT>


if hasattr(cls, '<STR_LIT>'):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>cls.__getattr__ = _prfx_getattr_<EOL>cls.__setattr__ = _prfx_setattr_<EOL>return cls<EOL>

Decorate that adds the prefix naming scheme


def compact_hdf5_file(filename, name=None, index=None, keep_backup=True):

if name is None and index is None:<EOL><INDENT>index = -<NUM_LIT:1><EOL><DEDENT>tmp_traj = load_trajectory(name, index, as_new=False, load_all=pypetconstants.LOAD_NOTHING,<EOL>force=True, filename=filename)<EOL>service = tmp_traj.v_storage_service<EOL>complevel = service.complevel<EOL>complib = service.complib<EOL>shuffle = service.shuffle<EOL>fletcher32 = service.fletcher32<EOL>name_wo_ext, ext = os.path.splitext(filename)<EOL>tmp_filename = name_wo_ext + '<STR_LIT>' + ext<EOL>abs_filename = os.path.abspath(filename)<EOL>abs_tmp_filename = os.path.abspath(tmp_filename)<EOL>command = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', complib,<EOL>'<STR_LIT>', str(complevel),<EOL>'<STR_LIT>', str(int(shuffle)),<EOL>'<STR_LIT>', str(int(fletcher32)),<EOL>abs_filename, abs_tmp_filename]<EOL>str_command = '<STR_LIT:U+0020>'.join(command)<EOL>print('<STR_LIT>' % str_command)<EOL>retcode = subprocess.call(command)<EOL>if retcode != <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' %<EOL>(filename, str(retcode)))<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>if keep_backup:<EOL><INDENT>backup_file_name = name_wo_ext + '<STR_LIT>' + ext<EOL>os.rename(filename, backup_file_name)<EOL><DEDENT>else:<EOL><INDENT>os.remove(filename)<EOL><DEDENT>os.rename(tmp_filename, filename)<EOL>print('<STR_LIT>')<EOL><DEDENT>return retcode<EOL>

Can compress an HDF5 to reduce file size.

    The properties on how to compress the new file are taken from a given
    trajectory in the file.
    Simply calls ``ptrepack`` from the command line.
    (Se also https://pytables.github.io/usersguide/utilities.html#ptrepackdescr)

    Currently only supported under Linux, no guarantee for Windows usage.

    :param filename:

        Name of the file to compact

    :param name:

        The name of the trajectory from which the compression properties are taken

    :param index:

        Instead of a name you could also specify an index, i.e -1 for the last trajectory
        in the file.

    :param keep_backup:

        If a back up version of the original file should be kept.
        The backup file is named as the original but `_backup` is appended to the end.

    :return:

        The return/error code of ptrepack


def _handle_sigint(self, signum, frame):

if self.hit:<EOL><INDENT>prompt = '<STR_LIT>'<EOL>raise KeyboardInterrupt(prompt)<EOL><DEDENT>else:<EOL><INDENT>self.hit = True<EOL>prompt = ('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>sys.stderr.write(prompt)<EOL><DEDENT>

Handler of SIGINT

        Does nothing if SIGINT is encountered once but raises a KeyboardInterrupt in case it
        is encountered twice.
        immediatly.


wrap_mode = kwargs['<STR_LIT>']<EOL>traj = kwargs['<STR_LIT>']<EOL>traj.v_storage_service = _pool_single_run.storage_service<EOL>if wrap_mode == pypetconstants.WRAP_MODE_LOCAL:<EOL><INDENT>traj.v_storage_service.free_references()<EOL><DEDENT>return _sigint_handling_single_run(kwargs)<EOL>

Starts a pool single run and passes the storage service


idx = kwargs.pop('<STR_LIT>')<EOL>frozen_kwargs = _frozen_pool_single_run.kwargs<EOL>frozen_kwargs.update(kwargs)  <EOL>traj = frozen_kwargs['<STR_LIT>']<EOL>traj.f_set_crun(idx)<EOL>return _sigint_handling_single_run(frozen_kwargs)<EOL>

Single run wrapper for the frozen pool, makes a single run and passes kwargs


_pool_single_run.storage_service = kwargs['<STR_LIT>']<EOL>_configure_niceness(kwargs)<EOL>_configure_logging(kwargs, extract=False)<EOL>

Configures the pool and keeps the storage service


_frozen_pool_single_run.kwargs = kwargs<EOL>_configure_niceness(kwargs)<EOL>_configure_logging(kwargs, extract=False)<EOL>traj = kwargs['<STR_LIT>']<EOL>traj.v_full_copy = kwargs['<STR_LIT>']<EOL>

Configures the frozen pool and keeps all kwargs


_configure_niceness(kwargs)<EOL>_configure_logging(kwargs)<EOL>result_queue = kwargs['<STR_LIT>']<EOL>result = _sigint_handling_single_run(kwargs)<EOL>result_queue.put(result)<EOL>result_queue.close()<EOL>

Wrapper function that first configures logging and starts a single run afterwards.


def _delete_old_scoop_rev_data(old_scoop_rev):<EOL><INDENT>if old_scoop_rev is not None:<EOL><INDENT>try:<EOL><INDENT>elements = shared.elements<EOL>for key in elements:<EOL><INDENT>var_dict = elements[key]<EOL>if old_scoop_rev in var_dict:<EOL><INDENT>del var_dict[old_scoop_rev]<EOL><DEDENT><DEDENT>logging.getLogger('<STR_LIT>').debug('<STR_LIT>'<EOL>'<STR_LIT>' % old_scoop_rev)<EOL><DEDENT>except AttributeError:<EOL><INDENT>logging.getLogger('<STR_LIT>').error('<STR_LIT>'<EOL>'<STR_LIT>' % old_scoop_rev)<EOL><DEDENT><DEDENT><DEDENT>scoop_rev = kwargs.pop('<STR_LIT>')<EOL>try:<EOL><INDENT>old_scoop_rev = _frozen_scoop_single_run.kwargs['<STR_LIT>']<EOL>configured = old_scoop_rev == scoop_rev<EOL><DEDENT>except (AttributeError, KeyError):<EOL><INDENT>old_scoop_rev = None<EOL>configured = False<EOL><DEDENT>if not configured:<EOL><INDENT>_frozen_scoop_single_run.kwargs = shared.getConst(scoop_rev, timeout=<NUM_LIT>)<EOL>frozen_kwargs = _frozen_scoop_single_run.kwargs<EOL>frozen_kwargs['<STR_LIT>'] = scoop_rev<EOL>frozen_kwargs['<STR_LIT>'].v_full_copy = frozen_kwargs['<STR_LIT>']<EOL>if not scoop.IS_ORIGIN:<EOL><INDENT>_configure_niceness(frozen_kwargs)<EOL>_configure_logging(frozen_kwargs, extract=False)<EOL><DEDENT>_delete_old_scoop_rev_data(old_scoop_rev)<EOL>logging.getLogger('<STR_LIT>').info('<STR_LIT>' % str(scoop.worker))<EOL><DEDENT>

Wrapper function that configures a frozen SCOOP set up.

    Deletes of data if necessary.


try:<EOL><INDENT>try:<EOL><INDENT>is_origin = scoop.IS_ORIGIN<EOL><DEDENT>except AttributeError:<EOL><INDENT>is_origin = True<EOL><DEDENT>if not is_origin:<EOL><INDENT>_configure_niceness(kwargs)<EOL>_configure_logging(kwargs)<EOL><DEDENT>return _single_run(kwargs)<EOL><DEDENT>except Exception:<EOL><INDENT>scoop.logger.exception('<STR_LIT>')<EOL>raise<EOL><DEDENT>

Wrapper function for scoop, that does not configure logging


try:<EOL><INDENT>logging_manager = kwargs['<STR_LIT>']<EOL>if extract:<EOL><INDENT>logging_manager.extract_replacements(kwargs['<STR_LIT>'])<EOL><DEDENT>logging_manager.make_logging_handlers_and_tools(multiproc=True)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>sys.stderr.write('<STR_LIT>' % repr(exc))<EOL>traceback.print_exc()<EOL><DEDENT>

Requests the logging manager to configure logging.

    :param extract:

        If naming data should be extracted from the trajectory


niceness = kwargs['<STR_LIT>']<EOL>if niceness is not None:<EOL><INDENT>try:<EOL><INDENT>try:<EOL><INDENT>current = os.nice(<NUM_LIT:0>)<EOL>if niceness - current > <NUM_LIT:0>:<EOL><INDENT>os.nice(niceness-current)<EOL><DEDENT><DEDENT>except AttributeError:<EOL><INDENT>psutil.Process().nice(niceness)<EOL><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>sys.stderr.write('<STR_LIT>' % repr(exc))<EOL>traceback.print_exc()<EOL><DEDENT><DEDENT>


try:<EOL><INDENT>graceful_exit = kwargs['<STR_LIT>']<EOL>if graceful_exit:<EOL><INDENT>sigint_handling.start()<EOL>if sigint_handling.hit:<EOL><INDENT>result = (sigint_handling.SIGINT, None)<EOL><DEDENT>else:<EOL><INDENT>result = _single_run(kwargs)<EOL>if sigint_handling.hit:<EOL><INDENT>result = (sigint_handling.SIGINT, result)<EOL><DEDENT><DEDENT>return result<EOL><DEDENT>return _single_run(kwargs)<EOL><DEDENT>except:<EOL><INDENT>pypet_root_logger = logging.getLogger('<STR_LIT>')<EOL>pypet_root_logger.exception('<STR_LIT>')<EOL>raise<EOL><DEDENT>

Wrapper that allow graceful exits of single runs


pypet_root_logger = logging.getLogger('<STR_LIT>')<EOL>traj = kwargs['<STR_LIT>']<EOL>runfunc = kwargs['<STR_LIT>']<EOL>runargs = kwargs['<STR_LIT>']<EOL>kwrunparams = kwargs['<STR_LIT>']<EOL>clean_up_after_run = kwargs['<STR_LIT>']<EOL>automatic_storing = kwargs['<STR_LIT>']<EOL>wrap_mode = kwargs['<STR_LIT>']<EOL>idx = traj.v_idx<EOL>total_runs = len(traj)<EOL>pypet_root_logger.info('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (idx, total_runs))<EOL>traj.f_start_run(turn_into_run=True)<EOL>result = runfunc(traj, *runargs, **kwrunparams)<EOL>if automatic_storing:<EOL><INDENT>traj.f_store()<EOL><DEDENT>if wrap_mode == pypetconstants.WRAP_MODE_LOCAL:<EOL><INDENT>result = ((traj.v_idx, result),<EOL>traj.f_get_run_information(traj.v_idx, copy=False),<EOL>traj.v_storage_service.references)<EOL>traj.v_storage_service.free_references()<EOL><DEDENT>else:<EOL><INDENT>result = ((traj.v_idx, result),<EOL>traj.f_get_run_information(traj.v_idx, copy=False))<EOL><DEDENT>traj.f_finalize_run(store_meta_data=False,<EOL>clean_up=clean_up_after_run)<EOL>pypet_root_logger.info('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (idx, total_runs))<EOL>return result<EOL>

Performs a single run of the experiment.

    :param kwargs: Dict of arguments

        traj: The trajectory containing all parameters set to the corresponding run index.

        runfunc: The user's job function

        runargs: The arguments handed to the user's job function (as *args)

        runkwargs: The keyword arguments handed to the user's job function (as **kwargs)

        clean_up_after_run: Whether to clean up after the run

        automatic_storing: Whether or not the data should be automatically stored

        result_queue: A queue object to store results into in case a pool is used, otherwise None

    :return:

        Results computed by the user's job function which are not stored into the trajectory.
        Returns a nested tuple of run index and result and run information:
        ``((traj.v_idx, result), run_information_dict)``


_configure_logging(kwargs, extract=False)<EOL>handler=kwargs['<STR_LIT>']<EOL>graceful_exit = kwargs['<STR_LIT>']<EOL>if graceful_exit:<EOL><INDENT>sigint_handling.start()<EOL><DEDENT>handler.run()<EOL>

Starts running a queue handler and creates a log file for the queue.


repr_string = '<STR_LIT>' % (self.__class__.__name__, self.name,<EOL>self.trajectory.v_name)<EOL>return repr_string<EOL>


self._logging_manager.finalize(remove_all_handlers)<EOL>

Removes all logging handlers and stops logging to files and logging stdout.

        :param remove_all_handlers:

            If `True` all logging handlers are removed.
            If you want to keep the handlers set to `False`.


@kwargs_api_change('<STR_LIT>', '<STR_LIT>')<EOL><INDENT>def resume(self, trajectory_name=None, resume_folder=None):<DEDENT>

if trajectory_name is None:<EOL><INDENT>self._trajectory_name = self.trajectory.v_name<EOL><DEDENT>else:<EOL><INDENT>self._trajectory_name = trajectory_name<EOL><DEDENT>if resume_folder is not None:<EOL><INDENT>self._resume_folder = resume_folder<EOL><DEDENT>return self._execute_runs(None)<EOL>

Resumes crashed trajectories.

        :param trajectory_name:

            Name of trajectory to resume, if not specified the name passed to the environment
            is used. Be aware that if `add_time=True` the name you passed to the environment is
            altered and the current date is added.

        :param resume_folder:

            The folder where resume files can be found. Do not pass the name of the sub-folder
            with the trajectory name, but to the name of the parental folder.
            If not specified the resume folder passed to the environment is used.

        :return:

            List of the individual results returned by your run function.

            Returns a LIST OF TUPLES, where first entry is the run idx and second entry
            is the actual result. In case of multiprocessing these are not necessarily
            ordered according to their run index, but ordered according to their finishing time.

            Does not contain results stored in the trajectory!
            In order to access these simply interact with the trajectory object,
            potentially after calling`~pypet.trajectory.Trajectory.f_update_skeleton`
            and loading all results at once with :func:`~pypet.trajectory.f_load`
            or loading manually with :func:`~pypet.trajectory.f_load_items`.

            Even if you use multiprocessing without a pool the results returned by
            `runfunc` still need to be pickled.


return self._traj<EOL>


return self.trajectory<EOL>


return self._current_idx<EOL>

The current run index that is the next one to be executed.

        Can be set manually to make the environment consider old non-completed ones.


return self._hexsha<EOL>

The SHA1 identifier of the environment.

        It is identical to the SHA1 of the git commit.
        If version control is not used, the environment hash is computed from the
        trajectory name, the current timestamp and your current *pypet* version.


return self._time<EOL>

Time of the creation of the environment, human readable.


return self._timestamp<EOL>

Time of creation as python datetime float


return self._name<EOL>


def add_postprocessing(self, postproc, *args, **kwargs):

self._postproc = postproc<EOL>self._postproc_args = args<EOL>self._postproc_kwargs = kwargs<EOL>

Adds a post processing function.

        The environment will call this function via
        ``postproc(traj, result_list, *args, **kwargs)`` after the completion of the
        single runs.

        This function can load parts of the trajectory id needed and add additional results.

        Moreover, the function can be used to trigger an expansion of the trajectory.
        This can be useful if the user has an `optimization` task.

        Either the function calls `f_expand` directly on the trajectory or returns
        an dictionary. If latter `f_expand` is called by the environment.

        Note that after expansion of the trajectory, the post-processing function is called
        again (and again for further expansions). Thus, this allows an iterative approach
        to parameter exploration.

        Note that in case post-processing is called after all runs have been executed,
        the storage service of the trajectory is no longer multiprocessing safe.
        If you want to use multiprocessing in your post-processing you can still
        manually wrap the storage service with the :class:`~pypet.environment.MultiprocessWrapper`.

        In case you use **immediate** postprocessing, the storage service of your trajectory
        is still multiprocessing safe (except when using the wrap_mode ``'LOCAL'``).
        Accordingly, you could even use multiprocessing in your immediate post-processing phase
        if you dare, like use a multiprocessing pool_, for instance.

        You can easily check in your post-processing function if the storage service is
        multiprocessing safe via the ``multiproc_safe`` attribute, i.e.
        ``traj.v_storage_service.multiproc_safe``.

        :param postproc:

            The post processing function

        :param args:

            Additional arguments passed to the post-processing function

        :param kwargs:

            Additional keyword arguments passed to the postprocessing function

        :return:


self._user_pipeline = True<EOL>self._map_arguments = False<EOL>return self._execute_runs(pipeline)<EOL>

You can make *pypet* supervise your whole experiment by defining a pipeline.

        `pipeline` is a function that defines the entire experiment. From pre-processing
        including setting up the trajectory over defining the actual simulation runs to
        post processing.

        The `pipeline` function needs to return TWO tuples with a maximum of three entries each.

        For example:

        ::

            return (runfunc, args, kwargs), (postproc, postproc_args, postproc_kwargs)

        Where `runfunc` is the actual simulation function thet gets passed the trajectory
        container and potentially additional arguments `args` and keyword arguments `kwargs`.
        This will be run by your environment with all parameter combinations.

        `postproc` is a post processing function that handles your computed results.
        The function must accept as arguments the trajectory container, a list of
        results (list of tuples (run idx, result) ) and potentially
        additional arguments `postproc_args` and keyword arguments `postproc_kwargs`.

        As for :func:`~pypet.environment.Environment.f_add_postproc`, this function can
        potentially extend the trajectory.

        If you don't want to apply post-processing, your pipeline function can also simply
        return the run function and the arguments:

        ::

            return runfunc, args, kwargs

        Or

        ::

            return runfunc, args

        Or

        ::

            return runfunc

        ``return runfunc, kwargs`` does NOT work, if you don't want to pass `args` do
        ``return runfunc, (), kwargs``.

        Analogously combinations like

        ::

            return (runfunc, args), (postproc,)

        work as well.

        :param pipeline:

            The pipleine function, taking only a single argument `traj`.
            And returning all functions necessary for your experiment.

        :return:

            List of the individual results returned by `runfunc`.

            Returns a LIST OF TUPLES, where first entry is the run idx and second entry
            is the actual result. In case of multiprocessing these are not necessarily
            ordered according to their run index, but ordered according to their finishing time.

            Does not contain results stored in the trajectory!
            In order to access these simply interact with the trajectory object,
            potentially after calling :func:`~pypet.trajectory.Trajectory.f_update_skeleton`
            and loading all results at once with :func:`~pypet.trajectory.f_load`
            or loading manually with :func:`~pypet.trajectory.f_load_items`.

            Even if you use multiprocessing without a pool the results returned by
            `runfunc` still need to be pickled.

            Results computed from `postproc` are not returned. `postproc` should not
            return any results except dictionaries if the trajectory should be expanded.


self._user_pipeline = True<EOL>self._map_arguments = True<EOL>return self._execute_runs(pipeline)<EOL>

Creates a pipeline with iterable arguments


def run(self, runfunc, *args, **kwargs):

pipeline = lambda traj: ((runfunc, args, kwargs),<EOL>(self._postproc, self._postproc_args, self._postproc_kwargs))<EOL>self._user_pipeline = False<EOL>self._map_arguments = False<EOL>return self._execute_runs(pipeline)<EOL>

Runs the experiments and explores the parameter space.

        :param runfunc: The task or job to do

        :param args: Additional arguments (not the ones in the trajectory) passed to `runfunc`

        :param kwargs:

            Additional keyword arguments (not the ones in the trajectory) passed to `runfunc`

        :return:

            List of the individual results returned by `runfunc`.

            Returns a LIST OF TUPLES, where first entry is the run idx and second entry
            is the actual result. They are always ordered according to the run index.

            Does not contain results stored in the trajectory!
            In order to access these simply interact with the trajectory object,
            potentially after calling`~pypet.trajectory.Trajectory.f_update_skeleton`
            and loading all results at once with :func:`~pypet.trajectory.f_load`
            or loading manually with :func:`~pypet.trajectory.f_load_items`.

            If you use multiprocessing without a pool the results returned by
            `runfunc` still need to be pickled.


def run_map(self, runfunc, *iter_args, **iter_kwargs):

if len(iter_args) == <NUM_LIT:0> and len(iter_kwargs) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>pipeline = lambda traj: ((runfunc, iter_args, iter_kwargs),<EOL>(self._postproc, self._postproc_args, self._postproc_kwargs))<EOL>self._user_pipeline = False<EOL>self._map_arguments = True<EOL>return self._execute_runs(pipeline)<EOL>

Calls runfunc with different args and kwargs each time.

        Similar to `:func:`~pypet.environment.Environment.run`
        but all ``iter_args`` and ``iter_kwargs`` need to be iterables,
        iterators, or generators that return new arguments for each run.


dump_dict = {}<EOL>dump_filename = os.path.join(self._resume_path, '<STR_LIT>')<EOL>prev_full_copy = self._traj.v_full_copy<EOL>dump_dict['<STR_LIT>'] = prev_full_copy<EOL>self._traj.v_full_copy = True<EOL>prev_storage_service = self._traj.v_storage_service<EOL>self._traj.v_storage_service = self._storage_service<EOL>dump_dict['<STR_LIT>'] = self._traj<EOL>dump_dict['<STR_LIT:args>'] = self._args<EOL>dump_dict['<STR_LIT>'] = self._kwargs<EOL>dump_dict['<STR_LIT>'] = self._runfunc<EOL>dump_dict['<STR_LIT>'] = self._postproc<EOL>dump_dict['<STR_LIT>'] = self._postproc_args<EOL>dump_dict['<STR_LIT>'] = self._postproc_kwargs<EOL>dump_dict['<STR_LIT>'] = self._start_timestamp<EOL>dump_file = open(dump_filename, '<STR_LIT:wb>')<EOL>dill.dump(dump_dict, dump_file, protocol=<NUM_LIT:2>)<EOL>dump_file.flush()<EOL>dump_file.close()<EOL>self._traj.v_full_copy = prev_full_copy<EOL>self._traj.v_storage_service = prev_storage_service<EOL>

Makes the trajectory continuable in case the user wants that


reason = self._sumatra_reason<EOL>if reason:<EOL><INDENT>reason += '<STR_LIT>'<EOL><DEDENT>if self._traj.v_comment:<EOL><INDENT>commentstr = '<STR_LIT>' % self._traj.v_comment<EOL><DEDENT>else:<EOL><INDENT>commentstr = '<STR_LIT>'<EOL><DEDENT>reason += '<STR_LIT>' %(self._traj.v_name,<EOL>commentstr,<EOL>str(list(self._traj._explored_parameters.keys())))<EOL>self._logger.info('<STR_LIT>' % reason)<EOL>self._sumatra_reason = reason<EOL>self._loaded_sumatatra_project = load_project(self._sumatra_project)<EOL>if self._traj.f_contains('<STR_LIT>', shortcuts=False):<EOL><INDENT>param_dict = self._traj.parameters.f_to_dict(fast_access=False)<EOL>for param_name in list(param_dict.keys()):<EOL><INDENT>param = param_dict[param_name]<EOL>if param.f_has_range():<EOL><INDENT>param_dict[param_name] = param.f_get_range()<EOL><DEDENT>else:<EOL><INDENT>param_dict[param_name] = param.f_get()<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>param_dict = {}<EOL><DEDENT>relpath = os.path.relpath(sys.modules['<STR_LIT:__main__>'].__file__, self._sumatra_project)<EOL>executable = PythonExecutable(path=sys.executable)<EOL>self._sumatra_record = self._loaded_sumatatra_project.new_record(<EOL>parameters=param_dict,<EOL>main_file=relpath,<EOL>executable=executable,<EOL>label=self._sumatra_label,<EOL>reason=reason)<EOL>


finish_time = self._start_timestamp - self._finish_timestamp<EOL>self._sumatra_record.duration = finish_time<EOL>self._sumatra_record.output_data = self._sumatra_record.datastore.find_new_data(<EOL>self._sumatra_record.timestamp)<EOL>self._loaded_sumatatra_project.add_record(self._sumatra_record)<EOL>self._loaded_sumatatra_project.save()<EOL>sumatra_label = self._sumatra_record.label<EOL>config_name = '<STR_LIT>' % str(sumatra_label)<EOL>conf_list = []<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>conf1 = self._traj.f_add_config(Parameter, config_name, str(sumatra_label),<EOL>comment='<STR_LIT>')<EOL>conf_list.append(conf1)<EOL><DEDENT>if self._sumatra_reason:<EOL><INDENT>config_name = '<STR_LIT>' % str(sumatra_label)<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>conf2 = self._traj.f_add_config(Parameter, config_name,<EOL>str(self._sumatra_reason),<EOL>comment='<STR_LIT>')<EOL>conf_list.append(conf2)<EOL><DEDENT><DEDENT>if self._automatic_storing and conf_list:<EOL><INDENT>self._traj.f_store_items(conf_list)<EOL><DEDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT:%s>' % str(self._sumatra_reason))<EOL>


if not self._resumable:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if not self._do_single_runs:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>self._resume_path = os.path.join(self._resume_folder, self._trajectory_name)<EOL>cnt_filename = os.path.join(self._resume_path, '<STR_LIT>')<EOL>cnt_file = open(cnt_filename, '<STR_LIT:rb>')<EOL>resume_dict = dill.load(cnt_file)<EOL>cnt_file.close()<EOL>traj = resume_dict['<STR_LIT>']<EOL>config_name = '<STR_LIT>' % self.name<EOL>if self._traj.f_contains(config_name, shortcuts=False):<EOL><INDENT>param = self._traj.f_get(config_name, shortcuts=False)<EOL>param.f_unlock()<EOL>param.f_set(traj.v_name)<EOL>param.f_lock()<EOL><DEDENT>config_name = '<STR_LIT>' % self.name<EOL>if self._traj.f_contains(config_name, shortcuts=False):<EOL><INDENT>param = self._traj.f_get(config_name, shortcuts=False)<EOL>param.f_unlock()<EOL>param.f_set(traj.v_timestamp)<EOL>param.f_lock()<EOL><DEDENT>if not traj.config.environment.f_contains(self.name, shortcuts=False):<EOL><INDENT>traj._merge_config(self._traj)<EOL><DEDENT>self._traj = traj<EOL>self._runfunc = resume_dict['<STR_LIT>']<EOL>self._args = resume_dict['<STR_LIT:args>']<EOL>self._kwargs = resume_dict['<STR_LIT>']<EOL>self._postproc = resume_dict['<STR_LIT>']<EOL>self._postproc_args = resume_dict['<STR_LIT>']<EOL>self._postproc_kwargs = resume_dict['<STR_LIT>']<EOL>self._traj.v_full_copy = resume_dict['<STR_LIT>']<EOL>self._traj.f_load(load_parameters=pypetconstants.LOAD_NOTHING,<EOL>load_derived_parameters=pypetconstants.LOAD_NOTHING,<EOL>load_results=pypetconstants.LOAD_NOTHING,<EOL>load_other_data=pypetconstants.LOAD_NOTHING)<EOL>result_list = []<EOL>full_filename_list = []<EOL>for filename in os.listdir(self._resume_path):<EOL><INDENT>_, ext = os.path.splitext(filename)<EOL>if ext != '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>full_filename = os.path.join(self._resume_path, filename)<EOL>cnt_file = open(full_filename, '<STR_LIT:rb>')<EOL>result_list.append(dill.load(cnt_file))<EOL>cnt_file.close()<EOL>full_filename_list.append(full_filename)<EOL><DEDENT>new_result_list = []<EOL>for result_tuple in result_list:<EOL><INDENT>run_information = result_tuple[<NUM_LIT:1>]<EOL>self._traj._update_run_information(run_information)<EOL>new_result_list.append(result_tuple[<NUM_LIT:0>])<EOL><DEDENT>result_sort(new_result_list)<EOL>config_name = '<STR_LIT>' % self.name<EOL>if not config_name in self._traj:<EOL><INDENT>self._traj.f_add_config(Parameter, config_name, True,<EOL>comment='<STR_LIT>')<EOL><DEDENT>self._logger.info('<STR_LIT>' % self._traj.v_name)<EOL>return new_result_list<EOL>

Prepares the continuation of a crashed trajectory


pip_result = pipeline(self._traj)  <EOL>raise_error = False<EOL>if pip_result is None:<EOL><INDENT>if self._do_single_runs:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>self._runfunc = None<EOL>return<EOL><DEDENT>elif (len(pip_result) == <NUM_LIT:2> and<EOL>isinstance(pip_result[<NUM_LIT:0>], tuple) and<EOL>isinstance(pip_result[<NUM_LIT:1>], tuple)):<EOL><INDENT>run_tuple = pip_result[<NUM_LIT:0>]<EOL>self._runfunc = run_tuple[<NUM_LIT:0>]<EOL>if len(run_tuple) > <NUM_LIT:1>:<EOL><INDENT>self._args = run_tuple[<NUM_LIT:1>]<EOL><DEDENT>if len(run_tuple) > <NUM_LIT:2>:<EOL><INDENT>self._kwargs = run_tuple[<NUM_LIT:2>]<EOL><DEDENT>if len(run_tuple) > <NUM_LIT:3>:<EOL><INDENT>raise_error = True<EOL><DEDENT>postproc_tuple = pip_result[<NUM_LIT:1>]<EOL>if len(postproc_tuple) > <NUM_LIT:0>:<EOL><INDENT>self._postproc = postproc_tuple[<NUM_LIT:0>]<EOL><DEDENT>if len(postproc_tuple) > <NUM_LIT:1>:<EOL><INDENT>self._postproc_args = postproc_tuple[<NUM_LIT:1>]<EOL><DEDENT>if len(postproc_tuple) > <NUM_LIT:2>:<EOL><INDENT>self._postproc_kwargs = postproc_tuple[<NUM_LIT:2>]<EOL><DEDENT>if len(run_tuple) > <NUM_LIT:3>:<EOL><INDENT>raise_error = True<EOL><DEDENT><DEDENT>elif len(pip_result) <= <NUM_LIT:3>:<EOL><INDENT>self._runfunc = pip_result[<NUM_LIT:0>]<EOL>if len(pip_result) > <NUM_LIT:1>:<EOL><INDENT>self._args = pip_result[<NUM_LIT:1>]<EOL><DEDENT>if len(pip_result) > <NUM_LIT:2>:<EOL><INDENT>self._kwargs = pip_result[<NUM_LIT:2>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise_error = True<EOL><DEDENT>if raise_error:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if self._runfunc is not None and not self._do_single_runs:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if self._resumable:<EOL><INDENT>racedirs(self._resume_path)<EOL>if os.listdir(self._resume_path):<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>' % self._resume_path)<EOL><DEDENT><DEDENT>if self._user_pipeline:<EOL><INDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>self._traj.v_name)<EOL><DEDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>self._traj._prepare_experiment()<EOL>self._logger.info('<STR_LIT>')<EOL>self._traj.f_store(only_init=True)<EOL>

Prepares the running of an experiment

        :param pipeline:

            A pipeline function that defines the task


def _show_progress(self, n, total_runs):

self._logging_manager.show_progress(n, total_runs)<EOL>


result_dict = {'<STR_LIT>': self._traj,<EOL>'<STR_LIT>': self._logging_manager,<EOL>'<STR_LIT>': self._runfunc,<EOL>'<STR_LIT>': self._args,<EOL>'<STR_LIT>': self._kwargs,<EOL>'<STR_LIT>': self._clean_up_runs,<EOL>'<STR_LIT>': self._automatic_storing,<EOL>'<STR_LIT>': self._wrap_mode,<EOL>'<STR_LIT>': self._niceness,<EOL>'<STR_LIT>': self._graceful_exit}<EOL>result_dict.update(kwargs)<EOL>if self._multiproc:<EOL><INDENT>if self._use_pool or self._use_scoop:<EOL><INDENT>if self._use_scoop:<EOL><INDENT>del result_dict['<STR_LIT>']<EOL><DEDENT>if self._freeze_input:<EOL><INDENT>result_dict['<STR_LIT>'] = self.traj.v_full_copy<EOL>if self._map_arguments:<EOL><INDENT>del result_dict['<STR_LIT>']<EOL>del result_dict['<STR_LIT>']<EOL><DEDENT><DEDENT>else:<EOL><INDENT>result_dict['<STR_LIT>'] = False<EOL>if self._use_pool:<EOL><INDENT>del result_dict['<STR_LIT>']<EOL>del result_dict['<STR_LIT>']<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>result_dict['<STR_LIT>'] = False<EOL><DEDENT><DEDENT>return result_dict<EOL>

Creates the keyword arguments for the single run handling


total_runs = len(self._traj)<EOL>for n in range(start_run_idx, total_runs):<EOL><INDENT>self._current_idx = n + <NUM_LIT:1><EOL>if self._stop_iteration:<EOL><INDENT>self._logger.debug('<STR_LIT>')<EOL>break<EOL><DEDENT>if not self._traj._is_completed(n):<EOL><INDENT>self._traj.f_set_crun(n)<EOL>yield n<EOL><DEDENT>else:<EOL><INDENT>self._logger.debug('<STR_LIT>' % n)<EOL><DEDENT><DEDENT>

Returns an iterator over the run indices that are not completed


def _make_iterator(self, start_run_idx, copy_data=False, **kwargs):

if (not self._freeze_input) or (not self._multiproc):<EOL><INDENT>kwargs = self._make_kwargs(**kwargs)<EOL><DEDENT>def _do_iter():<EOL><INDENT>if self._map_arguments:<EOL><INDENT>self._args = tuple(iter(arg) for arg in self._args)<EOL>for key in list(self._kwargs.keys()):<EOL><INDENT>self._kwargs[key] = iter(self._kwargs[key])<EOL><DEDENT>for idx in self._make_index_iterator(start_run_idx):<EOL><INDENT>iter_args = tuple(next(x) for x in self._args)<EOL>iter_kwargs = {}<EOL>for key in self._kwargs:<EOL><INDENT>iter_kwargs[key] = next(self._kwargs[key])<EOL><DEDENT>kwargs['<STR_LIT>'] = iter_args<EOL>kwargs['<STR_LIT>'] = iter_kwargs<EOL>if self._freeze_input:<EOL><INDENT>kwargs['<STR_LIT>'] = idx<EOL><DEDENT>if copy_data:<EOL><INDENT>copied_kwargs = kwargs.copy()<EOL>if not self._freeze_input:<EOL><INDENT>copied_kwargs['<STR_LIT>'] = self._traj.f_copy(copy_leaves='<STR_LIT>',<EOL>with_links=True)<EOL><DEDENT>yield copied_kwargs<EOL><DEDENT>else:<EOL><INDENT>yield kwargs<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for idx in self._make_index_iterator(start_run_idx):<EOL><INDENT>if self._freeze_input:<EOL><INDENT>kwargs['<STR_LIT>'] = idx<EOL><DEDENT>if copy_data:<EOL><INDENT>copied_kwargs = kwargs.copy()<EOL>if not self._freeze_input:<EOL><INDENT>copied_kwargs['<STR_LIT>'] = self._traj.f_copy(copy_leaves='<STR_LIT>',<EOL>with_links=True)<EOL><DEDENT>yield copied_kwargs<EOL><DEDENT>else:<EOL><INDENT>yield kwargs<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return _do_iter()<EOL>

Returns an iterator over all runs and yields the keyword arguments


repeat = False<EOL>start_run_idx = <NUM_LIT:0><EOL>new_runs = <NUM_LIT:0><EOL>self._traj._finalize(store_meta_data=True)<EOL>old_traj_length = len(self._traj)<EOL>postproc_res = self._postproc(self._traj, results,<EOL>*self._postproc_args, **self._postproc_kwargs)<EOL>if postproc_res is None:<EOL><INDENT>pass<EOL><DEDENT>elif isinstance(postproc_res, dict):<EOL><INDENT>if postproc_res:<EOL><INDENT>self._traj.f_expand(postproc_res)<EOL><DEDENT><DEDENT>elif isinstance(postproc_res, tuple):<EOL><INDENT>expand_dict = postproc_res[<NUM_LIT:0>]<EOL>if len(postproc_res) > <NUM_LIT:1>:<EOL><INDENT>self._args = postproc_res[<NUM_LIT:1>]<EOL><DEDENT>if len(postproc_res) > <NUM_LIT:2>:<EOL><INDENT>self._kwargs = postproc_res[<NUM_LIT:2>]<EOL><DEDENT>if len(postproc_res) > <NUM_LIT:3>:<EOL><INDENT>self._postproc_args = postproc_res[<NUM_LIT:3>]<EOL><DEDENT>if len(postproc_res) > <NUM_LIT:4>:<EOL><INDENT>self._postproc_kwargs = postproc_res[<NUM_LIT:4>]<EOL><DEDENT>if expand_dict:<EOL><INDENT>self._traj.f_expand(expand_dict)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self._logger.error('<STR_LIT>' % str(postproc_res))<EOL><DEDENT>new_traj_length = len(self._traj)<EOL>if new_traj_length != old_traj_length:<EOL><INDENT>start_run_idx = old_traj_length<EOL>repeat = True<EOL>if self._resumable:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>self._traj.f_store(only_init=True)<EOL>new_traj_length = len(self._traj)<EOL>new_runs = new_traj_length - old_traj_length<EOL><DEDENT>return repeat, start_run_idx, new_runs<EOL>

Executes a postprocessing function

        :param results:

            List of tuples containing the run indices and the results

        :return:

            1. Whether to new single runs, since the trajectory was enlarged
            2. Index of next new run
            3. Number of new runs


now = time.time()<EOL>if now - self._last_cpu_check >= <NUM_LIT:0.5>:<EOL><INDENT>try:<EOL><INDENT>self._last_cpu_usage = psutil.cpu_percent()<EOL>self._last_cpu_check = now<EOL><DEDENT>except (psutil.NoSuchProcess, ZeroDivisionError):<EOL><INDENT>pass  <EOL><DEDENT><DEDENT>return self._last_cpu_usage<EOL>

Estimates the cpu utilization within the last 500ms


n_processes = len(process_dict)<EOL>total_utilization = psutil.virtual_memory().percent<EOL>sum = <NUM_LIT:0.0><EOL>for proc in process_dict.values():<EOL><INDENT>try:<EOL><INDENT>sum += psutil.Process(proc.pid).memory_percent()<EOL><DEDENT>except (psutil.NoSuchProcess, ZeroDivisionError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>curr_all_processes = sum<EOL>missing_utilization = max(<NUM_LIT:0.0>, n_processes * self._est_per_process - curr_all_processes)<EOL>estimated_utilization = total_utilization<EOL>estimated_utilization += missing_utilization<EOL>estimated_utilization += self._est_per_process<EOL>return estimated_utilization<EOL>

Estimates memory utilization to come if process was started


if self._start_timestamp is None:<EOL><INDENT>self._start_timestamp = time.time()<EOL><DEDENT>if self._map_arguments and self._resumable:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if self._sumatra_project is not None:<EOL><INDENT>self._prepare_sumatra()<EOL><DEDENT>if pipeline is not None:<EOL><INDENT>results = []<EOL>self._prepare_runs(pipeline)<EOL><DEDENT>else:<EOL><INDENT>results = self._prepare_resume()<EOL><DEDENT>if self._runfunc is not None:<EOL><INDENT>self._traj._run_by_environment = True<EOL>if self._graceful_exit:<EOL><INDENT>sigint_handling.start()<EOL><DEDENT>try:<EOL><INDENT>self._inner_run_loop(results)<EOL><DEDENT>finally:<EOL><INDENT>self._traj._run_by_environment = False<EOL>self._stop_iteration = False<EOL>if self._graceful_exit:<EOL><INDENT>sigint_handling.finalize()<EOL><DEDENT><DEDENT><DEDENT>self._add_wildcard_config()<EOL>if self._automatic_storing:<EOL><INDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>self._traj.v_name)<EOL>self._traj.f_store()<EOL>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>self._traj.v_name)<EOL><DEDENT>self._finish_timestamp = time.time()<EOL>findatetime = datetime.datetime.fromtimestamp(self._finish_timestamp)<EOL>startdatetime = datetime.datetime.fromtimestamp(self._start_timestamp)<EOL>self._runtime = str(findatetime - startdatetime)<EOL>conf_list = []<EOL>config_name = '<STR_LIT>' % self.name<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>conf1 = self._traj.f_add_config(Parameter, config_name, self._start_timestamp,<EOL>comment='<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>conf_list.append(conf1)<EOL><DEDENT>config_name = '<STR_LIT>' % self.name<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>conf2 = self._traj.f_add_config(Parameter, config_name, self._finish_timestamp,<EOL>comment='<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>conf2 = self._traj.f_get('<STR_LIT>' + config_name)<EOL>conf2.f_unlock()<EOL>conf2.f_set(self._finish_timestamp)<EOL><DEDENT>conf_list.append(conf2)<EOL>config_name = '<STR_LIT>' % self.name<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>conf3 = self._traj.f_add_config(Parameter, config_name, self._runtime,<EOL>comment='<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>conf3 = self._traj.f_get('<STR_LIT>' + config_name)<EOL>conf3.f_unlock()<EOL>conf3.f_set(self._runtime)<EOL><DEDENT>conf_list.append(conf3)<EOL>if self._automatic_storing:<EOL><INDENT>self._traj.f_store_items(conf_list, store_data=pypetconstants.OVERWRITE_DATA)<EOL><DEDENT>if hasattr(self._traj.v_storage_service, '<STR_LIT>'):<EOL><INDENT>self._traj.v_storage_service.finalize()<EOL><DEDENT>incomplete = []<EOL>for run_name in self._traj.f_get_run_names():<EOL><INDENT>if not self._traj._is_completed(run_name):<EOL><INDENT>incomplete.append(run_name)<EOL><DEDENT><DEDENT>if len(incomplete) > <NUM_LIT:0>:<EOL><INDENT>self._logger.error('<STR_LIT>'<EOL>'<STR_LIT>' % (self._traj.v_name,<EOL>'<STR_LIT:U+002CU+0020>'.join(incomplete)))<EOL><DEDENT>else:<EOL><INDENT>self._logger.info('<STR_LIT>' %<EOL>self._traj.v_name)<EOL><DEDENT>if self._sumatra_project is not None:<EOL><INDENT>self._finish_sumatra()<EOL><DEDENT>return results<EOL>

Starts the individual single runs.

        Starts runs sequentially or initiates multiprocessing.

        :param pipeline:

            A pipeline function producing the run function the corresponding arguments
            and postprocessing function and arguments

        :return:

            List of tuples, where each tuple contains the run idx and the result.


for idx, pair in enumerate(self._traj._wildcard_functions.items()):<EOL><INDENT>wildcards, wc_function = pair<EOL>for jdx, wildcard in enumerate(wildcards):<EOL><INDENT>config_name = ('<STR_LIT>' %<EOL>(self.name, idx, jdx))<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>self._traj.f_add_config(Parameter, config_name, wildcard,<EOL>comment='<STR_LIT>').f_lock()<EOL><DEDENT><DEDENT>if hasattr(wc_function, '<STR_LIT>'):<EOL><INDENT>config_name = ('<STR_LIT>' %<EOL>(self.name, idx))<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>self._traj.f_add_config(Parameter, config_name, wc_function.__name__,<EOL>comment='<STR_LIT>').f_lock()<EOL><DEDENT><DEDENT>if wc_function.__doc__:<EOL><INDENT>config_name = ('<STR_LIT>' %<EOL>(self.name, idx))<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>self._traj.f_add_config(Parameter, config_name, wc_function.__doc__,<EOL>comment='<STR_LIT>').f_lock()<EOL><DEDENT><DEDENT>try:<EOL><INDENT>source = inspect.getsource(wc_function)<EOL>config_name = ('<STR_LIT>' %<EOL>(self.name, idx))<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>self._traj.f_add_config(Parameter, config_name, source,<EOL>comment='<STR_LIT>').f_lock()<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>

Adds config data about the wildcard functions


start_run_idx = self._current_idx<EOL>expanded_by_postproc = False<EOL>self._storage_service = self._traj.v_storage_service<EOL>self._multiproc_wrapper = None<EOL>if self._resumable:<EOL><INDENT>self._trigger_resume_snapshot()<EOL><DEDENT>self._logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>self._traj.v_name)<EOL>while True:<EOL><INDENT>if self._multiproc:<EOL><INDENT>expanded_by_postproc = self._execute_multiprocessing(start_run_idx, results)<EOL><DEDENT>else:<EOL><INDENT>iterator = self._make_iterator(start_run_idx)<EOL>n = start_run_idx<EOL>total_runs = len(self._traj)<EOL>self._show_progress(n - <NUM_LIT:1>, total_runs)<EOL>for task in iterator:<EOL><INDENT>result = _sigint_handling_single_run(task)<EOL>n = self._check_result_and_store_references(result, results,<EOL>n, total_runs)<EOL><DEDENT><DEDENT>repeat = False<EOL>if self._postproc is not None:<EOL><INDENT>self._logger.info('<STR_LIT>')<EOL>repeat, start_run_idx, new_runs = self._execute_postproc(results)<EOL><DEDENT>if not repeat:<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>expanded_by_postproc = True<EOL>self._logger.info('<STR_LIT>' %<EOL>new_runs)<EOL><DEDENT><DEDENT>self._traj._finalize(store_meta_data=True)<EOL>self._logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>self._traj.v_name)<EOL>if self._resumable and self._delete_resume:<EOL><INDENT>shutil.rmtree(self._resume_path)<EOL><DEDENT>if expanded_by_postproc:<EOL><INDENT>config_name = '<STR_LIT>' % self.name<EOL>if not self._traj.f_contains('<STR_LIT>' + config_name):<EOL><INDENT>self._traj.f_add_config(Parameter, config_name, True,<EOL>comment='<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>

Performs the inner loop of the run execution


def _get_results_from_queue(self, result_queue, results, n, total_runs):

<EOL>while not result_queue.empty():<EOL><INDENT>result = result_queue.get()<EOL>n = self._check_result_and_store_references(result, results, n, total_runs)<EOL><DEDENT>return n<EOL>

Extract all available results from the queue and returns the increased n


def _check_result_and_store_references(self, result, results, n, total_runs):

if result[<NUM_LIT:0>] == sigint_handling.SIGINT:<EOL><INDENT>self._stop_iteration = True<EOL>result = result[<NUM_LIT:1>]  <EOL><DEDENT>if result is not None:<EOL><INDENT>if self._wrap_mode == pypetconstants.WRAP_MODE_LOCAL:<EOL><INDENT>self._multiproc_wrapper.store_references(result[<NUM_LIT:2>])<EOL><DEDENT>self._traj._update_run_information(result[<NUM_LIT:1>])<EOL>results.append(result[<NUM_LIT:0>])<EOL>if self._resumable:<EOL><INDENT>self._trigger_result_snapshot(result[<NUM_LIT:0>:<NUM_LIT:2>])<EOL><DEDENT><DEDENT>self._show_progress(n, total_runs)<EOL>n += <NUM_LIT:1><EOL>return n<EOL>

Checks for SIGINT and if reference wrapping and stores references.


timestamp = result[<NUM_LIT:1>]['<STR_LIT>']<EOL>timestamp_str = repr(timestamp).replace('<STR_LIT:.>', '<STR_LIT:_>')<EOL>filename = '<STR_LIT>' % timestamp_str<EOL>extension = '<STR_LIT>'<EOL>dump_filename = os.path.join(self._resume_path, filename + extension)<EOL>dump_file = open(dump_filename, '<STR_LIT:wb>')<EOL>dill.dump(result, dump_file, protocol=<NUM_LIT:2>)<EOL>dump_file.flush()<EOL>dump_file.close()<EOL>extension = '<STR_LIT>'<EOL>rename_filename = os.path.join(self._resume_path, filename + extension)<EOL>shutil.move(dump_filename, rename_filename)<EOL>

Triggers a snapshot of the results for continuing

        :param result: Currently computed result


def _execute_multiprocessing(self, start_run_idx, results):

n = start_run_idx<EOL>total_runs = len(self._traj)<EOL>expanded_by_postproc = False<EOL>if (self._wrap_mode == pypetconstants.WRAP_MODE_NONE or<EOL>self._storage_service.multiproc_safe):<EOL><INDENT>self._logger.info('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>use_manager = (self._wrap_mode == pypetconstants.WRAP_MODE_QUEUE or<EOL>self._immediate_postproc)<EOL>self._multiproc_wrapper = MultiprocContext(self._traj,<EOL>self._wrap_mode,<EOL>full_copy=None,<EOL>manager=None,<EOL>use_manager=use_manager,<EOL>lock=None,<EOL>queue=None,<EOL>queue_maxsize=self._queue_maxsize,<EOL>port=self._url,<EOL>timeout=self._timeout,<EOL>gc_interval=self._gc_interval,<EOL>log_config=self._logging_manager.log_config,<EOL>log_stdout=self._logging_manager.log_stdout,<EOL>graceful_exit=self._graceful_exit)<EOL>self._multiproc_wrapper.start()<EOL><DEDENT>try:<EOL><INDENT>if self._use_pool:<EOL><INDENT>self._logger.info('<STR_LIT>' % self._ncores)<EOL>if self._freeze_input:<EOL><INDENT>self._logger.info('<STR_LIT>')<EOL>init_kwargs = self._make_kwargs()<EOL>pool_full_copy = self._traj.v_full_copy<EOL>self._traj.v_full_copy = True<EOL>initializer = _configure_frozen_pool<EOL>target = _frozen_pool_single_run<EOL><DEDENT>else:<EOL><INDENT>pool_service = self._traj.v_storage_service<EOL>self._traj.v_storage_service = None<EOL>init_kwargs = dict(logging_manager=self._logging_manager,<EOL>storage_service=pool_service,<EOL>niceness=self._niceness)<EOL>initializer = _configure_pool<EOL>target = _pool_single_run<EOL><DEDENT>try:<EOL><INDENT>iterator = self._make_iterator(start_run_idx)<EOL>mpool = multip.Pool(self._ncores, initializer=initializer,<EOL>initargs=(init_kwargs,))<EOL>pool_results = mpool.imap(target, iterator)<EOL>self._show_progress(n - <NUM_LIT:1>, total_runs)<EOL>for result in pool_results:<EOL><INDENT>n = self._check_result_and_store_references(result, results,<EOL>n, total_runs)<EOL><DEDENT>mpool.close()<EOL>mpool.join()<EOL><DEDENT>finally:<EOL><INDENT>if self._freeze_input:<EOL><INDENT>self._traj.v_full_copy = pool_full_copy<EOL><DEDENT>else:<EOL><INDENT>self._traj.v_storage_service = pool_service<EOL><DEDENT><DEDENT>self._logger.info('<STR_LIT>')<EOL>del mpool<EOL><DEDENT>elif self._use_scoop:<EOL><INDENT>self._logger.info('<STR_LIT>')<EOL>if self._freeze_input:<EOL><INDENT>self._logger.info('<STR_LIT>')<EOL>if not hasattr(_frozen_scoop_single_run, '<STR_LIT>'):<EOL><INDENT>_frozen_scoop_single_run.kwargs = {}<EOL><DEDENT>scoop_full_copy = self._traj.v_full_copy<EOL>self._traj.v_full_copy = True<EOL>init_kwargs = self._make_kwargs()<EOL>scoop_rev = self.name + '<STR_LIT:_>' + str(time.time()).replace('<STR_LIT:.>','<STR_LIT:_>')<EOL>shared.setConst(**{scoop_rev: init_kwargs})<EOL>iterator = self._make_iterator(start_run_idx,<EOL>copy_data=True,<EOL>scoop_rev=scoop_rev)<EOL>target = _frozen_scoop_single_run<EOL><DEDENT>else:<EOL><INDENT>iterator = self._make_iterator(start_run_idx,<EOL>copy_data=True)<EOL>target = _scoop_single_run<EOL><DEDENT>try:<EOL><INDENT>if scoop.IS_RUNNING:<EOL><INDENT>scoop_results = futures.map(target, iterator, timeout=self._timeout)<EOL><DEDENT>else:<EOL><INDENT>self._logger.error('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>scoop_results = map(target, iterator)<EOL><DEDENT>self._show_progress(n - <NUM_LIT:1>, total_runs)<EOL>for result in scoop_results:<EOL><INDENT>n = self._check_result_and_store_references(result, results,<EOL>n, total_runs)<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>if self._freeze_input:<EOL><INDENT>self._traj.v_full_copy = scoop_full_copy<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if self._immediate_postproc:<EOL><INDENT>maxsize = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>maxsize = total_runs<EOL><DEDENT>start_result_length = len(results)<EOL>result_queue = multip.Queue(maxsize=maxsize)<EOL>iterator = self._make_iterator(start_run_idx, result_queue=result_queue)<EOL>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>' % self._ncores)<EOL>if self._check_usage:<EOL><INDENT>self._logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(self._cpu_cap, self._memory_cap[<NUM_LIT:0>], self._swap_cap))<EOL><DEDENT>keep_running = True  <EOL>process_dict = {}  <EOL>cpu_usage_func = lambda: self._estimate_cpu_utilization()<EOL>memory_usage_func = lambda: self._estimate_memory_utilization(process_dict)<EOL>swap_usage_func = lambda: psutil.swap_memory().percent<EOL>signal_cap = True  <EOL>max_signals = <NUM_LIT:10>  <EOL>self._show_progress(n - <NUM_LIT:1>, total_runs)<EOL>while len(process_dict) > <NUM_LIT:0> or keep_running:<EOL><INDENT>for pid in list(process_dict.keys()):<EOL><INDENT>proc = process_dict[pid]<EOL>if not proc.is_alive():<EOL><INDENT>proc.join()<EOL>del process_dict[pid]<EOL>del proc<EOL><DEDENT><DEDENT>no_cap = True<EOL>if self._check_usage and self._ncores > len(process_dict) > <NUM_LIT:0>:<EOL><INDENT>for cap_name, cap_function, threshold in (<EOL>('<STR_LIT>', cpu_usage_func, self._cpu_cap),<EOL>('<STR_LIT>', memory_usage_func, self._memory_cap[<NUM_LIT:0>]),<EOL>('<STR_LIT>', swap_usage_func, self._swap_cap)):<EOL><INDENT>cap_value = cap_function()<EOL>if cap_value > threshold:<EOL><INDENT>no_cap = False<EOL>if signal_cap:<EOL><INDENT>if cap_name == '<STR_LIT>':<EOL><INDENT>add_on_str = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>add_on_str = '<STR_LIT>'<EOL><DEDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(len(process_dict), cap_name,<EOL>cap_value, threshold,<EOL>add_on_str))<EOL>signal_cap = False<EOL>max_signals -= <NUM_LIT:1><EOL>if max_signals == <NUM_LIT:0>:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>break  <EOL><DEDENT><DEDENT><DEDENT>if len(process_dict) < self._ncores and keep_running and no_cap:<EOL><INDENT>try:<EOL><INDENT>task = next(iterator)<EOL>proc = multip.Process(target=_process_single_run,<EOL>args=(task,))<EOL>proc.start()<EOL>process_dict[proc.pid] = proc<EOL>signal_cap = max_signals > <NUM_LIT:0>  <EOL><DEDENT>except StopIteration:<EOL><INDENT>keep_running = False<EOL>if self._postproc is not None and self._immediate_postproc:<EOL><INDENT>if self._wrap_mode == pypetconstants.WRAP_MODE_LOCAL:<EOL><INDENT>reference_service = self._traj._storage_service<EOL>self._traj.v_storage_service = self._storage_service<EOL><DEDENT>try:<EOL><INDENT>self._logger.info('<STR_LIT>')<EOL>keep_running, start_run_idx, new_runs =self._execute_postproc(results)<EOL><DEDENT>finally:<EOL><INDENT>if self._wrap_mode == pypetconstants.WRAP_MODE_LOCAL:<EOL><INDENT>self._traj._storage_service = reference_service<EOL><DEDENT><DEDENT>if keep_running:<EOL><INDENT>expanded_by_postproc = True<EOL>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % new_runs)<EOL>n = start_run_idx<EOL>total_runs = len(self._traj)<EOL>iterator = self._make_iterator(start_run_idx,<EOL>result_queue=result_queue)<EOL><DEDENT><DEDENT>if not keep_running:<EOL><INDENT>self._logger.debug('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>time.sleep(<NUM_LIT>)<EOL><DEDENT>n = self._get_results_from_queue(result_queue, results, n, total_runs)<EOL><DEDENT>self._get_results_from_queue(result_queue, results, n, total_runs)<EOL>result_queue.close()<EOL>result_queue.join_thread()<EOL>del result_queue<EOL>result_sort(results, start_result_length)<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>if self._multiproc_wrapper is not None:<EOL><INDENT>self._multiproc_wrapper.finalize()<EOL>self._multiproc_wrapper = None<EOL><DEDENT><DEDENT>return expanded_by_postproc<EOL>

Performs multiprocessing and signals expansion by postproc


self._reference_store.store_references(references)<EOL>

In case of reference wrapping, stores data.

        :param references: References dictionary from a ReferenceWrapper.

        :param gc_collect: If ``gc.collect`` should be called.

        :param n:

            Alternatively if ``gc_interval`` is set, a current index can be passed.
            Data is stored in case ``n % gc_interval == 0``.


Starts the multiprocess wrapping.

        Automatically called when used as context manager.


<EOL>self._traj.f_store(only_init=True)<EOL>if self._wrap_mode == pypetconstants.WRAP_MODE_QUEUE:<EOL><INDENT>self._prepare_queue()<EOL><DEDENT>elif self._wrap_mode == pypetconstants.WRAP_MODE_LOCK:<EOL><INDENT>self._prepare_lock()<EOL><DEDENT>elif self._wrap_mode == pypetconstants.WRAP_MODE_PIPE:<EOL><INDENT>self._prepare_pipe()<EOL><DEDENT>elif self._wrap_mode == pypetconstants.WRAP_MODE_LOCAL:<EOL><INDENT>self._prepare_local()<EOL><DEDENT>elif self._wrap_mode == pypetconstants.WRAP_MODE_NETLOCK:<EOL><INDENT>self._prepare_netlock()<EOL><DEDENT>elif self._wrap_mode == pypetconstants.WRAP_MODE_NETQUEUE:<EOL><INDENT>self._prepare_netqueue()<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (self._wrap_mode, pypetconstants.WRAP_MODE_QUEUE,<EOL>pypetconstants.WRAP_MODE_LOCK,<EOL>pypetconstants.WRAP_MODE_PIPE,<EOL>pypetconstants.WRAP_MODE_LOCAL,<EOL>pypetconstants.WRAP_MODE_NETLOCK))<EOL><DEDENT>


if not isinstance(self._port, str):<EOL><INDENT>url = port_to_tcp(self._port)<EOL>self._logger.info('<STR_LIT>' % url)<EOL><DEDENT>else:<EOL><INDENT>url = self._port<EOL><DEDENT>if self._lock is None:<EOL><INDENT>if hasattr(os, '<STR_LIT>'):<EOL><INDENT>self._lock = ForkAwareLockerClient(url)<EOL><DEDENT>else:<EOL><INDENT>self._lock = LockerClient(url)<EOL><DEDENT><DEDENT>if self._timeout is None:<EOL><INDENT>lock_server = LockerServer(url)<EOL><DEDENT>else:<EOL><INDENT>lock_server = TimeOutLockerServer(url, self._timeout)<EOL>self._logger.info('<STR_LIT>')<EOL><DEDENT>self._lock_process = multip.Process(name='<STR_LIT>', target=_wrap_handling,<EOL>args=(dict(handler=lock_server,<EOL>logging_manager=self._logging_manager,<EOL>graceful_exit=self._graceful_exit),))<EOL>self._lock_process.start()<EOL>self._lock.start()<EOL>lock_wrapper = LockWrapper(self._storage_service, self._lock)<EOL>self._traj.v_storage_service = lock_wrapper<EOL>self._lock_wrapper = lock_wrapper<EOL>

Replaces the trajectory's service with a LockWrapper


if self._lock is None:<EOL><INDENT>if self._use_manager:<EOL><INDENT>if self._manager is None:<EOL><INDENT>self._manager = multip.Manager()<EOL><DEDENT>self._lock = self._manager.Lock()<EOL><DEDENT>else:<EOL><INDENT>self._lock = multip.Lock()<EOL><DEDENT><DEDENT>lock_wrapper = LockWrapper(self._storage_service, self._lock)<EOL>self._traj.v_storage_service = lock_wrapper<EOL>self._lock_wrapper = lock_wrapper<EOL>


if self._pipe is None:<EOL><INDENT>self._pipe = multip.Pipe(True)<EOL><DEDENT>if self._lock is None:<EOL><INDENT>self._lock = multip.Lock()<EOL><DEDENT>self._logger.info('<STR_LIT>')<EOL>pipe_handler = PipeStorageServiceWriter(self._storage_service, self._pipe[<NUM_LIT:0>],<EOL>max_buffer_size=self._max_buffer_size)<EOL>self._pipe_process = multip.Process(name='<STR_LIT>', target=_wrap_handling,<EOL>args=(dict(handler=pipe_handler,<EOL>logging_manager=self._logging_manager,<EOL>graceful_exit=self._graceful_exit),))<EOL>self._pipe_process.start()<EOL>self._pipe_wrapper = PipeStorageServiceSender(self._pipe[<NUM_LIT:1>], self._lock)<EOL>self._traj.v_storage_service = self._pipe_wrapper<EOL>

Replaces the trajectory's service with a queue sender and starts the queue process.


if self._queue is None:<EOL><INDENT>if self._use_manager:<EOL><INDENT>if self._manager is None:<EOL><INDENT>self._manager = multip.Manager()<EOL><DEDENT>self._queue = self._manager.Queue(maxsize=self._queue_maxsize)<EOL><DEDENT>else:<EOL><INDENT>self._queue = multip.Queue(maxsize=self._queue_maxsize)<EOL><DEDENT><DEDENT>self._logger.info('<STR_LIT>')<EOL>queue_handler = QueueStorageServiceWriter(self._storage_service, self._queue,<EOL>self._gc_interval)<EOL>self._queue_process = multip.Process(name='<STR_LIT>', target=_wrap_handling,<EOL>args=(dict(handler=queue_handler,<EOL>logging_manager=self._logging_manager,<EOL>graceful_exit=self._graceful_exit),))<EOL>self._queue_process.start()<EOL>self._queue_wrapper = QueueStorageServiceSender(self._queue)<EOL>self._traj.v_storage_service = self._queue_wrapper<EOL>


self._logger.info('<STR_LIT>')<EOL>if not isinstance(self._port, str):<EOL><INDENT>url = port_to_tcp(self._port)<EOL>self._logger.info('<STR_LIT>' % url)<EOL><DEDENT>else:<EOL><INDENT>url = self._port<EOL><DEDENT>if self._queue is None:<EOL><INDENT>if hasattr(os, '<STR_LIT>'):<EOL><INDENT>self._queue = ForkAwareQueuingClient(url)<EOL><DEDENT>else:<EOL><INDENT>self._queue = QueuingClient(url)<EOL><DEDENT><DEDENT>queuing_server_handler = QueuingServer(url,<EOL>self._storage_service,<EOL>self._queue_maxsize,<EOL>self._gc_interval)<EOL>self._queue_process = multip.Process(name='<STR_LIT>', target=_wrap_handling,<EOL>args=(dict(handler=queuing_server_handler,<EOL>logging_manager=self._logging_manager,<EOL>graceful_exit=self._graceful_exit),))<EOL>self._queue_process.start()<EOL>self._queue.start()<EOL>self._queue_wrapper = QueueStorageServiceSender(self._queue)<EOL>self._traj.v_storage_service = self._queue_wrapper<EOL>


if (self._wrap_mode == pypetconstants.WRAP_MODE_QUEUE and<EOL>self._queue_process is not None):<EOL><INDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>self._traj.v_storage_service.queue = self._queue<EOL>self._traj.v_storage_service.send_done()<EOL>self._queue_process.join()<EOL>if hasattr(self._queue, '<STR_LIT>'):<EOL><INDENT>self._queue.join()<EOL><DEDENT>if hasattr(self._queue, '<STR_LIT>'):<EOL><INDENT>self._queue.close()<EOL><DEDENT>if hasattr(self._queue, '<STR_LIT>'):<EOL><INDENT>self._queue.join_thread()<EOL><DEDENT>self._logger.info('<STR_LIT>')<EOL><DEDENT>elif (self._wrap_mode == pypetconstants.WRAP_MODE_PIPE and<EOL>self._pipe_process is not None):<EOL><INDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>self._traj.v_storage_service.conn = self._pipe[<NUM_LIT:1>]<EOL>self._traj.v_storage_service.send_done()<EOL>self._pipe_process.join()<EOL>self._pipe[<NUM_LIT:1>].close()<EOL>self._pipe[<NUM_LIT:0>].close()<EOL><DEDENT>elif (self._wrap_mode == pypetconstants.WRAP_MODE_NETLOCK and<EOL>self._lock_process is not None):<EOL><INDENT>self._lock.send_done()<EOL>self._lock.finalize()<EOL>self._lock_process.join()<EOL><DEDENT>elif (self._wrap_mode == pypetconstants.WRAP_MODE_NETQUEUE and<EOL>self._queue_process is not None):<EOL><INDENT>self._queue.send_done()<EOL>self._queue.finalize()<EOL>self._queue_process.join()<EOL><DEDENT>if self._manager is not None:<EOL><INDENT>self._manager.shutdown()<EOL><DEDENT>self._manager = None<EOL>self._queue_process = None<EOL>self._queue = None<EOL>self._queue_wrapper = None<EOL>self._lock = None<EOL>self._lock_wrapper = None<EOL>self._lock_process = None<EOL>self._reference_wrapper = None<EOL>self._pipe = None<EOL>self._pipe_process = None<EOL>self._pipe_wrapper = None<EOL>self._logging_manager = None<EOL>self._traj._storage_service = self._storage_service<EOL>

Restores the original storage service.

        If a queue process and a manager were used both are shut down.

        Automatically called when used as context manager.


slots_iterator = (getattr(c, '<STR_LIT>', ()) for c in cls.__mro__)<EOL>slots_converted = ((slots,) if isinstance(slots, str) else slots<EOL>for slots in slots_iterator)<EOL>all_slots = set()<EOL>all_slots.update(*slots_converted)<EOL>return all_slots<EOL>

Iterates through a class' (`cls`) mro to get all slots as a set.


for key in state:<EOL><INDENT>setattr(self, key, state[key])<EOL><DEDENT>


result = set()<EOL>result.update(dir(self.__class__), self.__all_slots__)<EOL>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>result.update(self.__dict__.keys())<EOL><DEDENT>return list(result)<EOL>


return self.f_get(item)<EOL>


self.f_set(**{key: value})<EOL>

Almost equivalent to calling __setattr__.

        Treats integer values as `f_get`.


if copy:<EOL><INDENT>return self._dict.copy()<EOL><DEDENT>else:<EOL><INDENT>return self._dict<EOL><DEDENT>

Returns annotations as dictionary.

        :param copy: Whether to return a shallow copy or the real thing (aka _dict).


return len(self._dict) == <NUM_LIT:0><EOL>


self._dict_ = None<EOL>


if len(args) == <NUM_LIT:0>:<EOL><INDENT>if len(self._dict) == <NUM_LIT:1>:<EOL><INDENT>return self._dict[list(self._dict.keys())[<NUM_LIT:0>]]<EOL><DEDENT>elif len(self._dict) > <NUM_LIT:1>:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(str(list(self._dict.keys()))))<EOL><DEDENT>else:<EOL><INDENT>raise AttributeError('<STR_LIT>')<EOL><DEDENT><DEDENT>result_list = []<EOL>for name in args:<EOL><INDENT>name = self._translate_key(name)<EOL>try:<EOL><INDENT>result_list.append(self._dict[name])<EOL><DEDENT>except KeyError:<EOL><INDENT>raise AttributeError('<STR_LIT>' % name)<EOL><DEDENT><DEDENT>if len(args) == <NUM_LIT:1>:<EOL><INDENT>return result_list[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return tuple(result_list)<EOL><DEDENT>

Returns annotations

        If len(args)>1, then returns a list of annotations.

        `f_get(X)` with *X* integer will return the annotation with name `annotation_X`.

        If the annotation contains only a single entry you can call `f_get()` without arguments.
        If you call `f_get()` and the annotation contains more than one element a ValueError is
        thrown.


for idx, arg in enumerate(args):<EOL><INDENT>valstr = self._translate_key(idx)<EOL>self.f_set_single(valstr, arg)<EOL><DEDENT>for key, arg in kwargs.items():<EOL><INDENT>self.f_set_single(key, arg)<EOL><DEDENT>

Sets annotations

        Items in args are added as `annotation` and `annotation_X` where
        'X' is the position in args for following arguments.


key = self._translate_key(key)<EOL>try:<EOL><INDENT>del self._dict[key]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise AttributeError('<STR_LIT>' % key)<EOL><DEDENT>


self._dict[name] = data<EOL>


resstr = '<STR_LIT>'<EOL>for key in sorted(self._dict.keys()):<EOL><INDENT>resstr += '<STR_LIT>' % (key, str(self._dict[key]))<EOL><DEDENT>return resstr[:-<NUM_LIT:2>]<EOL>

Returns all annotations lexicographically sorted as a concatenated string.


return self._annotations<EOL>

Annotation feature of a trajectory node.

        Store some short additional information about your nodes here.
        If you use the standard HDF5 storage service, they will be stored as hdf5 node
        attributes_.

        .. _attributes: http://pytables.github.io/usersguide/libref/declarative_classes.html#the-attributeset-class


def f_set_annotations(self, *args, **kwargs):

self._annotations.f_set(*args, **kwargs)<EOL>

Sets annotations

        Equivalent to calling `v_annotations.f_set(*args,**kwargs)`


return self._annotations.f_get(*args)<EOL>

Returns annotations

        Equivalent to `v_annotations.f_get(*args)`


return self._annotations.f_ann_to_str()<EOL>

Returns annotations as string

        Equivalent to `v_annotations.f_ann_to_str()`


log_levels = kwargs.pop('<STR_LIT>', None)<EOL>log_folder = kwargs.pop('<STR_LIT>', '<STR_LIT>')<EOL>logger_names = kwargs.pop('<STR_LIT>', '<STR_LIT>')<EOL>if log_levels is None:<EOL><INDENT>log_levels = kwargs.pop('<STR_LIT>', logging.INFO)<EOL><DEDENT>log_multiproc = kwargs.pop('<STR_LIT>', True)<EOL>if not isinstance(logger_names, (tuple, list)):<EOL><INDENT>logger_names = [logger_names]<EOL><DEDENT>if not isinstance(log_levels, (tuple, list)):<EOL><INDENT>log_levels = [log_levels]<EOL><DEDENT>if len(log_levels) == <NUM_LIT:1>:<EOL><INDENT>log_levels = [log_levels[<NUM_LIT:0>] for _ in logger_names]<EOL><DEDENT>dictionary = copy.deepcopy(LOGGING_DICT)<EOL>prefixes = ['<STR_LIT>']<EOL>if not log_multiproc:<EOL><INDENT>for key in list(dictionary.keys()):<EOL><INDENT>if key.startswith('<STR_LIT>'):<EOL><INDENT>del dictionary[key]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>prefixes.append('<STR_LIT>')<EOL><DEDENT>for prefix in prefixes:<EOL><INDENT>for handler_dict in dictionary[prefix + '<STR_LIT>'].values():<EOL><INDENT>if '<STR_LIT:filename>' in handler_dict:<EOL><INDENT>filename = os.path.join(log_folder, handler_dict['<STR_LIT:filename>'])<EOL>filename = os.path.normpath(filename)<EOL>handler_dict['<STR_LIT:filename>'] = filename<EOL><DEDENT><DEDENT>dictionary[prefix + '<STR_LIT>'] = {}<EOL>logger_dict = dictionary[prefix + '<STR_LIT>']<EOL>for idx, logger_name in enumerate(logger_names):<EOL><INDENT>logger_dict[logger_name] = {<EOL>'<STR_LIT>': log_levels[idx],<EOL>'<STR_LIT>': list(dictionary[prefix + '<STR_LIT>'].keys())<EOL>}<EOL><DEDENT><DEDENT>kwargs['<STR_LIT>'] = dictionary<EOL>

Helper function to turn the simple logging kwargs into a `log_config`.


return any([x in kwargs for x in ('<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>')])<EOL>


@functools.wraps(func)<EOL>def new_func(self, *args, **kwargs):<EOL><INDENT>if use_simple_logging(kwargs):<EOL><INDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>_change_logging_kwargs(kwargs)<EOL><DEDENT>return func(self, *args, **kwargs)<EOL><DEDENT>return new_func<EOL>

Decorator to allow a simple logging configuration.

    This encompasses giving a `log_folder`, `logger_names` as well as `log_levels`.


try:<EOL><INDENT>dirname = os.path.dirname(os.path.normpath(filename))<EOL>racedirs(dirname)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>sys.stderr.write('<STR_LIT>'<EOL>'<STR_LIT>' % (filename, repr(exc)))<EOL><DEDENT>

Tries to make directories for a given `filename`.

    Ignores any error but notifies via stderr.


string_list = []<EOL>for elem in ast.walk(ast.parse(args)):<EOL><INDENT>if isinstance(elem, ast.Str):<EOL><INDENT>string_list.append(elem.s)<EOL><DEDENT><DEDENT>return string_list<EOL>

Returns all valid python strings inside a given argument string.


if pypetconstants.LOG_ENV in filename:<EOL><INDENT>if env_name is None:<EOL><INDENT>env_name = trajectory.v_environment_name<EOL><DEDENT>filename = filename.replace(pypetconstants.LOG_ENV, env_name)<EOL><DEDENT>if pypetconstants.LOG_TRAJ in filename:<EOL><INDENT>if traj_name is None:<EOL><INDENT>traj_name = trajectory.v_name<EOL><DEDENT>filename = filename.replace(pypetconstants.LOG_TRAJ, traj_name)<EOL><DEDENT>if pypetconstants.LOG_RUN in filename:<EOL><INDENT>if run_name is None:<EOL><INDENT>run_name = trajectory.f_wildcard('<STR_LIT:$>')<EOL><DEDENT>filename = filename.replace(pypetconstants.LOG_RUN, run_name)<EOL><DEDENT>if pypetconstants.LOG_SET in filename:<EOL><INDENT>if set_name is None:<EOL><INDENT>set_name = trajectory.f_wildcard('<STR_LIT>')<EOL><DEDENT>filename = filename.replace(pypetconstants.LOG_SET, set_name)<EOL><DEDENT>if pypetconstants.LOG_PROC in filename:<EOL><INDENT>if process_name is None:<EOL><INDENT>process_name = multip.current_process().name + '<STR_LIT:->' + str(os.getpid())<EOL><DEDENT>filename = filename.replace(pypetconstants.LOG_PROC, process_name)<EOL><DEDENT>if pypetconstants.LOG_HOST in filename:<EOL><INDENT>if host_name is None:<EOL><INDENT>host_name = socket.getfqdn().replace('<STR_LIT:.>', '<STR_LIT:->')<EOL><DEDENT>filename = filename.replace(pypetconstants.LOG_HOST, host_name)<EOL><DEDENT>return filename<EOL>

Renames a given `filename` with valid wildcard placements.

    :const:`~pypet.pypetconstants.LOG_ENV` ($env) is replaces by the name of the
    trajectory`s environment.

    :const:`~pypet.pypetconstants.LOG_TRAJ` ($traj) is replaced by the name of the
    trajectory.

    :const:`~pypet.pypetconstants.LOG_RUN` ($run) is replaced by the name of the current
    run. If the trajectory is not set to a run 'run_ALL' is used.

    :const:`~pypet.pypetconstants.LOG_SET` ($set) is replaced by the name of the current
    run set. If the trajectory is not set to a run 'run_set_ALL' is used.

    :const:`~pypet.pypetconstants.LOG_PROC` ($proc) is replaced by the name fo the
    current process.

    :const:`~pypet.pypetconstant.LOG_HOST` ($host) is replaced by the name of the current host.

    :param filename:  A filename string
    :param traj:  A trajectory container, leave `None` if you provide all the parameters below
    :param env_name: Name of environemnt, leave `None` to get it from `traj`
    :param traj_name: Name of trajectory, leave `None` to get it from `traj`
    :param set_name: Name of run set, leave `None` to get it from `traj`
    :param run_name: Name of run, leave `None` to get it from `traj`
    :param process_name:

        The name of the desired process. If `None` the name of the current process is
        taken determined by the multiprocessing module.

    :param host_name:

        Name of host, leave `None` to determine it automatically with the platform module.

    :return: The new filename


state_dict = super(HasLogger, self).__getstate__()<EOL>if '<STR_LIT>' in state_dict:<EOL><INDENT>state_dict['<STR_LIT>'] = self._logger.name<EOL><DEDENT>return state_dict<EOL>

Called for pickling.

        Removes the logger to allow pickling and returns a copy of `__dict__`.


super(HasLogger, self).__setstate__(statedict)<EOL>if '<STR_LIT>' in statedict:<EOL><INDENT>self._set_logger(statedict['<STR_LIT>'])<EOL><DEDENT>

Called after loading a pickle dump.

        Restores `__dict__` from `statedict` and adds a new logger.


if name is None:<EOL><INDENT>cls = self.__class__<EOL>name = '<STR_LIT>' % (cls.__module__, cls.__name__)<EOL><DEDENT>self._logger = logging.getLogger(name)<EOL>

Adds a logger with a given `name`.

        If no name is given, name is constructed as
        `type(self).__name__`.


self.env_name = trajectory.v_environment_name<EOL>self.traj_name = trajectory.v_name<EOL>self.set_name =  trajectory.f_wildcard('<STR_LIT>')<EOL>self.run_name = trajectory.f_wildcard('<STR_LIT:$>')<EOL>

Extracts the wildcards and file replacements from the `trajectory`


state_dict = self.__dict__.copy()<EOL>if isinstance(state_dict['<STR_LIT>'], cp.RawConfigParser):<EOL><INDENT>state_dict['<STR_LIT>'] = True<EOL><DEDENT>return state_dict<EOL>

ConfigParsers are not guaranteed to
        be picklable so we need to remove these.


if self.report_progress:<EOL><INDENT>percentage, logger_name, log_level = self.report_progress<EOL>if logger_name == '<STR_LIT>':<EOL><INDENT>logger = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>logger = logging.getLogger(logger_name)<EOL><DEDENT>if n == -<NUM_LIT:1>:<EOL><INDENT>digits = int(math.log10(total_runs + <NUM_LIT:0.1>)) + <NUM_LIT:1><EOL>self._format_string = '<STR_LIT>' + '<STR_LIT>' % digits + '<STR_LIT>'<EOL><DEDENT>fmt_string = self._format_string % (n + <NUM_LIT:1>, total_runs) + '<STR_LIT:%s>'<EOL>reprint = log_level == <NUM_LIT:0><EOL>progressbar(n, total_runs, percentage_step=percentage,<EOL>logger=logger, log_level=log_level,<EOL>fmt_string=fmt_string, reprint=reprint)<EOL><DEDENT>


root = logging.getLogger()<EOL>root.addHandler(self._null_handler)<EOL>

Adds a NullHandler to the root logger.

        This is simply added to avoid warnings that no
        logger has been configured.


root = logging.getLogger()<EOL>root.removeHandler(self._null_handler)<EOL>

Removes the NullHandler from the root logger.


erase_dict = {'<STR_LIT>': False, '<STR_LIT:version>': <NUM_LIT:1>}<EOL>dictConfig(erase_dict)<EOL>

Removes all loggers and logging handlers.


@staticmethod<EOL><INDENT>def _check_and_replace_parser_args(parser, section, option, rename_func, make_dirs=True):<DEDENT>

args = parser.get(section, option, raw=True)<EOL>strings = get_strings(args)<EOL>replace = False<EOL>for string in strings:<EOL><INDENT>isfilename = any(x in string for x in FILENAME_INDICATORS)<EOL>if isfilename:<EOL><INDENT>newstring = rename_func(string)<EOL>if make_dirs:<EOL><INDENT>try_make_dirs(newstring)<EOL><DEDENT>raw_string = string.replace('<STR_LIT:\\>', '<STR_LIT>')<EOL>raw_newstring = newstring.replace('<STR_LIT:\\>', '<STR_LIT>')<EOL>args = args.replace(raw_string, raw_newstring)<EOL>replace = True<EOL><DEDENT><DEDENT>if replace:<EOL><INDENT>parser.set(section, option, args)<EOL><DEDENT>

Searches for parser settings that define filenames.

        If such settings are found, they are renamed according to the wildcard
        rules. Moreover, it is also tried to create the corresponding folders.

        :param parser:  A config parser
        :param section: A config section
        :param option: The section option
        :param rename_func: A function to rename found files
        :param make_dirs: If the directories of the file should be created.


memory_file = StringIO()<EOL>parser.write(memory_file)<EOL>memory_file.flush()<EOL>memory_file.seek(<NUM_LIT:0>)<EOL>return memory_file<EOL>

Turns a ConfigParser into a StringIO stream.


sections = parser.sections()<EOL>if not any(section.startswith('<STR_LIT>') for section in sections):<EOL><INDENT>return None<EOL><DEDENT>mp_parser = NoInterpolationParser()<EOL>for section in sections:<EOL><INDENT>if section.startswith('<STR_LIT>'):<EOL><INDENT>new_section = section.replace('<STR_LIT>', '<STR_LIT>')<EOL>mp_parser.add_section(new_section)<EOL>options = parser.options(section)<EOL>for option in options:<EOL><INDENT>val = parser.get(section, option, raw=True)<EOL>mp_parser.set(new_section, option, val)<EOL><DEDENT><DEDENT><DEDENT>return mp_parser<EOL>

Searches for multiprocessing options within a ConfigParser.

        If such options are found, they are copied (without the `'multiproc_'` prefix)
        into a new parser.


if not any(key.startswith('<STR_LIT>') for key in dictionary.keys()):<EOL><INDENT>return None<EOL><DEDENT>mp_dictionary = {}<EOL>for key in dictionary.keys():<EOL><INDENT>if key.startswith('<STR_LIT>'):<EOL><INDENT>new_key = key.replace('<STR_LIT>', '<STR_LIT>')<EOL>mp_dictionary[new_key] = dictionary[key]<EOL><DEDENT><DEDENT>mp_dictionary['<STR_LIT:version>'] = dictionary['<STR_LIT:version>']<EOL>if '<STR_LIT>' in dictionary:<EOL><INDENT>mp_dictionary['<STR_LIT>'] = dictionary['<STR_LIT>']<EOL><DEDENT>return mp_dictionary<EOL>

Searches for multiprocessing options in a given `dictionary`.

        If found they are copied (without the `'multiproc_'` prefix)
        into a new dictionary