def _tree_store_nodes_dfs(self, parent_traj_node, name, store_data, with_links, recursive,<EOL>max_depth, current_depth,<EOL>parent_hdf5_group):

if max_depth is None:<EOL><INDENT>max_depth = float('<STR_LIT>')<EOL><DEDENT>store_list = [(parent_traj_node, name, current_depth, parent_hdf5_group)]<EOL>while store_list:<EOL><INDENT>parent_traj_node, name, current_depth, parent_hdf5_group = store_list.pop()<EOL>if name in parent_traj_node._links:<EOL><INDENT>if with_links:<EOL><INDENT>self._tree_store_link(parent_traj_node, name, parent_hdf5_group)<EOL><DEDENT>continue<EOL><DEDENT>traj_node = parent_traj_node._children[name]<EOL>if not hasattr(parent_hdf5_group, name):<EOL><INDENT>newly_created = True<EOL>new_hdf5_group = self._hdf5file.create_group(where=parent_hdf5_group,<EOL>name=name, filters=self._all_get_filters())<EOL><DEDENT>else:<EOL><INDENT>newly_created = False<EOL>new_hdf5_group = getattr(parent_hdf5_group, name)<EOL><DEDENT>if traj_node.v_is_leaf:<EOL><INDENT>self._prm_store_parameter_or_result(traj_node, store_data=store_data,<EOL>_hdf5_group=new_hdf5_group,<EOL>_newly_created=newly_created)<EOL><DEDENT>else:<EOL><INDENT>self._grp_store_group(traj_node, store_data=store_data, with_links=with_links,<EOL>recursive=False, max_depth=max_depth,<EOL>_hdf5_group=new_hdf5_group,<EOL>_newly_created=newly_created)<EOL>if recursive and current_depth < max_depth:<EOL><INDENT>for child in traj_node._children.keys():<EOL><INDENT>store_list.append((traj_node, child, current_depth + <NUM_LIT:1>, new_hdf5_group))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Stores a node to hdf5 and if desired stores recursively everything below it.

        :param parent_traj_node: The parental node
        :param name: Name of node to be stored
        :param store_data: How to store data
        :param with_links: If links should be stored
        :param recursive: Whether to store recursively the subtree
        :param max_depth: Maximum recursion depth in tree
        :param current_depth: Current depth
        :param parent_hdf5_group: Parent hdf5 group


def _tree_store_link(self, node_in_traj, link, hdf5_group):

if hasattr(hdf5_group, link):<EOL><INDENT>return<EOL><DEDENT>linked_traj_node = node_in_traj._links[link]<EOL>linking_name = linked_traj_node.v_full_name.replace('<STR_LIT:.>','<STR_LIT:/>')<EOL>linking_name = '<STR_LIT:/>' + self._trajectory_name + '<STR_LIT:/>' + linking_name<EOL>try:<EOL><INDENT>to_link_hdf5_group = self._hdf5file.get_node(where=linking_name)<EOL><DEDENT>except pt.NoSuchNodeError:<EOL><INDENT>self._logger.debug('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (link,<EOL>node_in_traj.v_full_name,<EOL>linked_traj_node.v_full_name))<EOL>root = node_in_traj._nn_interface._root_instance<EOL>self._tree_store_sub_branch(root, linked_traj_node.v_full_name,<EOL>store_data=pypetconstants.STORE_DATA_SKIPPING,<EOL>with_links=False, recursive=False,<EOL>hdf5_group=self._trajectory_group)<EOL>to_link_hdf5_group = self._hdf5file.get_node(where=linking_name)<EOL><DEDENT>self._hdf5file.create_soft_link(where=hdf5_group,<EOL>name=link,<EOL>target=to_link_hdf5_group)<EOL>

Creates a soft link.

        :param node_in_traj: parental node
        :param store_data: how to store data
        :param link: name of link
        :param hdf5_group: current parental hdf5 group


if store_data != pypetconstants.STORE_NOTHING:<EOL><INDENT>self._logger.debug('<STR_LIT>' % traj.v_crun)<EOL>if max_depth is None:<EOL><INDENT>max_depth = float('<STR_LIT>')<EOL><DEDENT>for name_pair in traj._new_nodes:<EOL><INDENT>_, name = name_pair<EOL>parent_group, child_node = traj._new_nodes[name_pair]<EOL>if not child_node._stored:<EOL><INDENT>self._tree_store_sub_branch(parent_group, name,<EOL>store_data=store_data,<EOL>with_links=True,<EOL>recursive=recursive,<EOL>max_depth=max_depth - child_node.v_depth,<EOL>hdf5_group=None)<EOL><DEDENT><DEDENT>for name_pair in traj._new_links:<EOL><INDENT>_, link = name_pair<EOL>parent_group, _ = traj._new_links[name_pair]<EOL>self._tree_store_sub_branch(parent_group, link,<EOL>store_data=store_data,<EOL>with_links=True,<EOL>recursive=recursive,<EOL>max_depth=max_depth - parent_group.v_depth - <NUM_LIT:1>,<EOL>hdf5_group=None)<EOL><DEDENT><DEDENT>

Stores a single run instance to disk (only meta data)


runsummary = '<STR_LIT>'<EOL>paramlist = sorted(paramlist, key=lambda name: name.v_name + name.v_location)<EOL>for idx, expparam in enumerate(paramlist):<EOL><INDENT>if idx > <NUM_LIT:0>:<EOL><INDENT>runsummary += '<STR_LIT:U+002CU+0020>'<EOL><DEDENT>valstr = expparam.f_val_to_str()<EOL>if len(valstr) >= pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH:<EOL><INDENT>valstr = valstr[<NUM_LIT:0>:pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH - <NUM_LIT:3>]<EOL>valstr += '<STR_LIT>'<EOL><DEDENT>if expparam.v_name in runsummary:<EOL><INDENT>param_name = expparam.v_full_name<EOL><DEDENT>else:<EOL><INDENT>param_name = expparam.v_name<EOL><DEDENT>runsummary = runsummary + param_name + '<STR_LIT>' + valstr<EOL><DEDENT>return runsummary<EOL>

Summarizes the parameter settings.

        :param run_name: Name of the single run

        :param paramlist: List of explored parameters

        :param add_table: Whether to add the overview table

        :param create_run_group:

            If a group with the particular name should be created if it does not exist.
            Might be necessary when trajectories are merged.


def _all_store_param_or_result_table_entry(self, instance, table, flags,<EOL>additional_info=None):

<EOL>location = instance.v_location<EOL>name = instance.v_name<EOL>fullname = instance.v_full_name<EOL>if (flags == (HDF5StorageService.ADD_ROW,) and table.nrows < <NUM_LIT:2><EOL>and '<STR_LIT:location>' in table.colnames):<EOL><INDENT>flags = (HDF5StorageService.ADD_ROW, HDF5StorageService.MODIFY_ROW)<EOL><DEDENT>if flags == (HDF5StorageService.ADD_ROW,):<EOL><INDENT>condvars = None<EOL>condition = None<EOL><DEDENT>else:<EOL><INDENT>condvars = {'<STR_LIT>': table.cols.name, '<STR_LIT>': table.cols.location,<EOL>'<STR_LIT:name>': name, '<STR_LIT:location>': location}<EOL>condition = """<STR_LIT>"""<EOL><DEDENT>if HDF5StorageService.REMOVE_ROW in flags:<EOL><INDENT>insert_dict = {}<EOL><DEDENT>else:<EOL><INDENT>colnames = set(table.colnames)<EOL>insert_dict = self._all_extract_insert_dict(instance, colnames, additional_info)<EOL><DEDENT>self._all_add_or_modify_row(fullname, insert_dict, table, condition=condition,<EOL>condvars=condvars, flags=flags)<EOL>

Stores a single row into an overview table

        :param instance: A parameter or result instance

        :param table: Table where row will be inserted

        :param flags:

            Flags how to insert into the table. Potential Flags are
            `ADD_ROW`, `REMOVE_ROW`, `MODIFY_ROW`

        :param additional_info:

            Dictionary containing information that cannot be extracted from
            `instance`, but needs to be inserted, too.


def _all_get_or_create_table(self, where, tablename, description, expectedrows=None):

where_node = self._hdf5file.get_node(where)<EOL>if not tablename in where_node:<EOL><INDENT>if not expectedrows is None:<EOL><INDENT>table = self._hdf5file.create_table(where=where_node, name=tablename,<EOL>description=description, title=tablename,<EOL>expectedrows=expectedrows,<EOL>filters=self._all_get_filters())<EOL><DEDENT>else:<EOL><INDENT>table = self._hdf5file.create_table(where=where_node, name=tablename,<EOL>description=description, title=tablename,<EOL>filters=self._all_get_filters())<EOL><DEDENT><DEDENT>else:<EOL><INDENT>table = where_node._f_get_child(tablename)<EOL><DEDENT>return table<EOL>

Creates a new table, or if the table already exists, returns it.


path_name = name.replace('<STR_LIT:.>', '<STR_LIT:/>')<EOL>where = '<STR_LIT>' % (self._trajectory_name, path_name)<EOL>return self._hdf5file.get_node(where=where)<EOL>

Returns an HDF5 node by the path specified in `name`


try:<EOL><INDENT>return getattr(ptitem._v_attrs, name)<EOL><DEDENT>except AttributeError:<EOL><INDENT>return None<EOL><DEDENT>

Gets an attribute `name` from `ptitem`, returns None if attribute does not exist.


@staticmethod<EOL><INDENT>def _all_set_attr(ptitem, name, value):<DEDENT>

return setattr(ptitem._v_attrs, name, value)<EOL>


@staticmethod<EOL><INDENT>def _all_set_attributes_to_recall_natives(data, ptitem, prefix):<DEDENT>

<EOL>if type(data) is tuple:<EOL><INDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,<EOL>HDF5StorageService.COLL_TUPLE)<EOL><DEDENT>elif type(data) is list:<EOL><INDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,<EOL>HDF5StorageService.COLL_LIST)<EOL><DEDENT>elif type(data) is np.ndarray:<EOL><INDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,<EOL>HDF5StorageService.COLL_NDARRAY)<EOL><DEDENT>elif type(data) is np.matrix:<EOL><INDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,<EOL>HDF5StorageService.COLL_MATRIX)<EOL><DEDENT>elif type(data) in pypetconstants.PARAMETER_SUPPORTED_DATA:<EOL><INDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,<EOL>HDF5StorageService.COLL_SCALAR)<EOL>strtype = type(data).__name__<EOL>if not strtype in pypetconstants.PARAMETERTYPEDICT:<EOL><INDENT>raise TypeError('<STR_LIT>' %<EOL>(str(data), repr(type(data))))<EOL><DEDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.SCALAR_TYPE, strtype)<EOL><DEDENT>elif type(data) is dict:<EOL><INDENT>if len(data) > <NUM_LIT:0>:<EOL><INDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,<EOL>HDF5StorageService.COLL_DICT)<EOL><DEDENT>else:<EOL><INDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,<EOL>HDF5StorageService.COLL_EMPTY_DICT)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TypeError('<STR_LIT>' %<EOL>(str(data), repr(type(data))))<EOL><DEDENT>if type(data) in (list, tuple):<EOL><INDENT>if len(data) > <NUM_LIT:0>:<EOL><INDENT>strtype = type(data[<NUM_LIT:0>]).__name__<EOL>if not strtype in pypetconstants.PARAMETERTYPEDICT:<EOL><INDENT>raise TypeError('<STR_LIT>'<EOL>'<STR_LIT>' % (str(data), strtype))<EOL><DEDENT>HDF5StorageService._all_set_attr(ptitem, prefix +<EOL>HDF5StorageService.SCALAR_TYPE, strtype)<EOL><DEDENT><DEDENT>elif (type(data) in (np.ndarray, np.matrix) and<EOL>np.issubdtype(data.dtype, str)):<EOL><INDENT>HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.SCALAR_TYPE,<EOL>str.__name__)<EOL><DEDENT>

Stores original data type to hdf5 node attributes for preserving the data type.

        :param data:

            Data to be stored

        :param ptitem:

            HDF5 node to store data types as attributes. Can also be just a PTItemMock.

        :param prefix:

            String prefix to label and name data in HDF5 attributes


def _all_recall_native_type(self, data, ptitem, prefix):

typestr = self._all_get_from_attrs(ptitem, prefix + HDF5StorageService.SCALAR_TYPE)<EOL>colltype = self._all_get_from_attrs(ptitem, prefix + HDF5StorageService.COLL_TYPE)<EOL>type_changed = False<EOL>if colltype == HDF5StorageService.COLL_SCALAR:<EOL><INDENT>if isinstance(data, np.ndarray):<EOL><INDENT>data = np.array([data])[<NUM_LIT:0>]<EOL>type_changed = True<EOL><DEDENT>if not typestr is None:<EOL><INDENT>if typestr != type(data).__name__:<EOL><INDENT>if typestr == str.__name__:<EOL><INDENT>data = data.decode(self._encoding)<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>data = pypetconstants.PARAMETERTYPEDICT[typestr](data)<EOL><DEDENT>except KeyError:<EOL><INDENT>data = pypetconstants.COMPATPARAMETERTYPEDICT[typestr](data)<EOL><DEDENT><DEDENT>type_changed = True<EOL><DEDENT><DEDENT><DEDENT>elif (colltype == HDF5StorageService.COLL_TUPLE or<EOL>colltype == HDF5StorageService.COLL_LIST):<EOL><INDENT>if type(data) is not list and type is not tuple:<EOL><INDENT>type_changed = True<EOL>data = list(data)<EOL><DEDENT>if len(data) > <NUM_LIT:0>:<EOL><INDENT>first_item = data[<NUM_LIT:0>]<EOL>if not typestr == type(first_item).__name__:<EOL><INDENT>if not isinstance(data, list):<EOL><INDENT>data = list(data)<EOL><DEDENT>for idx, item in enumerate(data):<EOL><INDENT>if typestr == str.__name__:<EOL><INDENT>data[idx] = data[idx].decode(self._encoding)<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>data[idx] = pypetconstants.PARAMETERTYPEDICT[typestr](item)<EOL><DEDENT>except KeyError:<EOL><INDENT>data[idx] = pypetconstants.COMPATPARAMETERTYPEDICT[typestr](item)<EOL><DEDENT><DEDENT>type_changed = True<EOL><DEDENT><DEDENT><DEDENT>if colltype == HDF5StorageService.COLL_TUPLE:<EOL><INDENT>if type(data) is not tuple:<EOL><INDENT>data = tuple(data)<EOL>type_changed = True<EOL><DEDENT><DEDENT><DEDENT>elif colltype == HDF5StorageService.COLL_EMPTY_DICT:<EOL><INDENT>data = {}<EOL>type_changed = True<EOL><DEDENT>elif isinstance(data, np.ndarray):<EOL><INDENT>if typestr == str.__name__:<EOL><INDENT>data = np.core.defchararray.decode(data, self._encoding)<EOL>type_changed = True<EOL><DEDENT>if colltype == HDF5StorageService.COLL_MATRIX:<EOL><INDENT>data = np.matrix(data)<EOL>type_changed = True<EOL><DEDENT><DEDENT>return data, type_changed<EOL>

Checks if loaded data has the type it was stored in. If not converts it.

        :param data: Data item to be checked and converted
        :param ptitem: HDf5 Node or Leaf from where data was loaded
        :param prefix: Prefix for recalling the data type from the hdf5 node attributes

        :return:

            Tuple, first item is the (converted) `data` item, second boolean whether
            item was converted or not.


def _all_add_or_modify_row(self, item_name, insert_dict, table, index=None, condition=None,<EOL>condvars=None,<EOL>flags=(ADD_ROW, MODIFY_ROW,)):

if len(flags) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>if index is not None and condition is not None:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>elif condition is not None:<EOL><INDENT>row_iterator = table.where(condition, condvars=condvars)<EOL><DEDENT>elif index is not None:<EOL><INDENT>row_iterator = table.iterrows(index, index + <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>row_iterator = None<EOL><DEDENT>try:<EOL><INDENT>row = next(row_iterator)<EOL><DEDENT>except TypeError:<EOL><INDENT>row = None<EOL><DEDENT>except StopIteration:<EOL><INDENT>row = None<EOL><DEDENT>if ((HDF5StorageService.MODIFY_ROW in flags or HDF5StorageService.ADD_ROW in flags) and<EOL>HDF5StorageService.REMOVE_ROW in flags):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if row is None and HDF5StorageService.ADD_ROW in flags:<EOL><INDENT>row = table.row<EOL>self._all_insert_into_row(row, insert_dict)<EOL>row.append()<EOL><DEDENT>elif row is not None and HDF5StorageService.MODIFY_ROW in flags:<EOL><INDENT>self._all_insert_into_row(row, insert_dict)<EOL>row.update()<EOL><DEDENT>elif HDF5StorageService.REMOVE_ROW in flags:<EOL><INDENT>if row is not None:<EOL><INDENT>row_number = row.nrow<EOL>try:<EOL><INDENT>table.remove_rows(start=row_number, stop=row_number+<NUM_LIT:1>)<EOL><DEDENT>except NotImplementedError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>self._all_kill_iterator(row_iterator)<EOL>table.flush()<EOL>if HDF5StorageService.REMOVE_ROW not in flags and row is None:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>' % (item_name, table._v_name))<EOL><DEDENT>

Adds or changes a row in a pytable.

        :param item_name: Name of item, the row is about, only important for throwing errors.

        :param insert_dict:

            Dictionary of data that is about to be inserted into the pytables row.

        :param table:

            The table to insert or modify a row in

        :param index:

            Index of row to be modified. Instead of an index a search condition can be
            used as well, see below.

        :param condition:

            Condition to search for in the table

        :param condvars:

            Variables for the search condition

        :param flags:

            Flags whether to add, modify, or remove a row in the table


def _all_insert_into_row(self, row, insert_dict):

for key, val in insert_dict.items():<EOL><INDENT>try:<EOL><INDENT>row[key] = val<EOL><DEDENT>except KeyError as ke:<EOL><INDENT>self._logger.warning('<STR_LIT>' % key + repr(ke))<EOL><DEDENT><DEDENT>

Copies data from `insert_dict` into a pytables `row`.


def _all_extract_insert_dict(self, item, colnames, additional_info=None):

insert_dict = {}<EOL>if '<STR_LIT>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT>'] = len(item)<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>comment = self._all_cut_string(item.v_comment.encode('<STR_LIT:utf-8>'),<EOL>pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH,<EOL>self._logger)<EOL>insert_dict['<STR_LIT>'] = comment<EOL><DEDENT>if '<STR_LIT:location>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT:location>'] = item.v_location.encode('<STR_LIT:utf-8>')<EOL><DEDENT>if '<STR_LIT:name>' in colnames:<EOL><INDENT>name = item._name if (not item.v_is_root or not item.v_is_run) else item._crun<EOL>insert_dict['<STR_LIT:name>'] = name.encode('<STR_LIT:utf-8>')<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT>'] = item.f_get_class_name().encode('<STR_LIT:utf-8>')<EOL><DEDENT>if '<STR_LIT:value>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT:value>'] = self._all_cut_string(<EOL>item.f_val_to_str().encode('<STR_LIT:utf-8>'),<EOL>pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH,<EOL>self._logger)<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT>'] = additional_info['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT>'] = item.v_idx<EOL><DEDENT>if '<STR_LIT:time>' in colnames:<EOL><INDENT>time_ = item._time<EOL>insert_dict['<STR_LIT:time>'] = time_.encode('<STR_LIT:utf-8>')<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>timestamp = item._timestamp<EOL>insert_dict['<STR_LIT>'] = timestamp<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>third_length = pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH // <NUM_LIT:3> + <NUM_LIT:10><EOL>item_range = itools.islice(item.f_get_range(copy=False), <NUM_LIT:0>, third_length)<EOL>range_string = '<STR_LIT:U+002CU+0020>'.join([repr(x) for x in item_range])<EOL>insert_dict['<STR_LIT>'] = self._all_cut_string(<EOL>range_string.encode('<STR_LIT:utf-8>'),<EOL>pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH,<EOL>self._logger)<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>third_length = pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH // <NUM_LIT:3> + <NUM_LIT:10><EOL>item_range = itools.islice(item.f_get_range(copy=False), <NUM_LIT:0>, third_length)<EOL>range_string = '<STR_LIT:U+002CU+0020>'.join([repr(x) for x in item_range])<EOL>insert_dict['<STR_LIT>'] = self._all_cut_string(<EOL>range_string.encode('<STR_LIT:utf-8>'),<EOL>pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH,<EOL>self._logger)<EOL><DEDENT>if '<STR_LIT:version>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT:version>'] = item.v_version.encode('<STR_LIT:utf-8>')<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT>'] = item.v_python.encode('<STR_LIT:utf-8>')<EOL><DEDENT>if '<STR_LIT>' in colnames:<EOL><INDENT>insert_dict['<STR_LIT>'] = item._finish_timestamp_run<EOL><DEDENT>return insert_dict<EOL>

Extracts information from a given item to be stored into a pytable row.

        Items can be a variety of things here, trajectories, single runs, group node,
        parameters, results.

        :param item: Item from which data should be extracted

        :param colnames: Names of the columns in the pytable

        :param additional_info: (dict)

            Additional information that should be stored into the pytable row that cannot be
            read out from `item`.

        :return: Dictionary containing the data to be inserted into a row


@staticmethod<EOL><INDENT>def _all_cut_string(string, max_length, logger):<DEDENT>

if len(string) > max_length:<EOL><INDENT>logger.debug('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(string, max_length))<EOL>string = string[<NUM_LIT:0>:max_length - <NUM_LIT:3>] + '<STR_LIT>'.encode('<STR_LIT:utf-8>')<EOL><DEDENT>return string<EOL>

Cuts string data to the maximum length allowed in a pytables column
        if string is too long.

        :param string: String to be cut
        :param max_length: Maximum allowed string length
        :param logger: Logger where messages about truncating should be written

        :return: String, cut if too long


def _all_create_or_get_group(self, name, parent_hdf5_group=None):

if not name in parent_hdf5_group:<EOL><INDENT>new_hdf5_group = self._hdf5file.create_group(where=parent_hdf5_group,<EOL>name=name,<EOL>title=name,<EOL>filters=self._all_get_filters())<EOL>return new_hdf5_group, True<EOL><DEDENT>else:<EOL><INDENT>new_hdf5_group = parent_hdf5_group._f_get_child(name)<EOL>return new_hdf5_group, False<EOL><DEDENT>


def _all_create_or_get_groups(self, key, start_hdf5_group=None):

if start_hdf5_group is None:<EOL><INDENT>newhdf5_group = self._trajectory_group<EOL><DEDENT>else:<EOL><INDENT>newhdf5_group = start_hdf5_group<EOL><DEDENT>created = False<EOL>if key == '<STR_LIT>':<EOL><INDENT>return newhdf5_group, created<EOL><DEDENT>split_key = key.split('<STR_LIT:.>')<EOL>for name in split_key:<EOL><INDENT>newhdf5_group, created = self._all_create_or_get_group(name, newhdf5_group)<EOL><DEDENT>return newhdf5_group, created<EOL>

Creates new or follows existing group nodes along a given colon separated `key`.

        :param key:

            Colon separated path along hdf5 file, e.g. `parameters.mobiles.cars`.

        :param start_hdf5_group:

            HDF5 group from where to start, leave `None` for the trajectory group.

        :return:

            Final group node, e.g. group node with name `cars`.


def _ann_store_annotations(self, item_with_annotations, node, overwrite=False):

<EOL>if overwrite is True or overwrite == '<STR_LIT>':<EOL><INDENT>annotated = self._all_get_from_attrs(node, HDF5StorageService.ANNOTATED)<EOL>if annotated:<EOL><INDENT>current_attrs = node._v_attrs<EOL>for attr_name in current_attrs._v_attrnames:<EOL><INDENT>if attr_name.startswith(HDF5StorageService.ANNOTATION_PREFIX):<EOL><INDENT>delattr(current_attrs, attr_name)<EOL><DEDENT><DEDENT>delattr(current_attrs, HDF5StorageService.ANNOTATED)<EOL>self._hdf5file.flush()<EOL><DEDENT><DEDENT>if not item_with_annotations.v_annotations.f_is_empty():<EOL><INDENT>anno_dict = item_with_annotations.v_annotations._dict<EOL>current_attrs = node._v_attrs<EOL>changed = False<EOL>for field_name in anno_dict:<EOL><INDENT>val = anno_dict[field_name]<EOL>field_name_with_prefix = HDF5StorageService.ANNOTATION_PREFIX + field_name<EOL>if field_name_with_prefix not in current_attrs:<EOL><INDENT>setattr(current_attrs, field_name_with_prefix, val)<EOL>changed = True<EOL><DEDENT><DEDENT>if changed:<EOL><INDENT>setattr(current_attrs, HDF5StorageService.ANNOTATED, True)<EOL>self._hdf5file.flush()<EOL><DEDENT><DEDENT>


def _ann_load_annotations(self, item_with_annotations, node):

annotated = self._all_get_from_attrs(node, HDF5StorageService.ANNOTATED)<EOL>if annotated:<EOL><INDENT>annotations = item_with_annotations.v_annotations<EOL>if not annotations.f_is_empty():<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>current_attrs = node._v_attrs<EOL>for attr_name in current_attrs._v_attrnames:<EOL><INDENT>if attr_name.startswith(HDF5StorageService.ANNOTATION_PREFIX):<EOL><INDENT>key = attr_name<EOL>key = key.replace(HDF5StorageService.ANNOTATION_PREFIX, '<STR_LIT>')<EOL>data = getattr(current_attrs, attr_name)<EOL>setattr(annotations, key, data)<EOL><DEDENT><DEDENT><DEDENT>


def _grp_store_group(self, traj_group, store_data=pypetconstants.STORE_DATA,<EOL>with_links=True, recursive=False, max_depth=None,<EOL>_hdf5_group=None, _newly_created=False):

if store_data == pypetconstants.STORE_NOTHING:<EOL><INDENT>return<EOL><DEDENT>elif store_data == pypetconstants.STORE_DATA_SKIPPING and traj_group._stored:<EOL><INDENT>self._logger.debug('<STR_LIT>' %<EOL>traj_group.v_full_name)<EOL><DEDENT>elif not recursive:<EOL><INDENT>if _hdf5_group is None:<EOL><INDENT>_hdf5_group, _newly_created = self._all_create_or_get_groups(traj_group.v_full_name)<EOL><DEDENT>overwrite = store_data == pypetconstants.OVERWRITE_DATA<EOL>if (traj_group.v_comment != '<STR_LIT>' and<EOL>(HDF5StorageService.COMMENT not in _hdf5_group._v_attrs or overwrite)):<EOL><INDENT>setattr(_hdf5_group._v_attrs, HDF5StorageService.COMMENT, traj_group.v_comment)<EOL><DEDENT>if ((_newly_created or overwrite) and<EOL>type(traj_group) not in (nn.NNGroupNode, nn.ConfigGroup, nn.ParameterGroup,<EOL>nn.DerivedParameterGroup, nn.ResultGroup)):<EOL><INDENT>setattr(_hdf5_group._v_attrs, HDF5StorageService.CLASS_NAME,<EOL>traj_group.f_get_class_name())<EOL><DEDENT>self._ann_store_annotations(traj_group, _hdf5_group, overwrite=overwrite)<EOL>self._hdf5file.flush()<EOL>traj_group._stored = True<EOL>self._node_processing_timer.signal_update()<EOL><DEDENT>if recursive:<EOL><INDENT>parent_traj_group = traj_group.f_get_parent()<EOL>parent_hdf5_group = self._all_create_or_get_groups(parent_traj_group.v_full_name)[<NUM_LIT:0>]<EOL>self._tree_store_nodes_dfs(parent_traj_group, traj_group.v_name, store_data=store_data,<EOL>with_links=with_links, recursive=recursive,<EOL>max_depth=max_depth, current_depth=<NUM_LIT:0>,<EOL>parent_hdf5_group=parent_hdf5_group)<EOL><DEDENT>

Stores a group node.

        For group nodes only annotations and comments need to be stored.


def _grp_load_group(self, traj_group, load_data=pypetconstants.LOAD_DATA, with_links=True,<EOL>recursive=False, max_depth=None,<EOL>_traj=None, _as_new=False, _hdf5_group=None):

if _hdf5_group is None:<EOL><INDENT>_hdf5_group = self._all_get_node_by_name(traj_group.v_full_name)<EOL>_traj = traj_group.v_root<EOL><DEDENT>if recursive:<EOL><INDENT>parent_traj_node = traj_group.f_get_parent()<EOL>self._tree_load_nodes_dfs(parent_traj_node, load_data=load_data, with_links=with_links,<EOL>recursive=recursive, max_depth=max_depth,<EOL>current_depth=<NUM_LIT:0>,<EOL>trajectory=_traj, as_new=_as_new,<EOL>hdf5_group=_hdf5_group)<EOL><DEDENT>else:<EOL><INDENT>if load_data == pypetconstants.LOAD_NOTHING:<EOL><INDENT>return<EOL><DEDENT>elif load_data == pypetconstants.OVERWRITE_DATA:<EOL><INDENT>traj_group.v_annotations.f_empty()<EOL>traj_group.v_comment = '<STR_LIT>'<EOL><DEDENT>self._all_load_skeleton(traj_group, _hdf5_group)<EOL>traj_group._stored = not _as_new<EOL>self._node_processing_timer.signal_update()<EOL><DEDENT>

Loads a group node and potentially everything recursively below


def _all_load_skeleton(self, traj_node, hdf5_group):

if traj_node.v_annotations.f_is_empty():<EOL><INDENT>self._ann_load_annotations(traj_node, hdf5_group)<EOL><DEDENT>if traj_node.v_comment == '<STR_LIT>':<EOL><INDENT>comment = self._all_get_from_attrs(hdf5_group, HDF5StorageService.COMMENT)<EOL>if comment is None:<EOL><INDENT>comment = '<STR_LIT>'<EOL><DEDENT>traj_node.v_comment = comment<EOL><DEDENT>


for key, data in data_dict.items():<EOL><INDENT>if not key in flags_dict:<EOL><INDENT>dtype = type(data)<EOL>if (dtype is np.ndarray or dtype is dict) and len(data) == <NUM_LIT:0>:<EOL><INDENT>flags_dict[key] = HDF5StorageService.ARRAY<EOL>continue<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>flags_dict[key] = HDF5StorageService.TYPE_FLAG_MAPPING[dtype]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise pex.NoSuchServiceError('<STR_LIT>'<EOL>'<STR_LIT>' % (key, str(dtype)))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Extracts storage flags for data in `data_dict`
        if they were not specified in `flags_dict`.

        See :const:`~pypet.storageservice.HDF5StorageService.TYPE_FLAG_MAPPING`
        for how to store different types of data per default.


if instance.v_comment == '<STR_LIT>':<EOL><INDENT>return False<EOL><DEDENT>where = instance.v_branch<EOL>definitely_store_comment = True<EOL>bytes_comment = instance.v_comment.encode('<STR_LIT:utf-8>')<EOL>hexdigest = hashlib.sha1(bytes_comment).hexdigest()<EOL>hexdigest = hexdigest.encode('<STR_LIT:utf-8>')<EOL>table_name = where + '<STR_LIT>'<EOL>if table_name in self._overview_group:<EOL><INDENT>table = getattr(self._overview_group, table_name)<EOL><DEDENT>else:<EOL><INDENT>return definitely_store_comment<EOL><DEDENT>try:<EOL><INDENT>condvars = {'<STR_LIT>': table.cols.hexdigest,<EOL>'<STR_LIT>': hexdigest}<EOL>condition = """<STR_LIT>"""<EOL>row_iterator = table.where(condition, condvars=condvars)<EOL>row = None<EOL>try:<EOL><INDENT>row = next(row_iterator)<EOL><DEDENT>except StopIteration:<EOL><INDENT>pass<EOL><DEDENT>if row is None:<EOL><INDENT>self._all_store_param_or_result_table_entry(instance, table,<EOL>flags=(<EOL>HDF5StorageService.ADD_ROW,),<EOL>additional_info={<EOL>'<STR_LIT>': hexdigest})<EOL>definitely_store_comment = True<EOL><DEDENT>else:<EOL><INDENT>definitely_store_comment = False<EOL>self._all_kill_iterator(row_iterator)<EOL><DEDENT><DEDENT>except pt.NoSuchNodeError:<EOL><INDENT>definitely_store_comment = True<EOL><DEDENT>return definitely_store_comment<EOL>

Adds data to the summary tables and returns if `instance`s comment has to be stored.

        Also moves comments upwards in the hierarchy if purge_duplicate_comments is true
        and a lower index run has completed. Only necessary for *multiprocessing*.

        :return: Tuple

            * String specifying the subtree

            * Boolean whether to store the comment to `instance`s hdf5 node


def _prm_add_meta_info(self, instance, group, overwrite=False):

if overwrite:<EOL><INDENT>flags = ()<EOL><DEDENT>else:<EOL><INDENT>flags = (HDF5StorageService.ADD_ROW,)<EOL><DEDENT>definitely_store_comment = True<EOL>try:<EOL><INDENT>definitely_store_comment = self._prm_meta_add_summary(instance)<EOL>try:<EOL><INDENT>table_name = instance.v_branch + '<STR_LIT>'<EOL>table = getattr(self._overview_group, table_name)<EOL>if len(table) < pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH:<EOL><INDENT>self._all_store_param_or_result_table_entry(instance, table,<EOL>flags=flags)<EOL><DEDENT><DEDENT>except pt.NoSuchNodeError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>self._logger.error('<STR_LIT>' % repr(exc))<EOL><DEDENT>if ((not self._purge_duplicate_comments or definitely_store_comment) and<EOL>instance.v_comment != '<STR_LIT>'):<EOL><INDENT>setattr(group._v_attrs, HDF5StorageService.COMMENT, instance.v_comment)<EOL><DEDENT>setattr(group._v_attrs, HDF5StorageService.CLASS_NAME, instance.f_get_class_name())<EOL>setattr(group._v_attrs, HDF5StorageService.LEAF, True)<EOL>if instance.v_is_parameter and instance.v_explored:<EOL><INDENT>try:<EOL><INDENT>tablename = '<STR_LIT>'<EOL>table = getattr(self._overview_group, tablename)<EOL>if len(table) < pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH:<EOL><INDENT>self._all_store_param_or_result_table_entry(instance, table,<EOL>flags=flags)<EOL><DEDENT><DEDENT>except pt.NoSuchNodeError:<EOL><INDENT>pass<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self._logger.error('<STR_LIT>'<EOL>'<STR_LIT>' % repr(exc))<EOL><DEDENT><DEDENT>

Adds information to overview tables and meta information to
        the `instance`s hdf5 `group`.

        :param instance: Instance to store meta info about
        :param group: HDF5 group of instance
        :param overwrite: If data should be explicitly overwritten


def _prm_store_from_dict(self, fullname, store_dict, hdf5_group, store_flags, kwargs):

for key, data_to_store in store_dict.items():<EOL><INDENT>original_hdf5_group = None<EOL>flag = store_flags[key]<EOL>if '<STR_LIT:.>' in key:<EOL><INDENT>original_hdf5_group = hdf5_group<EOL>split_key = key.split('<STR_LIT:.>')<EOL>key = split_key.pop()<EOL>for inner_key in split_key:<EOL><INDENT>hdf5_group, newly_created =  self._all_create_or_get_group(inner_key,<EOL>hdf5_group)<EOL>if newly_created:<EOL><INDENT>setattr(hdf5_group._v_attrs, HDF5StorageService.STORAGE_TYPE,<EOL>HDF5StorageService.NESTED_GROUP)<EOL><DEDENT>else:<EOL><INDENT>store_type = self._all_get_from_attrs(hdf5_group, HDF5StorageService.STORAGE_TYPE)<EOL>if store_type != HDF5StorageService.NESTED_GROUP:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' % (hdf5_group._v_name, store_type))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if key in hdf5_group:<EOL><INDENT>self._logger.debug(<EOL>'<STR_LIT>' %<EOL>(key, fullname))<EOL>continue<EOL><DEDENT>if flag == HDF5StorageService.TABLE:<EOL><INDENT>self._prm_write_into_pytable(key, data_to_store, hdf5_group, fullname,<EOL>**kwargs)<EOL><DEDENT>elif flag == HDF5StorageService.DICT:<EOL><INDENT>self._prm_write_dict_as_table(key, data_to_store, hdf5_group, fullname,<EOL>**kwargs)<EOL><DEDENT>elif flag == HDF5StorageService.ARRAY:<EOL><INDENT>self._prm_write_into_array(key, data_to_store, hdf5_group, fullname,<EOL>**kwargs)<EOL><DEDENT>elif flag in (HDF5StorageService.CARRAY,<EOL>HDF5StorageService.EARRAY,<EOL>HDF5StorageService.VLARRAY):<EOL><INDENT>self._prm_write_into_other_array(key, data_to_store,<EOL>hdf5_group, fullname,<EOL>flag=flag, **kwargs)<EOL><DEDENT>elif flag in (HDF5StorageService.SERIES,<EOL>HDF5StorageService.FRAME,<EOL>):<EOL><INDENT>self._prm_write_pandas_data(key, data_to_store, hdf5_group, fullname,<EOL>flag, **kwargs)<EOL><DEDENT>elif flag == HDF5StorageService.SHARED_DATA:<EOL><INDENT>pass  <EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>if original_hdf5_group is not None:<EOL><INDENT>hdf5_group = original_hdf5_group<EOL><DEDENT><DEDENT>


if store_data == pypetconstants.STORE_NOTHING:<EOL><INDENT>return<EOL><DEDENT>elif store_data == pypetconstants.STORE_DATA_SKIPPING and instance._stored:<EOL><INDENT>self._logger.debug('<STR_LIT>' %<EOL>instance.v_full_name)<EOL>return<EOL><DEDENT>elif store_data == pypetconstants.OVERWRITE_DATA:<EOL><INDENT>if not overwrite:<EOL><INDENT>overwrite = True<EOL><DEDENT><DEDENT>fullname = instance.v_full_name<EOL>self._logger.debug('<STR_LIT>' % fullname)<EOL>if _hdf5_group is None:<EOL><INDENT>_hdf5_group, _newly_created = self._all_create_or_get_groups(fullname)<EOL><DEDENT>store_dict = {}<EOL>if store_flags is None:<EOL><INDENT>store_flags = {}<EOL><DEDENT>try:<EOL><INDENT>if not instance.f_is_empty():<EOL><INDENT>store_dict = instance._store()<EOL><DEDENT>try:<EOL><INDENT>instance_flags = instance._store_flags().copy() <EOL><DEDENT>except AttributeError:<EOL><INDENT>instance_flags = {}<EOL><DEDENT>instance_flags.update(store_flags)<EOL>store_flags = instance_flags<EOL>self._prm_extract_missing_flags(store_dict, store_flags)<EOL>if overwrite:<EOL><INDENT>if isinstance(overwrite, str):<EOL><INDENT>overwrite = [overwrite]<EOL><DEDENT>if overwrite is True:<EOL><INDENT>to_delete = [key for key in store_dict.keys() if key in _hdf5_group]<EOL>self._all_delete_parameter_or_result_or_group(instance,<EOL>delete_only=to_delete,<EOL>_hdf5_group=_hdf5_group)<EOL><DEDENT>elif isinstance(overwrite, (list, tuple)):<EOL><INDENT>overwrite_set = set(overwrite)<EOL>key_set = set(store_dict.keys())<EOL>stuff_not_to_be_overwritten = overwrite_set - key_set<EOL>if overwrite!='<STR_LIT>' and len(stuff_not_to_be_overwritten) > <NUM_LIT:0>:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>str(stuff_not_to_be_overwritten))<EOL><DEDENT>stuff_to_overwrite = overwrite_set & key_set<EOL>if len(stuff_to_overwrite) > <NUM_LIT:0>:<EOL><INDENT>self._all_delete_parameter_or_result_or_group(instance,<EOL>delete_only=list(<EOL>stuff_to_overwrite))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % str(overwrite))<EOL><DEDENT><DEDENT>self._prm_store_from_dict(fullname, store_dict, _hdf5_group, store_flags, kwargs)<EOL>self._ann_store_annotations(instance, _hdf5_group, overwrite=overwrite)<EOL>if _newly_created or overwrite is True:<EOL><INDENT>self._prm_add_meta_info(instance, _hdf5_group,<EOL>overwrite=not _newly_created)<EOL><DEDENT>instance._stored = True<EOL>self._node_processing_timer.signal_update()<EOL><DEDENT>except:<EOL><INDENT>self._logger.error(<EOL>'<STR_LIT>' % fullname)<EOL>for key in store_dict.keys():<EOL><INDENT>if key in  _hdf5_group:<EOL><INDENT>hdf5_child = _hdf5_group._f_get_child(key)<EOL>hdf5_child._f_remove(recursive=True)<EOL><DEDENT><DEDENT>if _hdf5_group._v_nchildren == <NUM_LIT:0>:<EOL><INDENT>_hdf5_group._f_remove(recursive=True)<EOL><DEDENT>raise<EOL><DEDENT>

Stores a parameter or result to hdf5.

        :param instance:

            The instance to be stored

        :param store_data:

            How to store data

        :param store_flags:

            Dictionary containing how to store individual data, usually empty.

        :param overwrite:

            Instructions how to overwrite data

        :param with_links:

            Placeholder because leaves have no links

        :param recursive:

            Placeholder, because leaves have no children

        :param _hdf5_group:

            The hdf5 group for storing the parameter or result

        :param _newly_created:

            If should be created in a new form


def _prm_select_shared_pandas_data(self, pd_node, full_name, **kwargs):

try:<EOL><INDENT>pathname = pd_node._v_pathname<EOL>pandas_store = self._hdf5store<EOL>return pandas_store.select(pathname, **kwargs)<EOL><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (pd_node._v_name, full_name))<EOL>raise<EOL><DEDENT>

Reads a DataFrame from dis.

        :param pd_node:

            hdf5 node storing the pandas DataFrame

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :param kwargs:

            Arguments passed to pandas' select method


def _prm_write_shared_array(self, key, data, hdf5_group, full_name, flag, **kwargs):

if flag == HDF5StorageService.ARRAY:<EOL><INDENT>self._prm_write_into_array(key, data, hdf5_group, full_name, **kwargs)<EOL><DEDENT>elif flag in (HDF5StorageService.CARRAY,<EOL>HDF5StorageService.EARRAY,<EOL>HDF5StorageService.VLARRAY):<EOL><INDENT>self._prm_write_into_other_array(key, data, hdf5_group, full_name,<EOL>flag=flag, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>' %<EOL>(flag, key, full_name))<EOL><DEDENT>self._hdf5file.flush()<EOL>

Creates and array that can be used with an HDF5 array object


def _prm_write_shared_table(self, key, hdf5_group, fullname, **kwargs):

first_row = None<EOL>description = None<EOL>if '<STR_LIT>' in kwargs:<EOL><INDENT>first_row = kwargs.pop('<STR_LIT>')<EOL>if not '<STR_LIT:description>' in kwargs:<EOL><INDENT>description = {}<EOL>for colname in first_row:<EOL><INDENT>data = first_row[colname]<EOL>column = self._all_get_table_col(key, [data], fullname)<EOL>description[colname] = column<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT:description>' in kwargs:<EOL><INDENT>description = kwargs.pop('<STR_LIT:description>')<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>filters = kwargs.pop('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>filters = self._all_get_filters(kwargs)<EOL><DEDENT>table = self._hdf5file.create_table(where=hdf5_group, name=key,<EOL>description=description,<EOL>filters=filters,<EOL>**kwargs)<EOL>table.flush()<EOL>if first_row is not None:<EOL><INDENT>row = table.row<EOL>for key in description:<EOL><INDENT>row[key] = first_row[key]<EOL><DEDENT>row.append()<EOL>table.flush()<EOL><DEDENT>


def _prm_write_dict_as_table(self, key, data_to_store, group, fullname, **kwargs):

if key in group:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>')<EOL><DEDENT>if key in group:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>temp_dict = {}<EOL>for innerkey in data_to_store:<EOL><INDENT>val = data_to_store[innerkey]<EOL>temp_dict[innerkey] = [val]<EOL><DEDENT>objtable = ObjectTable(data=temp_dict)<EOL>self._prm_write_into_pytable(key, objtable, group, fullname, **kwargs)<EOL>new_table = group._f_get_child(key)<EOL>self._all_set_attributes_to_recall_natives(temp_dict, new_table,<EOL>HDF5StorageService.DATA_PREFIX)<EOL>setattr(new_table._v_attrs, HDF5StorageService.STORAGE_TYPE,<EOL>HDF5StorageService.DICT)<EOL>self._hdf5file.flush()<EOL>

Stores a python dictionary as pytable

        :param key:

            Name of data item to store

        :param data_to_store:

            Dictionary to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.


def _prm_write_pandas_data(self, key, data, group, fullname, flag, **kwargs):

try:<EOL><INDENT>if '<STR_LIT>' not in kwargs:<EOL><INDENT>filters = self._all_get_filters(kwargs)<EOL>kwargs['<STR_LIT>'] = filters<EOL><DEDENT>if '<STR_LIT>' not in kwargs:<EOL><INDENT>kwargs['<STR_LIT>'] = self.pandas_format<EOL><DEDENT>if '<STR_LIT>' not in kwargs:<EOL><INDENT>kwargs['<STR_LIT>'] = self.encoding<EOL><DEDENT>overwrite = kwargs.pop('<STR_LIT>', False)<EOL>if key in group and not (overwrite or kwargs.get('<STR_LIT>', False)):<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (key, fullname))<EOL><DEDENT>else:<EOL><INDENT>self._logger.debug('<STR_LIT>' % (key, fullname))<EOL><DEDENT>if data is not None and (kwargs['<STR_LIT>'] == '<STR_LIT:f>' or kwargs['<STR_LIT>'] == '<STR_LIT>'):<EOL><INDENT>kwargs['<STR_LIT>'] = data.shape[<NUM_LIT:0>]<EOL><DEDENT>name = group._v_pathname + '<STR_LIT:/>' + key<EOL>self._hdf5store.put(name, data, **kwargs)<EOL>self._hdf5store.flush()<EOL>self._hdf5file.flush()<EOL>frame_group = group._f_get_child(key)<EOL>setattr(frame_group._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)<EOL>self._hdf5file.flush()<EOL><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (key, fullname))<EOL>raise<EOL><DEDENT>

Stores a pandas DataFrame into hdf5.

        :param key:

            Name of data item to store

        :param data:

            Pandas Data to Store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param flag:

            If it is a series, frame or panel


def _prm_write_into_other_array(self, key, data, group, fullname,<EOL>flag, **kwargs):

try:<EOL><INDENT>if flag == HDF5StorageService.CARRAY:<EOL><INDENT>factory = self._hdf5file.create_carray<EOL><DEDENT>elif flag == HDF5StorageService.EARRAY:<EOL><INDENT>factory = self._hdf5file.create_earray<EOL><DEDENT>elif flag == HDF5StorageService.VLARRAY:<EOL><INDENT>factory = self._hdf5file.create_vlarray<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>if key in group:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>filters = kwargs.pop('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>filters = self._all_get_filters(kwargs)<EOL><DEDENT>try:<EOL><INDENT>other_array = factory(where=group, name=key, obj=data,<EOL>filters=filters, **kwargs)<EOL><DEDENT>except (ValueError, TypeError) as exc:<EOL><INDENT>try:<EOL><INDENT>conv_data = data[:]<EOL>conv_data = np.core.defchararray.encode(conv_data, self.encoding)<EOL>other_array = factory(where=group, name=key,<EOL>obj=conv_data,<EOL>filters=filters, **kwargs)<EOL><DEDENT>except Exception:<EOL><INDENT>raise exc<EOL><DEDENT><DEDENT>if data is not None:<EOL><INDENT>self._all_set_attributes_to_recall_natives(data, other_array,<EOL>HDF5StorageService.DATA_PREFIX)<EOL><DEDENT>setattr(other_array._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)<EOL>self._hdf5file.flush()<EOL><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (flag, key, fullname))<EOL>raise<EOL><DEDENT>

Stores data as carray, earray or vlarray depending on `flag`.

        :param key:

            Name of data item to store

        :param data:

            Data to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param recall:

            If container type and data type for perfect recall should be stored

        :param flag:

            How to store:
                CARRAY, EARRAY, VLARRAY


def _prm_write_into_array(self, key, data, group, fullname, **kwargs):

try:<EOL><INDENT>if key in group:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>array = self._hdf5file.create_array(where=group,<EOL>name=key, obj=data, **kwargs)<EOL><DEDENT>except (TypeError, ValueError) as exc:<EOL><INDENT>try:<EOL><INDENT>if type(data) is dict and len(data) == <NUM_LIT:0>:<EOL><INDENT>conv_data = ()<EOL><DEDENT>elif isinstance(data, str):<EOL><INDENT>conv_data = data.encode(self._encoding)<EOL><DEDENT>elif isinstance(data, int):<EOL><INDENT>conv_data = np.int64(data)<EOL><DEDENT>else:<EOL><INDENT>conv_data = []<EOL>for string in data:<EOL><INDENT>conv_data.append(string.encode(self._encoding))<EOL><DEDENT><DEDENT>array = self._hdf5file.create_array(where=group,<EOL>name=key, obj=conv_data, **kwargs)<EOL><DEDENT>except Exception:<EOL><INDENT>raise exc<EOL><DEDENT><DEDENT>if data is not None:<EOL><INDENT>self._all_set_attributes_to_recall_natives(data, array,<EOL>HDF5StorageService.DATA_PREFIX)<EOL><DEDENT>setattr(array._v_attrs, HDF5StorageService.STORAGE_TYPE,<EOL>HDF5StorageService.ARRAY)<EOL>self._hdf5file.flush()<EOL><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (key, fullname))<EOL>raise<EOL><DEDENT>

Stores data as array.

        :param key:

            Name of data item to store

        :param data:

            Data to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param recall:

            If container type and data type for perfect recall should be stored


translated_name = '<STR_LIT:/>' + self._trajectory_name + '<STR_LIT:/>' + link_name.replace('<STR_LIT:.>','<STR_LIT:/>')<EOL>link = self._hdf5file.get_node(where=translated_name)<EOL>link._f_remove()<EOL>


split_name = instance.v_location.split('<STR_LIT:.>')<EOL>if _hdf5_group is None:<EOL><INDENT>where = '<STR_LIT:/>' + self._trajectory_name + '<STR_LIT:/>' + '<STR_LIT:/>'.join(split_name)<EOL>node_name = instance.v_name<EOL>_hdf5_group = self._hdf5file.get_node(where=where, name=node_name)<EOL><DEDENT>if delete_only is None:<EOL><INDENT>if instance.v_is_group and not recursive and len(_hdf5_group._v_children) != <NUM_LIT:0>:<EOL><INDENT>raise TypeError('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>instance.v_full_name)<EOL><DEDENT>_hdf5_group._f_remove(recursive=True)<EOL><DEDENT>else:<EOL><INDENT>if not instance.v_is_leaf:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if isinstance(delete_only, str):<EOL><INDENT>delete_only = [delete_only]<EOL><DEDENT>for delete_item in delete_only:<EOL><INDENT>if (remove_from_item and<EOL>hasattr(instance, '<STR_LIT>') and<EOL>hasattr(instance, '<STR_LIT>') and<EOL>delete_item in instance):<EOL><INDENT>delattr(instance, delete_item)<EOL><DEDENT>try:<EOL><INDENT>_hdf5_sub_group = self._hdf5file.get_node(where=_hdf5_group,<EOL>name=delete_item)<EOL>_hdf5_sub_group._f_remove(recursive=True)<EOL><DEDENT>except pt.NoSuchNodeError:<EOL><INDENT>self._logger.warning('<STR_LIT>' %<EOL>(delete_item, instance.v_full_name))<EOL><DEDENT><DEDENT><DEDENT>

Removes a parameter or result or group from the hdf5 file.

        :param instance: Instance to be removed

        :param delete_only:

            List of elements if you only want to delete parts of a leaf node. Note that this
            needs to list the names of the hdf5 subnodes. BE CAREFUL if you erase parts of a leaf.
            Erasing partly happens at your own risk, it might be the case that you can
            no longer reconstruct the leaf from the leftovers!

        :param remove_from_item:

            If using `delete_only` and `remove_from_item=True` after deletion the data item is
            also removed from the `instance`.

        :param recursive:

            If a group node has children, you will can delete it if recursive is True.


def _prm_write_into_pytable(self, tablename, data, hdf5_group, fullname, **kwargs):

datasize = data.shape[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>description_dict, data_type_dict = self._prm_make_description(data, fullname)<EOL>description_dicts = [{}]<EOL>if len(description_dict) > ptpa.MAX_COLUMNS:<EOL><INDENT>new_table_group = self._hdf5file.create_group(where=hdf5_group,<EOL>name=tablename,<EOL>filters=self._all_get_filters(kwargs.copy()))<EOL>count = <NUM_LIT:0><EOL>for innerkey in description_dict:<EOL><INDENT>val = description_dict[innerkey]<EOL>if count == ptpa.MAX_COLUMNS:<EOL><INDENT>description_dicts.append({})<EOL>count = <NUM_LIT:0><EOL><DEDENT>description_dicts[-<NUM_LIT:1>][innerkey] = val<EOL>count += <NUM_LIT:1><EOL><DEDENT>setattr(new_table_group._v_attrs, HDF5StorageService.STORAGE_TYPE,<EOL>HDF5StorageService.TABLE)<EOL>setattr(new_table_group._v_attrs, HDF5StorageService.SPLIT_TABLE, <NUM_LIT:1>)<EOL>hdf5_group = new_table_group<EOL><DEDENT>else:<EOL><INDENT>description_dicts = [description_dict]<EOL><DEDENT>for idx, descr_dict in enumerate(description_dicts):<EOL><INDENT>if idx == <NUM_LIT:0>:<EOL><INDENT>tblname = tablename<EOL><DEDENT>else:<EOL><INDENT>tblname = tablename + '<STR_LIT>' % idx<EOL><DEDENT>table = self._hdf5file.create_table(where=hdf5_group, name=tblname,<EOL>description=descr_dict,<EOL>title=tblname,<EOL>expectedrows=datasize,<EOL>filters=self._all_get_filters(kwargs.copy()))<EOL>row = table.row<EOL>for n in range(datasize):<EOL><INDENT>for key in descr_dict:<EOL><INDENT>row[key] = data[key][n]<EOL><DEDENT>row.append()<EOL><DEDENT>if idx == <NUM_LIT:0> and len(description_dict) <= ptpa.MAX_COLUMNS:<EOL><INDENT>for field_name in data_type_dict:<EOL><INDENT>type_description = data_type_dict[field_name]<EOL>self._all_set_attr(table, field_name, type_description)<EOL><DEDENT>setattr(table._v_attrs, HDF5StorageService.STORAGE_TYPE,<EOL>HDF5StorageService.TABLE)<EOL><DEDENT>table.flush()<EOL>self._hdf5file.flush()<EOL><DEDENT>if len(description_dict) > ptpa.MAX_COLUMNS:<EOL><INDENT>tblname = tablename + '<STR_LIT>' + HDF5StorageService.STORAGE_TYPE<EOL>field_names, data_types = list(zip(*data_type_dict.items()))<EOL>data_type_table_dict = {'<STR_LIT>': field_names, '<STR_LIT>': data_types}<EOL>descr_dict, _ = self._prm_make_description(data_type_table_dict, fullname)<EOL>table = self._hdf5file.create_table(where=hdf5_group, name=tblname,<EOL>description=descr_dict,<EOL>title=tblname,<EOL>expectedrows=len(field_names),<EOL>filters=self._all_get_filters(kwargs))<EOL>row = table.row<EOL>for n in range(len(field_names)):<EOL><INDENT>for key in data_type_table_dict:<EOL><INDENT>row[key] = data_type_table_dict[key][n]<EOL><DEDENT>row.append()<EOL><DEDENT>setattr(table._v_attrs, HDF5StorageService.DATATYPE_TABLE, <NUM_LIT:1>)<EOL>table.flush()<EOL>self._hdf5file.flush()<EOL><DEDENT><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (tablename, fullname))<EOL>raise<EOL><DEDENT>

Stores data as pytable.

        :param tablename:

            Name of the data table

        :param data:

            Data to store

        :param hdf5_group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.


def _prm_make_description(self, data, fullname):

def _convert_lists_and_tuples(series_of_data):<EOL><INDENT>"""<STR_LIT>"""<EOL>if isinstance(series_of_data[<NUM_LIT:0>],<EOL>(list, tuple)):  <EOL><INDENT>for idx, item in enumerate(series_of_data):<EOL><INDENT>series_of_data[idx] = np.array(item)<EOL><DEDENT><DEDENT><DEDENT>descriptiondict = {}  <EOL>original_data_type_dict = {}  <EOL>for key in data:<EOL><INDENT>val = data[key]<EOL>self._all_set_attributes_to_recall_natives(val[<NUM_LIT:0>], PTItemMock(original_data_type_dict),<EOL>HDF5StorageService.FORMATTED_COLUMN_PREFIX %<EOL>key)<EOL>_convert_lists_and_tuples(val)<EOL>col = self._all_get_table_col(key, val, fullname)<EOL>descriptiondict[key] = col<EOL><DEDENT>return descriptiondict, original_data_type_dict<EOL>

Returns a description dictionary for pytables table creation


def _all_get_table_col(self, key, column, fullname):

val = column[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>if type(val) is int:<EOL><INDENT>return pt.IntCol()<EOL><DEDENT>if isinstance(val, (str, bytes)):<EOL><INDENT>itemsize = int(self._prm_get_longest_stringsize(column))<EOL>return pt.StringCol(itemsize)<EOL><DEDENT>if isinstance(val, np.ndarray):<EOL><INDENT>if (np.issubdtype(val.dtype, str) or<EOL>np.issubdtype(val.dtype, bytes)):<EOL><INDENT>itemsize = int(self._prm_get_longest_stringsize(column))<EOL>return pt.StringCol(itemsize, shape=val.shape)<EOL><DEDENT>else:<EOL><INDENT>return pt.Col.from_dtype(np.dtype((val.dtype, val.shape)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return pt.Col.from_dtype(np.dtype(type(val)))<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>self._logger.error('<STR_LIT>'<EOL>'<STR_LIT>' % (key, fullname, repr(type(val))))<EOL>raise<EOL><DEDENT>

Creates a pytables column instance.

        The type of column depends on the type of `column[0]`.
        Note that data in `column` must be homogeneous!


maxlength = <NUM_LIT:1><EOL>for stringar in string_list:<EOL><INDENT>if isinstance(stringar, np.ndarray):<EOL><INDENT>if stringar.ndim > <NUM_LIT:0>:<EOL><INDENT>for string in stringar.ravel():<EOL><INDENT>maxlength = max(len(string), maxlength)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>maxlength = max(len(stringar.tolist()), maxlength)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>maxlength = max(len(stringar), maxlength)<EOL><DEDENT><DEDENT>return int(maxlength * <NUM_LIT>)<EOL>

Returns the longest string size for a string entry across data.


def _prm_load_into_dict(self, full_name, load_dict, hdf5_group, instance,<EOL>load_only, load_except, load_flags, _prefix = '<STR_LIT>'):

for node in hdf5_group:<EOL><INDENT>load_type = self._all_get_from_attrs(node, HDF5StorageService.STORAGE_TYPE)<EOL>if _prefix:<EOL><INDENT>load_name = '<STR_LIT>' % (_prefix, node._v_name)<EOL><DEDENT>else:<EOL><INDENT>load_name = node._v_name<EOL><DEDENT>if load_type == HDF5StorageService.NESTED_GROUP:<EOL><INDENT>self._prm_load_into_dict(full_name=full_name,<EOL>load_dict=load_dict,<EOL>hdf5_group=node,<EOL>instance=instance,<EOL>load_only=load_only,<EOL>load_except=load_except,<EOL>load_flags=load_flags,<EOL>_prefix=load_name)<EOL>continue<EOL><DEDENT>if load_only is not None:<EOL><INDENT>if load_name not in load_only:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>load_only.remove(load_name)<EOL><DEDENT><DEDENT>elif load_except is not None:<EOL><INDENT>if load_name in load_except:<EOL><INDENT>load_except.remove(load_name)<EOL>continue<EOL><DEDENT><DEDENT>if load_name in load_flags:<EOL><INDENT>load_type = load_flags[load_name]<EOL><DEDENT>if load_type == HDF5StorageService.DICT:<EOL><INDENT>to_load = self._prm_read_dictionary(node, full_name)<EOL><DEDENT>elif load_type == HDF5StorageService.TABLE:<EOL><INDENT>to_load = self._prm_read_table(node, full_name)<EOL><DEDENT>elif load_type in (HDF5StorageService.ARRAY, HDF5StorageService.CARRAY,<EOL>HDF5StorageService.EARRAY, HDF5StorageService.VLARRAY):<EOL><INDENT>to_load = self._prm_read_array(node, full_name)<EOL><DEDENT>elif load_type in (HDF5StorageService.FRAME,<EOL>HDF5StorageService.SERIES,<EOL>):<EOL><INDENT>to_load = self._prm_read_pandas(node, full_name)<EOL><DEDENT>elif load_type.startswith(HDF5StorageService.SHARED_DATA):<EOL><INDENT>to_load = self._prm_read_shared_data(node, instance)<EOL><DEDENT>else:<EOL><INDENT>raise pex.NoSuchServiceError('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(full_name, str(node), str(load_type)))<EOL><DEDENT>if to_load is None:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>load_dict[load_name] = to_load<EOL><DEDENT>


if load_data == pypetconstants.LOAD_NOTHING:<EOL><INDENT>return<EOL><DEDENT>if _hdf5_group is None:<EOL><INDENT>_hdf5_group = self._all_get_node_by_name(instance.v_full_name)<EOL><DEDENT>if load_data == pypetconstants.OVERWRITE_DATA:<EOL><INDENT>if instance.v_is_parameter and instance.v_locked:<EOL><INDENT>self._logger.debug('<STR_LIT>' %<EOL>instance.v_full_name)<EOL>return<EOL><DEDENT>instance.f_empty()<EOL>instance.v_annotations.f_empty()<EOL>instance.v_comment = '<STR_LIT>'<EOL><DEDENT>self._all_load_skeleton(instance, _hdf5_group)<EOL>instance._stored = True<EOL>if isinstance(load_only, str):<EOL><INDENT>load_only = [load_only]<EOL><DEDENT>if isinstance(load_except, str):<EOL><INDENT>load_except = [load_except]<EOL><DEDENT>if load_data == pypetconstants.LOAD_SKELETON:<EOL><INDENT>self._node_processing_timer.signal_update()<EOL>return<EOL><DEDENT>elif load_only is not None:<EOL><INDENT>if load_except is not None:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>elif instance.v_is_parameter and instance.v_locked:<EOL><INDENT>raise pex.ParameterLockedException('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>instance.v_full_name)<EOL><DEDENT>self._logger.debug('<STR_LIT>' %<EOL>str(load_only))<EOL>load_only = set(load_only)<EOL><DEDENT>elif load_except is not None:<EOL><INDENT>if instance.v_is_parameter and instance.v_locked:<EOL><INDENT>raise pex.ParameterLockedException('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>instance.v_full_name)<EOL><DEDENT>self._logger.debug('<STR_LIT>' %<EOL>str(load_except))<EOL>load_except = set(load_except)<EOL><DEDENT>elif not instance.f_is_empty():<EOL><INDENT>self._node_processing_timer.signal_update()<EOL>return<EOL><DEDENT>full_name = instance.v_full_name<EOL>self._logger.debug('<STR_LIT>' % full_name)<EOL>load_dict = {}  <EOL>if load_flags is None:<EOL><INDENT>load_flags = {}<EOL><DEDENT>try:<EOL><INDENT>instance_flags = instance._load_flags().copy() <EOL><DEDENT>except AttributeError:<EOL><INDENT>instance_flags = {}<EOL><DEDENT>instance_flags.update(load_flags)<EOL>load_flags = instance_flags<EOL>self._prm_load_into_dict(full_name=full_name,<EOL>load_dict=load_dict,<EOL>hdf5_group=_hdf5_group,<EOL>instance=instance,<EOL>load_only=load_only,<EOL>load_except=load_except,<EOL>load_flags=load_flags)<EOL>if load_only is not None:<EOL><INDENT>if len(load_only) > <NUM_LIT:0>:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(str(load_only), full_name))<EOL><DEDENT><DEDENT>elif load_except is not None:<EOL><INDENT>if len(load_except) > <NUM_LIT:0>:<EOL><INDENT>self._logger.warning(('<STR_LIT>'<EOL>'<STR_LIT>' % (str(load_except), full_name)))<EOL><DEDENT><DEDENT>if load_dict:<EOL><INDENT>try:<EOL><INDENT>instance._load(load_dict)<EOL>if instance.v_is_parameter:<EOL><INDENT>instance.f_lock()<EOL><DEDENT><DEDENT>except:<EOL><INDENT>self._logger.error(<EOL>'<STR_LIT>' % full_name)<EOL>raise<EOL><DEDENT><DEDENT>self._node_processing_timer.signal_update()<EOL>

Loads a parameter or result from disk.

        :param instance:

            Empty parameter or result instance

        :param load_data:

            How to load stuff

        :param load_only:

            List of data keys if only parts of a result should be loaded

        :param load_except:

            List of data key that should NOT be loaded.

        :param load_flags:

            Dictionary to determine how something is loaded

        :param with_links:

            Placeholder, because leaves have no links

        :param recursive:

            Dummy variable, no-op because leaves have no children

        :param max_depth:

            Dummy variable, no-op because leaves have no children

        :param _hdf5_group:

            The corresponding hdf5 group of the instance


def _prm_read_dictionary(self, leaf, full_name):

try:<EOL><INDENT>temp_table = self._prm_read_table(leaf, full_name)<EOL>temp_dict = temp_table.to_dict('<STR_LIT:list>')<EOL>innder_dict = {}<EOL>for innerkey, vallist in temp_dict.items():<EOL><INDENT>innder_dict[innerkey] = vallist[<NUM_LIT:0>]<EOL><DEDENT>return innder_dict<EOL><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (leaf._v_name, full_name))<EOL>raise<EOL><DEDENT>

Loads data that was originally a dictionary when stored

        :param leaf:

            PyTables table containing the dictionary data

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to be loaded


def _prm_read_shared_data(self, shared_node, instance):

try:<EOL><INDENT>data_type = self._all_get_from_attrs(shared_node,<EOL>HDF5StorageService.SHARED_DATA_TYPE)<EOL>constructor = shared.FLAG_CLASS_MAPPING[data_type]<EOL>name = shared_node._v_name<EOL>result = constructor(name=name, parent=instance)<EOL>return result<EOL><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (shared_node._v_name,<EOL>instance.v_full_name))<EOL>raise<EOL><DEDENT>

Reads shared data and constructs the appropraite class.

        :param shared_node:

            hdf5 node storing the pandas DataFrame

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load


def _prm_read_pandas(self, pd_node, full_name):

try:<EOL><INDENT>name = pd_node._v_name<EOL>pathname = pd_node._v_pathname<EOL>pandas_store = self._hdf5store<EOL>pandas_data = pandas_store.get(pathname)<EOL>return pandas_data<EOL><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (pd_node._v_name, full_name))<EOL>raise<EOL><DEDENT>

Reads a DataFrame from dis.

        :param pd_node:

            hdf5 node storing the pandas DataFrame

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load


def _prm_read_table(self, table_or_group, full_name):

try:<EOL><INDENT>result_table = None<EOL>if self._all_get_from_attrs(table_or_group, HDF5StorageService.SPLIT_TABLE):<EOL><INDENT>table_name = table_or_group._v_name<EOL>data_type_table_name = table_name + '<STR_LIT>' + HDF5StorageService.STORAGE_TYPE<EOL>data_type_table = table_or_group._v_children[data_type_table_name]<EOL>data_type_dict = {}<EOL>for row in data_type_table:<EOL><INDENT>fieldname = row['<STR_LIT>'].decode('<STR_LIT:utf-8>')<EOL>data_type_dict[fieldname] = row['<STR_LIT>'].decode('<STR_LIT:utf-8>')<EOL><DEDENT>for sub_table in table_or_group:<EOL><INDENT>sub_table_name = sub_table._v_name<EOL>if sub_table_name == data_type_table_name:<EOL><INDENT>continue<EOL><DEDENT>for colname in sub_table.colnames:<EOL><INDENT>col = sub_table.col(colname)<EOL>data_list = list(col)<EOL>prefix = HDF5StorageService.FORMATTED_COLUMN_PREFIX % colname<EOL>for idx, data in enumerate(data_list):<EOL><INDENT>data, type_changed = self._all_recall_native_type(data,<EOL>PTItemMock(<EOL>data_type_dict),<EOL>prefix)<EOL>if type_changed:<EOL><INDENT>data_list[idx] = data<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if result_table is None:<EOL><INDENT>result_table = ObjectTable(data={colname: data_list})<EOL><DEDENT>else:<EOL><INDENT>result_table[colname] = data_list<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for colname in table_or_group.colnames:<EOL><INDENT>col = table_or_group.col(colname)<EOL>data_list = list(col)<EOL>prefix = HDF5StorageService.FORMATTED_COLUMN_PREFIX % colname<EOL>for idx, data in enumerate(data_list):<EOL><INDENT>data, type_changed = self._all_recall_native_type(data, table_or_group,<EOL>prefix)<EOL>if type_changed:<EOL><INDENT>data_list[idx] = data<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if result_table is None:<EOL><INDENT>result_table = ObjectTable(data={colname: data_list})<EOL><DEDENT>else:<EOL><INDENT>result_table[colname] = data_list<EOL><DEDENT><DEDENT><DEDENT>return result_table<EOL><DEDENT>except:<EOL><INDENT>self._logger.error(<EOL>'<STR_LIT>' % (table_or_group._v_name, full_name))<EOL>raise<EOL><DEDENT>

Reads a non-nested PyTables table column by column and created a new ObjectTable for
        the loaded data.

        :param table_or_group:

            PyTables table to read from or a group containing subtables.

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to be loaded


def _prm_read_array(self, array, full_name):

try:<EOL><INDENT>result = self._svrc_read_array(array)<EOL>result, dummy = self._all_recall_native_type(result, array,<EOL>HDF5StorageService.DATA_PREFIX)<EOL>return result<EOL><DEDENT>except:<EOL><INDENT>self._logger.error('<STR_LIT>' % (array._v_name, full_name))<EOL>raise<EOL><DEDENT>

Reads data from an array or carray

        :param array:

            PyTables array or carray to read from

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load


def make_ordinary_result(result, key, trajectory=None, reload=True):

shared_data = result.f_get(key)<EOL>if trajectory is not None:<EOL><INDENT>shared_data.traj = trajectory<EOL><DEDENT>shared_data._request_data('<STR_LIT>')<EOL>result.f_remove(key)<EOL>if reload:<EOL><INDENT>trajectory.f_load_item(result, load_data=pypetconstants.OVERWRITE_DATA)<EOL><DEDENT>return result<EOL>

Turns a given shared data item into a an ordinary one.

    :param result: Result container with shared data
    :param key: The name of the shared data
    :param trajectory:

        The trajectory, only needed if shared data has
        no access to the trajectory, yet.

    :param reload: If data should be reloaded after conversion

    :return: The result


def make_shared_result(result, key, trajectory, new_class=None):

data = result.f_get(key)<EOL>if new_class is None:<EOL><INDENT>if isinstance(data, ObjectTable):<EOL><INDENT>new_class = SharedTable<EOL><DEDENT>elif isinstance(data, pd.DataFrame):<EOL><INDENT>new_class = SharedPandasFrame<EOL><DEDENT>elif isinstance(data, (tuple, list)):<EOL><INDENT>new_class = SharedArray<EOL><DEDENT>elif isinstance(data, (np.ndarray, np.matrix)):<EOL><INDENT>new_class = SharedCArray<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>' % key)<EOL><DEDENT><DEDENT>shared_data = new_class(result.f_translate_key(key), result, trajectory=trajectory)<EOL>result[key] = shared_data<EOL>shared_data._request_data('<STR_LIT>')<EOL>return result<EOL>

Turns an ordinary data item into a shared one.

    Removes the old result from the trajectory and replaces it.
    Empties the given result.

    :param result: The result containing ordinary data
    :param key: Name of ordinary data item
    :param trajectory: Trajectory container
    :param new_class:

        Class of new shared data item.
        Leave `None` for automatic detection.

    :return: The `result`


service = self._traj.v_storage_service<EOL>if not service.is_open:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>service.store(pypetconstants.CLOSE_FILE, None)<EOL>

Closes store manually not needed if used with `with`


service = self._traj.v_storage_service<EOL>if service.is_open:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>service.store(pypetconstants.OPEN_FILE, None,<EOL>trajectory_name=self._traj.v_name)<EOL>

Opens store manually not needed if used with `with`


service = self._traj.v_storage_service<EOL>if not service.is_open:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>service.store(pypetconstants.FLUSH, None)<EOL>

Flushes data to the storage can be called at any time when storage is open.


if '<STR_LIT>' not in kwargs:<EOL><INDENT>kwargs['<STR_LIT>'] = self.FLAG<EOL><DEDENT>if '<STR_LIT:data>' in kwargs:<EOL><INDENT>kwargs['<STR_LIT>'] = kwargs.pop('<STR_LIT:data>')<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>self.traj = kwargs.pop('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>self.traj = kwargs.pop('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT:name>' in kwargs:<EOL><INDENT>self.name = kwargs.pop['<STR_LIT:name>']<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>self.parent = kwargs.pop('<STR_LIT>')<EOL>if self.name is not None:<EOL><INDENT>self.parent[self.name] = self<EOL><DEDENT><DEDENT>return self._request_data('<STR_LIT>', kwargs=kwargs)<EOL>

Creates shared data on disk with a StorageService on disk.

        Needs to be called before shared data can be used later on.

        Actual arguments of ``kwargs`` depend on the type of data to be
        created. For instance, creating an array one can use the keyword
        ``obj`` to pass a numpy array (``obj=np.zeros((10,20,30))``).
        Whereas for a PyTables table may need a description dictionary
        (``description={'column_1': pt.StringCol(2, pos=0),'column_2': pt.FloatCol( pos=1)}``)
        Refer to the PyTables documentation on how to create tables.


def _request_data(self, request, args=None, kwargs=None):

return self._storage_service.store(pypetconstants.ACCESS_DATA,<EOL>self.parent.v_full_name,<EOL>self.name,<EOL>request, args, kwargs,<EOL>trajectory_name=self.traj.v_name)<EOL>

Interface with the underlying storage.

        Passes request to the StorageService that performs the appropriate action.
        For example, given a shared table ``t``.
        ``t.remove_row(4)`` is parsed into ``request='remove_row', args=(4,)`` and
        passed onto the storage service. In case of the HDF5StorageService,
        this is again translated back into ``hdf5_table_node.remove_row(4)``.


if not self._storage_service.is_open:<EOL><INDENT>warnings.warn('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>category=RuntimeWarning)<EOL><DEDENT>return self._request_data('<STR_LIT>')<EOL>

Returns the actula node of the underlying data.

        In case one uses HDF5 this will be the HDF5 leaf node.


result = super(SharedResult, self)._supports(item)<EOL>result = result or type(item) in SharedResult.SUPPORTED_DATA<EOL>return result<EOL>

Checks if outer data structure is supported.


def create_shared_data(self, name=None, **kwargs):

if name is None:<EOL><INDENT>item = self.f_get()<EOL><DEDENT>else:<EOL><INDENT>item = self.f_get(name)<EOL><DEDENT>return item.create_shared_data(**kwargs)<EOL>

Calls the corresponding function of the shared data item


return False<EOL>

Normally the file is opened and closed after each insertion.

        However, the storage service may provide to keep the store open and signals
        this via this property.


return True<EOL>

This wrapper guarantees multiprocessing safety


return True<EOL>

Hook that can be used to temper with the server before responding

        :param response: Response to be send

        :return: Boolean value if response should be send or not


def _lock(self, name, client_id, request_id):

if name in self._locks:<EOL><INDENT>other_client_id, other_request_id = self._locks[name]<EOL>if other_client_id == client_id:<EOL><INDENT>response = (self.LOCK_ERROR + self.DELIMITER +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (name, client_id, other_request_id, request_id))<EOL>self._logger.warning(response)<EOL>return response<EOL><DEDENT>else:<EOL><INDENT>return self.WAIT<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self._locks[name] = (client_id, request_id)<EOL>return self.GO<EOL><DEDENT>

Hanldes locking of locks

        If a lock is already locked sends a WAIT command,
        else LOCKs it and sends GO.

        Complains if a given client re-locks a lock without releasing it before.


def _unlock(self, name, client_id, request_id):

if name in self._locks:<EOL><INDENT>other_client_id, other_request_id = self._locks[name]<EOL>if other_client_id != client_id:<EOL><INDENT>response = (self.RELEASE_ERROR + self.DELIMITER +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (name,<EOL>other_client_id,<EOL>other_request_id,<EOL>client_id,<EOL>request_id))<EOL>self._logger.error(response)<EOL>return response<EOL><DEDENT>else:<EOL><INDENT>del self._locks[name]<EOL>return self.RELEASED<EOL><DEDENT><DEDENT>else:<EOL><INDENT>response = (self.RELEASE_ERROR + self.DELIMITER +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (name, client_id, request_id))<EOL>self._logger.error(response)<EOL>return response<EOL><DEDENT>

Handles unlocking

        Complains if a non-existent lock should be released or
        if a lock should be released that was acquired by
        another client before.


try:<EOL><INDENT>self._start()<EOL>running = True<EOL>while running:<EOL><INDENT>msg = '<STR_LIT>'<EOL>name = '<STR_LIT>'<EOL>client_id = '<STR_LIT>'<EOL>request_id = '<STR_LIT>'<EOL>request = self._socket.recv_string()<EOL>self._logger.log(<NUM_LIT:1>, '<STR_LIT>', request)<EOL>split_msg = request.split(self.DELIMITER)<EOL>if len(split_msg) == <NUM_LIT:4>:<EOL><INDENT>msg, name, client_id, request_id = split_msg<EOL><DEDENT>if msg == self.LOCK:<EOL><INDENT>response = self._lock(name, client_id, request_id)<EOL><DEDENT>elif msg == self.UNLOCK:<EOL><INDENT>response = self._unlock(name, client_id, request_id)<EOL><DEDENT>elif msg == self.PING:<EOL><INDENT>response = self.PONG<EOL><DEDENT>elif msg == self.DONE:<EOL><INDENT>response = self.CLOSED<EOL>running = False<EOL><DEDENT>else:<EOL><INDENT>response = (self.MSG_ERROR + self.DELIMITER +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % request)<EOL>self._logger.error(response)<EOL><DEDENT>respond = self._pre_respond_hook(response)<EOL>if respond:<EOL><INDENT>self._logger.log(<NUM_LIT:1>, '<STR_LIT>',<EOL>response, client_id, request_id)<EOL>self._socket.send_string(response)<EOL><DEDENT><DEDENT>self._close()<EOL><DEDENT>except Exception:<EOL><INDENT>self._logger.exception('<STR_LIT>')<EOL>raise<EOL><DEDENT>


if name in self._locks:<EOL><INDENT>other_client_id, other_request_id, lock_time = self._locks[name]<EOL>if other_client_id == client_id:<EOL><INDENT>response = (self.LOCK_ERROR + self.DELIMITER +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (name, client_id, other_request_id, request_id))<EOL>self._logger.warning(response)<EOL>return response<EOL><DEDENT>else:<EOL><INDENT>current_time = time.time()<EOL>if current_time - lock_time < self._timeout:<EOL><INDENT>return self.WAIT<EOL><DEDENT>else:<EOL><INDENT>response = (self.GO + self.DELIMITER + '<STR_LIT>'<EOL>'<STR_LIT>' % (name,<EOL>other_client_id,<EOL>other_request_id))<EOL>self._logger.info(response)<EOL>self._locks[name] = (client_id, request_id, time.time())<EOL>self._timeout_locks[(name, other_client_id)] = (request_id, lock_time)<EOL>return response<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self._locks[name] = (client_id, request_id, time.time())<EOL>return self.GO<EOL><DEDENT>

Handles locking

        Locking time is stored to determine time out.
        If a lock is timed out it can be acquired by a different client.


if name in self._locks:<EOL><INDENT>other_client_id, other_request_id, lock_time = self._locks[name]<EOL>if other_client_id != client_id:<EOL><INDENT>response = (self.RELEASE_ERROR + self.DELIMITER +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (name,<EOL>other_client_id,<EOL>other_request_id,<EOL>client_id,<EOL>request_id))<EOL>self._logger.error(response)<EOL>return response<EOL><DEDENT>else:<EOL><INDENT>del self._locks[name]<EOL>return self.RELEASED<EOL><DEDENT><DEDENT>elif (name, client_id) in self._timeout_locks:<EOL><INDENT>other_request_id, lock_time = self._timeout_locks[(name, client_id)]<EOL>timeout = time.time() - lock_time - self._timeout<EOL>response = (self.RELEASE_ERROR + self.DELIMITER +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (name, timeout, client_id, other_request_id))<EOL>return response<EOL><DEDENT>else:<EOL><INDENT>response = (self.RELEASE_ERROR + self.DELIMITER +<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (name, client_id, request_id))<EOL>self._logger.warning(response)<EOL>return response<EOL><DEDENT>


self.start(test_connection=False)<EOL>self._logger.debug('<STR_LIT>')<EOL>self._req_rep(ZMQServer.DONE)<EOL>


if self._context is not None:<EOL><INDENT>if self._socket is not None:<EOL><INDENT>self._close_socket(confused=False)<EOL><DEDENT>self._context.term()<EOL>self._context = None<EOL>self._poll = None<EOL><DEDENT>

Closes socket and terminates context

        NO-OP if already closed.


return self._req_rep_retry(request)[<NUM_LIT:0>]<EOL>

Returns server response on `request_sketch`


retries_left = self.RETRIES<EOL>while retries_left:<EOL><INDENT>self._logger.log(<NUM_LIT:1>, '<STR_LIT>', request)<EOL>self._send_request(request)<EOL>socks = dict(self._poll.poll(self.TIMEOUT))<EOL>if socks.get(self._socket) == zmq.POLLIN:<EOL><INDENT>response = self._receive_response()<EOL>self._logger.log(<NUM_LIT:1>, '<STR_LIT>', response)<EOL>return response, self.RETRIES - retries_left<EOL><DEDENT>else:<EOL><INDENT>self._logger.debug('<STR_LIT>' %<EOL>retries_left)<EOL>self._close_socket(confused=True)<EOL>retries_left -= <NUM_LIT:1><EOL>if retries_left == <NUM_LIT:0>:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>time.sleep(self.SLEEP)<EOL>self._start_socket()<EOL><DEDENT><DEDENT>


Actual sending of the request over network


return self._socket.recv_string()<EOL>


self.start(test_connection=False)<EOL>while True:<EOL><INDENT>str_response, retries = self._req_rep_retry(LockerServer.LOCK)<EOL>response = str_response.split(LockerServer.DELIMITER)<EOL>if response[<NUM_LIT:0>] == LockerServer.GO:<EOL><INDENT>return True<EOL><DEDENT>elif response[<NUM_LIT:0>] == LockerServer.LOCK_ERROR and retries > <NUM_LIT:0>:<EOL><INDENT>self._logger.error(str_response + '<STR_LIT>')<EOL>return True<EOL><DEDENT>elif response[<NUM_LIT:0>] == LockerServer.WAIT:<EOL><INDENT>time.sleep(self.SLEEP)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>' % response)<EOL><DEDENT><DEDENT>

Acquires lock and returns `True`

        Blocks until lock is available.


<EOL>str_response, retries = self._req_rep_retry(LockerServer.UNLOCK)<EOL>response = str_response.split(LockerServer.DELIMITER)<EOL>if response[<NUM_LIT:0>] == LockerServer.RELEASED:<EOL><INDENT>pass  <EOL><DEDENT>elif response[<NUM_LIT:0>] == LockerServer.RELEASE_ERROR and retries > <NUM_LIT:0>:<EOL><INDENT>self._logger.error(str_response + '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>' % response)<EOL><DEDENT>


count = <NUM_LIT:0><EOL>self._start()<EOL>while True:<EOL><INDENT>result = self._socket.recv_pyobj()<EOL>if isinstance(result, tuple):<EOL><INDENT>request, data = result<EOL><DEDENT>else:<EOL><INDENT>request = result<EOL>data = None<EOL><DEDENT>if request == self.SPACE:<EOL><INDENT>if self.queue.qsize() + count < self.queue_maxsize:<EOL><INDENT>self._socket.send_string(self.SPACE_AVAILABLE)<EOL>count += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>self._socket.send_string(self.SPACE_NOT_AVAILABLE)<EOL><DEDENT><DEDENT>elif request == self.PING:<EOL><INDENT>self._socket.send_string(self.PONG)<EOL><DEDENT>elif request == self.DATA:<EOL><INDENT>self._socket.send_string(self.STORING)<EOL>self.queue.put(data)<EOL>count -= <NUM_LIT:1><EOL><DEDENT>elif request == self.DONE:<EOL><INDENT>self._socket.send_string(ZMQServer.CLOSED)<EOL>self.queue.put(('<STR_LIT>', [], {}))<EOL>self._close()<EOL>break<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>' % request)<EOL><DEDENT><DEDENT>

Handles listening requests from the client.

        There are 4 types of requests:

        1- Check space in the queue
        2- Tests the socket
        3- If there is a space, it sends data
        4- after data is sent, puts it to queue for storing


self.start(test_connection=False)<EOL>while True:<EOL><INDENT>response = self._req_rep(QueuingServerMessageListener.SPACE)<EOL>if response == QueuingServerMessageListener.SPACE_AVAILABLE:<EOL><INDENT>self._req_rep((QueuingServerMessageListener.DATA, data))<EOL>break<EOL><DEDENT>else:<EOL><INDENT>time.sleep(<NUM_LIT>)<EOL><DEDENT><DEDENT>

If there is space it sends data to server

        If no space in the queue

        It returns the request in every 10 millisecond

        until there will be space in the queue.


if self._pid is None:<EOL><INDENT>self._pid = os.getpid()<EOL><DEDENT>if self._context is not None:<EOL><INDENT>current_pid = os.getpid()<EOL>if current_pid != self._pid:<EOL><INDENT>self._logger.debug('<STR_LIT>'<EOL>'<STR_LIT>' % (str(self._pid), str(current_pid)))<EOL>self._context = None<EOL>self._pid = current_pid<EOL><DEDENT><DEDENT>

Detects if lock client was forked.

        Forking is detected by comparing the PID of the current
        process with the stored PID.


@retry(<NUM_LIT:9>, Exception, <NUM_LIT>, '<STR_LIT>')<EOL><INDENT>def _put_on_queue(self, to_put):<DEDENT>

old = self.pickle_queue<EOL>self.pickle_queue = False<EOL>try:<EOL><INDENT>self.queue.put(to_put, block=True)<EOL><DEDENT>finally:<EOL><INDENT>self.pickle_queue = old<EOL><DEDENT>


self._put_on_queue(('<STR_LIT>', args, kwargs))<EOL>

Puts data to store on queue.

        Note that the queue will no longer be pickled if the Sender is pickled.


self._put_on_queue(('<STR_LIT>', [], {}))<EOL>

Signals the writer that it can stop listening to the queue


@retry(<NUM_LIT:9>, Exception, <NUM_LIT>, '<STR_LIT>')<EOL><INDENT>def _put_on_pipe(self, to_put):<DEDENT>

self.acquire_lock()<EOL>self._send_chunks(to_put)<EOL>self.release_lock()<EOL>


self._put_on_pipe(('<STR_LIT>', args, kwargs))<EOL>


self._put_on_pipe(('<STR_LIT>', [], {}))<EOL>


def _handle_data(self, msg, args, kwargs):

stop = False<EOL>try:<EOL><INDENT>if msg == '<STR_LIT>':<EOL><INDENT>stop = True<EOL><DEDENT>elif msg == '<STR_LIT>':<EOL><INDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>store_msg = kwargs.pop('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>store_msg = args[<NUM_LIT:0>]<EOL>args = args[<NUM_LIT:1>:]<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>stuff_to_store = kwargs.pop('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>stuff_to_store = args[<NUM_LIT:0>]<EOL>args = args[<NUM_LIT:1>:]<EOL><DEDENT>trajectory_name = kwargs['<STR_LIT>']<EOL>if self._trajectory_name != trajectory_name:<EOL><INDENT>if self._storage_service.is_open:<EOL><INDENT>self._close_file()<EOL><DEDENT>self._trajectory_name = trajectory_name<EOL>self._open_file()<EOL><DEDENT>self._storage_service.store(store_msg, stuff_to_store, *args, **kwargs)<EOL>self._storage_service.store(pypetconstants.FLUSH, None)<EOL>self._check_and_collect_garbage()<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % msg)<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>self._logger.exception('<STR_LIT>')<EOL>time.sleep(<NUM_LIT>)<EOL>pass  <EOL><DEDENT>return stop<EOL>

Handles data and returns `True` or `False` if everything is done.


try:<EOL><INDENT>while True:<EOL><INDENT>msg, args, kwargs = self._receive_data()<EOL>stop = self._handle_data(msg, args, kwargs)<EOL>if stop:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>if self._storage_service.is_open:<EOL><INDENT>self._close_file()<EOL><DEDENT>self._trajectory_name = '<STR_LIT>'<EOL><DEDENT>


@retry(<NUM_LIT:9>, Exception, <NUM_LIT>, '<STR_LIT>')<EOL><INDENT>def _receive_data(self):<DEDENT>

result = self.queue.get(block=True)<EOL>if hasattr(self.queue, '<STR_LIT>'):<EOL><INDENT>self.queue.task_done()<EOL><DEDENT>return result<EOL>


while True:<EOL><INDENT>while len(self._buffer) < self.max_size and self.conn.poll():<EOL><INDENT>data = self._read_chunks()<EOL>if data is not None:<EOL><INDENT>self._buffer.append(data)<EOL><DEDENT><DEDENT>if len(self._buffer) > <NUM_LIT:0>:<EOL><INDENT>return self._buffer.popleft()<EOL><DEDENT><DEDENT>


return self._storage_service.is_open<EOL>

Normally the file is opened and closed after each insertion.

        However, the storage service may provide the option to keep the store open and signals
        this via this property.


return True<EOL>

Usually storage services are not supposed to be multiprocessing safe


try:<EOL><INDENT>self.acquire_lock()<EOL>return self._storage_service.store(*args, **kwargs)<EOL><DEDENT>finally:<EOL><INDENT>if self.lock is not None:<EOL><INDENT>try:<EOL><INDENT>self.release_lock()<EOL><DEDENT>except RuntimeError:<EOL><INDENT>self._logger.error('<STR_LIT>' % str(self.lock))<EOL><DEDENT><DEDENT><DEDENT>

Acquires a lock before storage and releases it afterwards.


try:<EOL><INDENT>self.acquire_lock()<EOL>return self._storage_service.load(*args, **kwargs)<EOL><DEDENT>finally:<EOL><INDENT>if self.lock is not None:<EOL><INDENT>try:<EOL><INDENT>self.release_lock()<EOL><DEDENT>except RuntimeError:<EOL><INDENT>self._logger.error('<STR_LIT>' % str(self.lock))<EOL><DEDENT><DEDENT><DEDENT>

Acquires a lock before loading and releases it afterwards.


def store(self, msg, stuff_to_store, *args, **kwargs):

trajectory_name = kwargs['<STR_LIT>']<EOL>if trajectory_name not in self.references:<EOL><INDENT>self.references[trajectory_name] = []<EOL><DEDENT>self.references[trajectory_name].append((msg, cp.copy(stuff_to_store), args, kwargs))<EOL>

Simply keeps a reference to the stored data


raise NotImplementedError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>


for trajectory_name in references:<EOL><INDENT>self._storage_service.store(pypetconstants.LIST, references[trajectory_name], trajectory_name=trajectory_name)<EOL><DEDENT>self._check_and_collect_garbage()<EOL>

Stores references to disk and may collect garbage.


return '<STR_LIT>' in sys.modules<EOL>

Checks if user is currently debugging.

    Debugging is checked via ``'pydevd' in sys.modules``.

    :return: True of False


flat_dict = {}<EOL>for key, val in nested_dict.items():<EOL><INDENT>if isinstance(val, dict):<EOL><INDENT>new_flat_dict = flatten_dictionary(val, separator)<EOL>for flat_key, inval in new_flat_dict.items():<EOL><INDENT>new_key = key + separator + flat_key<EOL>flat_dict[new_key] = inval<EOL><DEDENT><DEDENT>else:<EOL><INDENT>flat_dict[key] = val<EOL><DEDENT><DEDENT>return flat_dict<EOL>

Flattens a nested dictionary.

    New keys are concatenations of nested keys with the `separator` in between.


nested_dict = {}<EOL>for key, val in flat_dict.items():<EOL><INDENT>split_key = key.split(separator)<EOL>act_dict = nested_dict<EOL>final_key = split_key.pop()<EOL>for new_key in split_key:<EOL><INDENT>if not new_key in act_dict:<EOL><INDENT>act_dict[new_key] = {}<EOL><DEDENT>act_dict = act_dict[new_key]<EOL><DEDENT>act_dict[final_key] = val<EOL><DEDENT>return nested_dict<EOL>

Nests a given flat dictionary.

    Nested keys are created by splitting given keys around the `separator`.


def progressbar(index, total, percentage_step=<NUM_LIT:10>, logger='<STR_LIT>', log_level=logging.INFO,<EOL>reprint=True, time=True, length=<NUM_LIT:20>, fmt_string=None, reset=False):

return _progressbar(index=index, total=total, percentage_step=percentage_step,<EOL>logger=logger, log_level=log_level, reprint=reprint,<EOL>time=time, length=length, fmt_string=fmt_string, reset=reset)<EOL>

Plots a progress bar to the given `logger` for large for loops.

    To be used inside a for-loop at the end of the loop:

    .. code-block:: python

        for irun in range(42):
            my_costly_job() # Your expensive function
            progressbar(index=irun, total=42, reprint=True) # shows a growing progressbar

    There is no initialisation of the progressbar necessary before the for-loop.
    The progressbar will be reset automatically if used in another for-loop.

    :param index: Current index of for-loop
    :param total: Total size of for-loop
    :param percentage_step: Steps with which the bar should be plotted
    :param logger:

        Logger to write to - with level INFO. If string 'print' is given, the print statement is
        used. Use ``None`` if you don't want to print or log the progressbar statement.

    :param log_level: Log level with which to log.
    :param reprint:

        If no new line should be plotted but carriage return (works only for printing)

    :param time: If the remaining time should be estimated and displayed
    :param length: Length of the bar in `=` signs.
    :param fmt_string:

        A string which contains exactly one `%s` in order to incorporate the progressbar.
        If such a string is given, ``fmt_string % progressbar`` is printed/logged.

    :param reset:

        If the progressbar should be restarted. If progressbar is called with a lower
        index than the one before, the progressbar is automatically restarted.

    :return:

        The progressbar string or `None` if the string has not been updated.


if inspect.isclass(func):<EOL><INDENT>func = func.__init__<EOL><DEDENT>if not inspect.isfunction(func):<EOL><INDENT>return [], False<EOL><DEDENT>parameters = inspect.signature(func).parameters<EOL>args = []<EOL>uses_starstar = False<EOL>for par in parameters.values():<EOL><INDENT>if (par.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD or<EOL>par.kind == inspect.Parameter.KEYWORD_ONLY):<EOL><INDENT>args.append(par.name)<EOL><DEDENT>elif par.kind == inspect.Parameter.VAR_KEYWORD:<EOL><INDENT>uses_starstar = True<EOL><DEDENT><DEDENT>return args, uses_starstar<EOL>

Helper function to support both Python versions


args, uses_startstar = _get_argspec(func)<EOL>if uses_startstar:<EOL><INDENT>return kwargs.copy()<EOL><DEDENT>else:<EOL><INDENT>matching_kwargs = dict((k, kwargs[k]) for k in args if k in kwargs)<EOL>return matching_kwargs<EOL><DEDENT>

Takes a function and keyword arguments and returns the ones that can be passed.


if len(result_list) < <NUM_LIT:2>:<EOL><INDENT>return result_list<EOL><DEDENT>to_sort = result_list[start_index:]<EOL>minmax = [x[<NUM_LIT:0>] for x in to_sort]<EOL>minimum = min(minmax)<EOL>maximum = max(minmax)<EOL>sorted_list = [None for _ in range(minimum, maximum + <NUM_LIT:1>)]<EOL>for elem in to_sort:<EOL><INDENT>key = elem[<NUM_LIT:0>] - minimum<EOL>sorted_list[key] = elem<EOL><DEDENT>idx_count = start_index<EOL>for elem in sorted_list:<EOL><INDENT>if elem is not None:<EOL><INDENT>result_list[idx_count] = elem<EOL>idx_count += <NUM_LIT:1><EOL><DEDENT><DEDENT>return result_list<EOL>

Sorts a list of results in O(n) in place (since every run is unique)

    :param result_list: List of tuples [(run_idx, res), ...]
    :param start_index: Index with which to start, every entry before `start_index` is ignored


format_string = '<STR_LIT>'<EOL>formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime(format_string)<EOL>return formatted_time<EOL>

Formats timestamp to human readable format


<EOL>domain_name = socket.getfqdn()<EOL>try:<EOL><INDENT>addr_list = socket.getaddrinfo(domain_name, None)<EOL><DEDENT>except Exception:<EOL><INDENT>addr_list = socket.getaddrinfo('<STR_LIT:127.0.0.1>', None)<EOL><DEDENT>family, socktype, proto, canonname, sockaddr = addr_list[<NUM_LIT:0>]<EOL>host = convert_ipv6(sockaddr[<NUM_LIT:0>])<EOL>address =  '<STR_LIT>' + host<EOL>if port is None:<EOL><INDENT>port = ()<EOL><DEDENT>if not isinstance(port, int):<EOL><INDENT>context = zmq.Context()<EOL>try:<EOL><INDENT>socket_ = context.socket(zmq.REP)<EOL>socket_.ipv6 = is_ipv6(address)<EOL>port = socket_.bind_to_random_port(address, *port)<EOL><DEDENT>except Exception:<EOL><INDENT>print('<STR_LIT>'.format(address, addr_list))<EOL>pypet_root_logger = logging.getLogger('<STR_LIT>')<EOL>pypet_root_logger.exception('<STR_LIT>'.format(address))<EOL>raise<EOL><DEDENT>socket_.close()<EOL>context.term()<EOL><DEDENT>return address + '<STR_LIT::>' + str(port)<EOL>

Returns local tcp address for a given `port`, automatic port if `None`


if os.path.isfile(path):<EOL><INDENT>raise IOError('<STR_LIT>')<EOL><DEDENT>while True:<EOL><INDENT>try:<EOL><INDENT>if os.path.isdir(path):<EOL><INDENT>break<EOL><DEDENT>os.makedirs(path)<EOL><DEDENT>except EnvironmentError as exc:<EOL><INDENT>if exc.errno != <NUM_LIT>:<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>

Like os.makedirs but takes care about race conditions


def _reset(self, index, total, percentage_step, length):

self._start_time = datetime.datetime.now()<EOL>self._start_index = index<EOL>self._current_index = index<EOL>self._percentage_step = percentage_step<EOL>self._total = float(total)<EOL>self._total_minus_one = total - <NUM_LIT:1><EOL>self._length = length<EOL>self._norm_factor = total * percentage_step / <NUM_LIT><EOL>self._current_interval = int((index + <NUM_LIT:1.0>) / self._norm_factor)<EOL>

Resets to the progressbar to start a new one


try:<EOL><INDENT>current_time = datetime.datetime.now()<EOL>time_delta = current_time - self._start_time<EOL>try:<EOL><INDENT>total_seconds = time_delta.total_seconds()<EOL><DEDENT>except AttributeError:<EOL><INDENT>total_seconds = ((time_delta.microseconds +<EOL>(time_delta.seconds +<EOL>time_delta.days * <NUM_LIT> * <NUM_LIT>) * <NUM_LIT:10> ** <NUM_LIT:6>) / <NUM_LIT> ** <NUM_LIT:6>)<EOL><DEDENT>remaining_seconds = int((self._total - self._start_index - <NUM_LIT:1.0>) *<EOL>total_seconds / float(index - self._start_index) -<EOL>total_seconds)<EOL>remaining_delta = datetime.timedelta(seconds=remaining_seconds)<EOL>remaining_str = '<STR_LIT>' + str(remaining_delta)<EOL><DEDENT>except ZeroDivisionError:<EOL><INDENT>remaining_str = '<STR_LIT>'<EOL><DEDENT>return remaining_str<EOL>


def __call__(self, index, total, percentage_step=<NUM_LIT:5>, logger='<STR_LIT>', log_level=logging.INFO,<EOL>reprint=False, time=True, length=<NUM_LIT:20>, fmt_string=None,  reset=False):

reset = (reset or<EOL>index <= self._current_index or<EOL>total != self._total)<EOL>if reset:<EOL><INDENT>self._reset(index, total, percentage_step, length)<EOL><DEDENT>statement = None<EOL>indexp1 = index + <NUM_LIT:1.0><EOL>next_interval = int(indexp1 / self._norm_factor)<EOL>ending = index >= self._total_minus_one<EOL>if next_interval > self._current_interval or ending or reset:<EOL><INDENT>if time:<EOL><INDENT>remaining_str = self._get_remaining(index)<EOL><DEDENT>else:<EOL><INDENT>remaining_str = '<STR_LIT>'<EOL><DEDENT>if ending:<EOL><INDENT>statement = '<STR_LIT:[>' + '<STR_LIT:=>' * self._length +'<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>bars = int((indexp1 / self._total) * self._length)<EOL>spaces = self._length - bars<EOL>percentage = indexp1 / self._total * <NUM_LIT><EOL>if reset:<EOL><INDENT>statement = ('<STR_LIT:[>' + '<STR_LIT:=>' * bars +<EOL>'<STR_LIT:U+0020>' * spaces + '<STR_LIT:]>' + '<STR_LIT>' % percentage + '<STR_LIT:%>')<EOL><DEDENT>else:<EOL><INDENT>statement = ('<STR_LIT:[>' + '<STR_LIT:=>' * bars +<EOL>'<STR_LIT:U+0020>' * spaces + '<STR_LIT:]>' + '<STR_LIT>' % percentage + '<STR_LIT:%>' +<EOL>remaining_str)<EOL><DEDENT><DEDENT>if fmt_string:<EOL><INDENT>statement = fmt_string % statement<EOL><DEDENT>if logger == '<STR_LIT>':<EOL><INDENT>if reprint:<EOL><INDENT>print('<STR_LIT:\r>' + statement, end='<STR_LIT>', flush=True)<EOL><DEDENT>else:<EOL><INDENT>print(statement)<EOL><DEDENT><DEDENT>elif logger is not None:<EOL><INDENT>if isinstance(logger, str):<EOL><INDENT>logger = logging.getLogger(logger)<EOL><DEDENT>logger.log(msg=statement, level=log_level)<EOL><DEDENT><DEDENT>self._current_interval = next_interval<EOL>self._current_index = index<EOL>return statement<EOL>

Plots a progress bar to the given `logger` for large for loops.

        To be used inside a for-loop at the end of the loop.

        :param index: Current index of for-loop
        :param total: Total size of for-loop
        :param percentage_step: Percentage step with which the bar should be updated
        :param logger:

            Logger to write to, if string 'print' is given, the print statement is
            used. Use None if you don't want to print or log the progressbar statement.

        :param log_level: Log level with which to log.
        :param reprint:

            If no new line should be plotted but carriage return (works only for printing)

        :param time: If the remaining time should be calculated and displayed
        :param length: Length of the bar in `=` signs.
        :param fmt_string:

            A string which contains exactly one `%s` in order to incorporate the progressbar.
            If such a string is given, ``fmt_string % progressbar`` is printed/logged.

        :param reset:

            If the progressbar should be restarted. If progressbar is called with a lower
            index than the one before, the progressbar is automatically restarted.

        :return:

            The progressbar string or None if the string has not been updated.


in_dir = os.listdir(folder)<EOL>all_files = []<EOL>for file in in_dir:<EOL><INDENT>full_file = os.path.join(folder, file)<EOL>if os.path.isfile(full_file):<EOL><INDENT>_, extension = os.path.splitext(full_file)<EOL>if extension == ext:<EOL><INDENT>all_files.append(full_file)<EOL><DEDENT><DEDENT><DEDENT>all_files = sorted(all_files)<EOL>trajs = []<EOL>for full_file in all_files:<EOL><INDENT>traj = load_trajectory(index=-<NUM_LIT:1>,<EOL>storage_service=storage_service,<EOL>filename=full_file,<EOL>load_data=<NUM_LIT:0>,<EOL>force=force,<EOL>dynamic_imports=dynamic_imports)<EOL>trajs.append(traj)<EOL><DEDENT>first_traj = trajs.pop(<NUM_LIT:0>)<EOL>first_traj.f_merge_many(trajs,<EOL>ignore_data=ignore_data,<EOL>move_data=move_data,<EOL>delete_other_trajectory=False,<EOL>keep_info=keep_info,<EOL>keep_other_trajectory_info=keep_other_trajectory_info,<EOL>merge_config=merge_config,<EOL>backup=backup)<EOL>if delete_other_files:<EOL><INDENT>for file in all_files[<NUM_LIT:1>:]:<EOL><INDENT>os.remove(file)<EOL><DEDENT><DEDENT>return first_traj<EOL>

Merges all files in a given folder.

    IMPORTANT: Does not check if there are more than 1 trajectory in a file. Always
    uses the last trajectory in file and ignores the other ones.

    Trajectories are merged according to the alphabetical order of the files,
    i.e. the resulting merged trajectory is found in the first file
    (according to lexicographic ordering).

    :param folder: folder (not recursive) where to look for files
    :param ext: only files with the given extension are used
    :param dynamic_imports: Dynamic imports for loading
    :param storage_service: storage service to use, leave `None` to use the default one
    :param force: If loading should be forced.
    :param delete_other_files: Deletes files of merged trajectories

    All other parameters as in `f_merge_many` of the trajectory.

    :return: The merged traj


git_time_value = time.strftime('<STR_LIT>', time.localtime(commit.committed_date))<EOL>git_short_name = str(commit.hexsha[<NUM_LIT:0>:<NUM_LIT:7>])<EOL>git_commit_name = '<STR_LIT>' % git_short_name<EOL>git_commit_name = '<STR_LIT>' + git_commit_name + git_time_value<EOL>if not traj.f_contains('<STR_LIT>'+git_commit_name, shortcuts=False):<EOL><INDENT>git_commit_name += '<STR_LIT:.>'<EOL>traj.f_add_config(git_commit_name+'<STR_LIT>', commit.hexsha,<EOL>comment='<STR_LIT>')<EOL>traj.f_add_config(git_commit_name+'<STR_LIT>', commit.name_rev,<EOL>comment='<STR_LIT>'<EOL>'<STR_LIT>')<EOL>traj.f_add_config(git_commit_name+'<STR_LIT>',<EOL>commit.committed_date, comment='<STR_LIT>')<EOL>traj.f_add_config(git_commit_name+'<STR_LIT:message>', str(commit.message),<EOL>comment='<STR_LIT>')<EOL><DEDENT>

Adds commit information to the trajectory.