return self._get_param(param="<STR_LIT>", allowed_values=allowed_request_types[self.params['<STR_LIT>']])<EOL>

Find requested request type in GET request.


version = self._get_param(param="<STR_LIT:version>", allowed_values=allowed_versions[self.params['<STR_LIT>']],<EOL>optional=True)<EOL>if version is None and self._get_request_type() != "<STR_LIT>":<EOL><INDENT>raise OWSMissingParameterValue('<STR_LIT>', value="<STR_LIT:version>")<EOL><DEDENT>else:<EOL><INDENT>return version<EOL><DEDENT>


if "<STR_LIT>" in self.document.attrib:<EOL><INDENT>value = self.document.attrib["<STR_LIT>"].lower()<EOL>if value in allowed_service_types:<EOL><INDENT>self.params["<STR_LIT>"] = value<EOL><DEDENT>else:<EOL><INDENT>raise OWSInvalidParameterValue("<STR_LIT>" % value, value="<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise OWSMissingParameterValue('<STR_LIT>', value="<STR_LIT>")<EOL><DEDENT>return self.params["<STR_LIT>"]<EOL>

Check mandatory service name parameter in POST request.


value = self.document.tag.lower()<EOL>if value in allowed_request_types[self.params['<STR_LIT>']]:<EOL><INDENT>self.params["<STR_LIT>"] = value<EOL><DEDENT>else:<EOL><INDENT>raise OWSInvalidParameterValue("<STR_LIT>" % value, value="<STR_LIT>")<EOL><DEDENT>return self.params["<STR_LIT>"]<EOL>

Find requested request type in POST request.


if "<STR_LIT:version>" in self.document.attrib:<EOL><INDENT>value = self.document.attrib["<STR_LIT:version>"].lower()<EOL>if value in allowed_versions[self.params['<STR_LIT>']]:<EOL><INDENT>self.params["<STR_LIT:version>"] = value<EOL><DEDENT>else:<EOL><INDENT>raise OWSInvalidParameterValue("<STR_LIT>" % value, value="<STR_LIT:version>")<EOL><DEDENT><DEDENT>elif self._get_request_type() == "<STR_LIT>":<EOL><INDENT>self.params["<STR_LIT:version>"] = None<EOL><DEDENT>else:<EOL><INDENT>raise OWSMissingParameterValue('<STR_LIT>', value="<STR_LIT:version>")<EOL><DEDENT>return self.params["<STR_LIT:version>"]<EOL>


name = "<STR_LIT>" % (left[random.randint(<NUM_LIT:0>, len(left) - <NUM_LIT:1>)], right[random.randint(<NUM_LIT:0>, len(right) - <NUM_LIT:1>)])<EOL>if retry is True:<EOL><INDENT>name = "<STR_LIT>" % (name, random.randint(<NUM_LIT:0>, <NUM_LIT:100>))<EOL><DEDENT>return name<EOL>

generates a random name from the list of adjectives and birds in this package
formatted as "adjective_surname". For example 'loving_sugarbird'. If retry is non-zero, a random
integer between 0 and 100 will be added to the end of the name, e.g `loving_sugarbird3`


return [os.path.join(dp, f)<EOL>for dp, _, fn in os.walk(self.dir)<EOL>for f in fn]<EOL>


self.current += <NUM_LIT:1><EOL>if self.current == self.factor:<EOL><INDENT>sys.stdout.write('<STR_LIT:+>')<EOL>sys.stdout.flush()<EOL>self.current = <NUM_LIT:0><EOL><DEDENT>


def wrap(cls):<EOL><INDENT>warnings.warn(UserWarning('<STR_LIT>'))<EOL>these = {<EOL>name: attr.ib()<EOL>for name in inspect(cls).columns.keys()<EOL>}<EOL>return attr.s(cls, these=these, init=False)<EOL><DEDENT>if maybe_cls is None:<EOL><INDENT>return wrap<EOL><DEDENT>else:<EOL><INDENT>return wrap(maybe_cls)<EOL><DEDENT>

A class decorator that adds ``__repr__``, ``__eq__``, ``__cmp__``, and
``__hash__`` methods according to the fields defined on the SQLAlchemy
model class.


with codecs.open(os.path.join(HERE, *parts), '<STR_LIT:rb>', '<STR_LIT:utf-8>') as f:<EOL><INDENT>return f.read()<EOL><DEDENT>

Build an absolute path from *parts* and and return the contents of the
resulting file.  Assume UTF-8 encoding.


meta_match = re.search(<EOL>r"<STR_LIT>".format(meta=meta),<EOL>META_FILE, re.M<EOL>)<EOL>if meta_match:<EOL><INDENT>return meta_match.group(<NUM_LIT:1>)<EOL><DEDENT>raise RuntimeError('<STR_LIT>'.format(meta=meta))<EOL>


if not exists(folder_path):<EOL><INDENT>makedirs(folder_path)<EOL><DEDENT>return folder_path<EOL>

Check whether a folder exists, if not the folder is created.
    Always return folder_path.


logger.debug('<STR_LIT>', url)<EOL>remote_file_size = self.get_remote_file_size(url)<EOL>if exists(join(path, filename)):<EOL><INDENT>size = getsize(join(path, filename))<EOL>if size == remote_file_size:<EOL><INDENT>logger.error('<STR_LIT>' % filename)<EOL>print('<STR_LIT>' % filename)<EOL>return [join(path, filename), size]<EOL><DEDENT><DEDENT>logger.debug('<STR_LIT>' % filename)<EOL>print('<STR_LIT>' % filename)<EOL>fetch(url, path)<EOL>print('<STR_LIT>' % path)<EOL>logger.debug('<STR_LIT>' % path)<EOL>return [join(path, filename), remote_file_size]<EOL>

Verify if the file is already downloaded and complete. If they don't
        exists or if are not complete, use homura download function to fetch
        files. Return a list with the path of the downloaded file and the size
        of the remote file.


return requests.head(url).status_code == <NUM_LIT:200><EOL>

Check whether the remote file exists on Storage


headers = requests.head(url).headers<EOL>return int(headers['<STR_LIT>'])<EOL>


if not isinstance(bands, list):<EOL><INDENT>logger.error('<STR_LIT>')<EOL>raise TypeError('<STR_LIT>')<EOL><DEDENT>valid_bands = list(range(<NUM_LIT:1>, <NUM_LIT:12>)) + ['<STR_LIT>']<EOL>for band in bands:<EOL><INDENT>if band not in valid_bands:<EOL><INDENT>logger.error('<STR_LIT>' % band)<EOL>raise InvalidBandError('<STR_LIT>' % band)<EOL><DEDENT><DEDENT>


if self.sceneInfo.prefix not in self.__satellitesMap:<EOL><INDENT>logger.error('<STR_LIT>'<EOL>% (self.sceneInfo.name, self.sceneInfo.prefix))<EOL>raise WrongSceneNameError('<STR_LIT>'<EOL>% (self.sceneInfo.name, self.sceneInfo.prefix))<EOL><DEDENT>

Check scene name and whether remote file exists. Raises
        WrongSceneNameError if the scene name is wrong.


return super(GoogleDownloader, self).remote_file_exists(self.remote_file_url)<EOL>

Verify if the remote file exists. Returns True or False.


def download(self, bands, download_dir=None, metadata=False):

super(GoogleDownloader, self).validate_bands(bands)<EOL>pattern = re.compile('<STR_LIT>', re.I)<EOL>image_list = []<EOL>band_list = ['<STR_LIT>' % (i,) if isinstance(i, int) else i for i in bands]<EOL>if download_dir is None:<EOL><INDENT>download_dir = DOWNLOAD_DIR<EOL><DEDENT>check_create_folder(join(download_dir, self.sceneInfo.name))<EOL>filename = "<STR_LIT>" % (self.sceneInfo.name, self.__remote_file_ext)<EOL>downloaded = self.fetch(self.remote_file_url, download_dir, filename)<EOL>try:<EOL><INDENT>tar = tarfile.open(downloaded[<NUM_LIT:0>], '<STR_LIT:r>')<EOL>folder_path = join(download_dir, self.sceneInfo.name)<EOL>logger.debug('<STR_LIT>', folder_path)<EOL>tar.extractall(folder_path)<EOL>remove(downloaded[<NUM_LIT:0>])<EOL>images_path = listdir(folder_path)<EOL>for image_path in images_path:<EOL><INDENT>matched = pattern.match(image_path)<EOL>file_path = join(folder_path, image_path)<EOL>if matched and matched.group(<NUM_LIT:1>) in band_list:<EOL><INDENT>image_list.append([file_path, getsize(file_path)])<EOL><DEDENT>elif matched:<EOL><INDENT>remove(file_path)<EOL><DEDENT><DEDENT><DEDENT>except tarfile.ReadError as error:<EOL><INDENT>logger.error('<STR_LIT>', error)<EOL>print('<STR_LIT>')<EOL><DEDENT>return image_list<EOL>


if self.sceneInfo.prefix not in self.__prefixesValid:<EOL><INDENT>raise WrongSceneNameError('<STR_LIT>'<EOL>% (self.sceneInfo.name, self.sceneInfo.prefix))<EOL><DEDENT>

Check whether sceneInfo is valid to download from AWS Storage.


url = join(self.base_url, '<STR_LIT>')<EOL>return super(AWSDownloader, self).remote_file_exists(url)<EOL>

Verify whether the file (scene) exists on AWS Storage.


super(AWSDownloader, self).validate_bands(bands)<EOL>if download_dir is None:<EOL><INDENT>download_dir = DOWNLOAD_DIR<EOL><DEDENT>dest_dir = check_create_folder(join(download_dir, self.sceneInfo.name))<EOL>downloaded = []<EOL>for band in bands:<EOL><INDENT>if band == '<STR_LIT>':<EOL><INDENT>filename = '<STR_LIT>' % (self.sceneInfo.name, band, self.__remote_file_ext)<EOL><DEDENT>else:<EOL><INDENT>filename = '<STR_LIT>' % (self.sceneInfo.name, band, self.__remote_file_ext)<EOL><DEDENT>band_url = join(self.base_url, filename)<EOL>downloaded.append(self.fetch(band_url, dest_dir, filename))<EOL><DEDENT>if metadata:<EOL><INDENT>filename = '<STR_LIT>' % (self.sceneInfo.name)<EOL>url = join(self.base_url, filename)<EOL>self.fetch(url, dest_dir, filename)<EOL><DEDENT>return downloaded<EOL>

Download each specified band and metadata.


<EOL>ml_graph, cl_graph = {}, {}<EOL>for i in range(n):<EOL><INDENT>ml_graph[i] = set()<EOL>cl_graph[i] = set()<EOL><DEDENT>def add_both(d, i, j):<EOL><INDENT>d[i].add(j)<EOL>d[j].add(i)<EOL><DEDENT>for (i, j) in ml:<EOL><INDENT>ml_graph[i].add(j)<EOL>ml_graph[j].add(i)<EOL><DEDENT>for (i, j) in cl:<EOL><INDENT>cl_graph[i].add(j)<EOL>cl_graph[j].add(i)<EOL><DEDENT>def dfs(i, graph, visited, component):<EOL><INDENT>visited[i] = True<EOL>for j in graph[i]:<EOL><INDENT>if not visited[j]:<EOL><INDENT>dfs(j, graph, visited, component)<EOL><DEDENT><DEDENT>component.append(i)<EOL><DEDENT>visited = [False] * n<EOL>neighborhoods = []<EOL>for i in range(n):<EOL><INDENT>if not visited[i] and ml_graph[i]:<EOL><INDENT>component = []<EOL>dfs(i, ml_graph, visited, component)<EOL>for x1 in component:<EOL><INDENT>for x2 in component:<EOL><INDENT>if x1 != x2:<EOL><INDENT>ml_graph[x1].add(x2)<EOL><DEDENT><DEDENT><DEDENT>neighborhoods.append(component)<EOL><DEDENT><DEDENT>for (i, j) in cl:<EOL><INDENT>for x in ml_graph[i]:<EOL><INDENT>add_both(cl_graph, x, j)<EOL><DEDENT>for y in ml_graph[j]:<EOL><INDENT>add_both(cl_graph, i, y)<EOL><DEDENT>for x in ml_graph[i]:<EOL><INDENT>for y in ml_graph[j]:<EOL><INDENT>add_both(cl_graph, x, y)<EOL><DEDENT><DEDENT><DEDENT>for i in ml_graph:<EOL><INDENT>for j in ml_graph[i]:<EOL><INDENT>if j != i and j in cl_graph[i]:<EOL><INDENT>raise InconsistentConstraintsException('<STR_LIT>'.format(i, j))<EOL><DEDENT><DEDENT><DEDENT>return ml_graph, cl_graph, neighborhoods<EOL>

Create a graph of constraints for both must- and cannot-links


return scipy.spatial.distance.mahalanobis(x, y, A) ** <NUM_LIT:2><EOL>


if self.queries_cnt < self.max_queries_cnt:<EOL><INDENT>self.queries_cnt += <NUM_LIT:1><EOL>return self.labels[i] == self.labels[j]<EOL><DEDENT>else:<EOL><INDENT>raise MaximumQueriesExceeded<EOL><DEDENT>

Query the oracle to find out whether i and j should be must-linked


server = get_server_from_specifier(db_spec)<EOL>db = get_db_from_specifier(db_spec)<EOL>logger = logging.getLogger('<STR_LIT>')<EOL>logger.info('<STR_LIT>' % (db_spec,<EOL>repr_bytes(db.info()['<STR_LIT>']),))<EOL>logger.debug('<STR_LIT>' + urlparse.urljoin(db.resource.uri + '<STR_LIT:/>', '<STR_LIT>'))<EOL>resp_headers, resp_body = db.resource.post('<STR_LIT>')<EOL>if not poll_interval:<EOL><INDENT>if not (resp_body.get('<STR_LIT>', False) and<EOL>resp_headers['<STR_LIT:status>'] == '<STR_LIT>'):<EOL><INDENT>err = CompactionError('<STR_LIT>')<EOL>err.response = (resp_headers, resp_body)<EOL>raise err<EOL><DEDENT>def check_completed():<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>')<EOL>logger.debug('<STR_LIT>' + db.resource.uri + '<STR_LIT:/>')<EOL>db_info = db.info()<EOL>completed = not db_info.get('<STR_LIT>', False)<EOL>if completed and db_info.get('<STR_LIT>', None):<EOL><INDENT>logger.info('<STR_LIT>' % (db_spec,<EOL>repr_bytes(db_info['<STR_LIT>'])))<EOL><DEDENT>return completed<EOL><DEDENT>return check_completed<EOL><DEDENT>elif poll_interval > <NUM_LIT:0>:<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>')<EOL>logger.debug('<STR_LIT>' + db.resource.uri + '<STR_LIT:/>')<EOL>running = db.info().get('<STR_LIT>', False)<EOL>while running:<EOL><INDENT>time.sleep(poll_interval)<EOL>logger.debug(<EOL>'<STR_LIT>')<EOL>logger.debug('<STR_LIT>' + db.resource.uri + '<STR_LIT:/>')<EOL>running = db.info().get('<STR_LIT>', False)<EOL><DEDENT>size_after = db.info().get('<STR_LIT>', None)<EOL>if size_after:<EOL><INDENT>logger.info('<STR_LIT>' % (db_spec,<EOL>repr_bytes(size_after)))<EOL><DEDENT>return True<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>

Compact a CouchDB database with optional synchronicity.

The ``compact`` function will compact a CouchDB database stored on an
running CouchDB server. By default, this process occurs *asynchronously*,
meaning that the compaction will occur in the background. Often, you'll want
to know when the process has completed; for this reason, ``compact`` will
return a function which, when called, will return the state of the
compaction. If it has completed, ``True`` will be returned; otherwise,
``False``. This may be called multiple times.

Alternatively, you may opt to run ``compact`` in synchronous mode, for
debugging or profiling purposes. If this is the case, an optional keyword
argument ``poll_interval`` is accepted, which should be a number (in
seconds) representing the time to take between polls. A sensible default
may be around 0.5 (seconds).

Because this function operates on database specifiers, you can choose to
operate on the local server or any remote server.


local_match = LOCAL_RE.match(db_spec)<EOL>remote_match = REMOTE_RE.match(db_spec)<EOL>plain_match = PLAIN_RE.match(db_spec)<EOL>if local_match:<EOL><INDENT>return local_match.groupdict()['<STR_LIT>']<EOL><DEDENT>elif remote_match:<EOL><INDENT>hostname, portnum, database = map(remote_match.groupdict().get,<EOL>('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'))<EOL>local_url = settings._('<STR_LIT>', '<STR_LIT>')<EOL>localhost, localport = urlparse.urlparse(local_url)[<NUM_LIT:1>].split('<STR_LIT::>')<EOL>if (localhost == hostname) and (localport == portnum):<EOL><INDENT>return database<EOL><DEDENT>return '<STR_LIT>' % (hostname, portnum, database)<EOL><DEDENT>elif plain_match:<EOL><INDENT>return plain_match.groupdict()['<STR_LIT>']<EOL><DEDENT>raise ValueError('<STR_LIT>' % (db_spec,))<EOL>

Return the database string for a database specifier.

The database specifier takes a custom format for specifying local and remote
databases. A local database is specified by the following format:

    local:<db_name>

For example, a database called 'sessions' would be specified by the string
``'local:sessions'``. Remote databases are specified like this:

    remote:<host>:<port_num>:<db_name>

For example, a database called 'log' on the server 'dev.example.com' at port
number 5984 would be specified by ``'remote:dev.example.com:5984:log'``.

These specifiers are translated into strings acceptable to CouchDB; local
specs are turned into the database name alone, and remote specs are turned
into ``'http://host:port/db_name'`` URLs.


local_match = PLAIN_RE.match(db_string)<EOL>remote_match = URL_RE.match(db_string)<EOL>if local_match:<EOL><INDENT>return '<STR_LIT>' + local_match.groupdict()['<STR_LIT>']<EOL><DEDENT>elif remote_match:<EOL><INDENT>hostname, portnum, database = map(remote_match.groupdict().get,<EOL>('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'))<EOL>local_url = settings._('<STR_LIT>', '<STR_LIT>')<EOL>localhost, localport = urlparse.urlparse(local_url)[<NUM_LIT:1>].split('<STR_LIT::>')<EOL>if (localhost == hostname) and (localport == portnum):<EOL><INDENT>return '<STR_LIT>' + database<EOL><DEDENT>return '<STR_LIT>' % (hostname, portnum, database)<EOL><DEDENT>raise ValueError('<STR_LIT>' % (db_string,))<EOL>

Return the database specifier for a database string.

This accepts a database name or URL, and returns a database specifier in the
format accepted by ``specifier_to_db``. It is recommended that you consult
the documentation for that function for an explanation of the format.


local_match = PLAIN_RE.match(db_string)<EOL>remote_match = URL_RE.match(db_string)<EOL>if local_match:<EOL><INDENT>return shortcuts.get_server()<EOL><DEDENT>elif remote_match:<EOL><INDENT>hostname, portnum, database = map(remote_match.groupdict().get,<EOL>('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'))<EOL>local_url = settings._('<STR_LIT>', '<STR_LIT>')<EOL>localhost, localport = urlparse.urlparse(local_url)[<NUM_LIT:1>].split('<STR_LIT::>')<EOL>if (localhost == hostname) and (localport == portnum):<EOL><INDENT>return shortcuts.get_server()<EOL><DEDENT>return shortcuts.get_server(<EOL>server_url=('<STR_LIT>' % (hostname, portnum)))<EOL><DEDENT>raise ValueError('<STR_LIT>' % (db_string,))<EOL>

Return a CouchDB server instance from a database string.


return get_server_from_db(specifier_to_db(db_spec))<EOL>

Return a CouchDB server instance from a database specifier.


server = get_server_from_db(db_string)<EOL>local_match = PLAIN_RE.match(db_string)<EOL>remote_match = URL_RE.match(db_string)<EOL>if local_match:<EOL><INDENT>return server[local_match.groupdict()['<STR_LIT>']]<EOL><DEDENT>elif remote_match:<EOL><INDENT>return server[remote_match.groupdict()['<STR_LIT>']]<EOL><DEDENT>raise ValueError('<STR_LIT>' % (db_string,))<EOL>

Return a CouchDB database instance from a database string.


return get_db_from_db(specifier_to_db(db_spec))<EOL>

Return a CouchDB database instance from a database specifier.


local_match = LOCAL_RE.match(db_spec)<EOL>remote_match = REMOTE_RE.match(db_spec)<EOL>plain_match = PLAIN_RE.match(db_spec)<EOL>if local_match:<EOL><INDENT>db_name = local_match.groupdict().get('<STR_LIT>')<EOL>server = shortcuts.get_server()<EOL>if db_name not in server:<EOL><INDENT>server.create(db_name)<EOL><DEDENT>return True<EOL><DEDENT>elif remote_match:<EOL><INDENT>hostname, portnum, database = map(remote_match.groupdict().get,<EOL>('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'))<EOL>server = shortcuts.get_server(<EOL>server_url=('<STR_LIT>' % (hostname, portnum)))<EOL>if database not in server:<EOL><INDENT>server.create(database)<EOL><DEDENT>return True<EOL><DEDENT>elif plain_match:<EOL><INDENT>db_name = plain_match.groupdict().get('<STR_LIT>')<EOL>server = shortcuts.get_server()<EOL>if db_name not in server:<EOL><INDENT>server.create(db_name)<EOL><DEDENT>return True<EOL><DEDENT>return False<EOL>

Make sure a DB specifier exists, creating it if necessary.


return ensure_specifier_exists(db_to_specifier(db_string))<EOL>

Make sure a DB string exists, creating it if necessary.


<EOL>server = shortcuts.get_server()<EOL>logger = logging.getLogger('<STR_LIT>')<EOL>logger.debug('<STR_LIT>' + urllib.parse.urljoin(server.resource.uri, '<STR_LIT>'))<EOL>source, target = specifier_to_db(source_db), specifier_to_db(target_db)<EOL>logger.debug('<STR_LIT>' % (source,))<EOL>logger.debug('<STR_LIT>' % (target,))<EOL>try:<EOL><INDENT>resp_headers, resp_body = server.resource.post(path='<STR_LIT>',<EOL>content=json.dumps({'<STR_LIT:source>': source, '<STR_LIT:target>': target}))<EOL><DEDENT>except couchdb.client.ServerError as exc:<EOL><INDENT>logger.error('<STR_LIT>')<EOL>raise ReplicationError(exc.args)<EOL><DEDENT>result = resp_body['<STR_LIT>'][<NUM_LIT:0>]<EOL>if resp_body['<STR_LIT>']:<EOL><INDENT>logger.info('<STR_LIT>' % (<EOL>resp_body['<STR_LIT>'][:<NUM_LIT:6>],))<EOL>logger.info('<STR_LIT>' + result['<STR_LIT>'])<EOL>logger.info('<STR_LIT>' + result['<STR_LIT>'])<EOL>result['<STR_LIT>'] = datetime.datetime.strptime(result['<STR_LIT>'],<EOL>'<STR_LIT>')<EOL>result['<STR_LIT>'] = datetime.datetime.strptime(result['<STR_LIT>'],<EOL>'<STR_LIT>')<EOL>timedelta = result['<STR_LIT>'] - result['<STR_LIT>']<EOL>if timedelta.days:<EOL><INDENT>logger.info('<STR_LIT>' % (<EOL>timedelta.days,<EOL>timedelta.seconds + (timedelta.microseconds * (<NUM_LIT>))))<EOL><DEDENT>else:<EOL><INDENT>logger.info('<STR_LIT>' % (<EOL>timedelta.seconds + (timedelta.microseconds * (<NUM_LIT>))))<EOL><DEDENT>result['<STR_LIT>'] = resp_body['<STR_LIT>']<EOL>result['<STR_LIT>'] = resp_body['<STR_LIT>']<EOL>result['<STR_LIT>'] = resp_body['<STR_LIT>']<EOL>if result['<STR_LIT>'] == <NUM_LIT:1>:<EOL><INDENT>docs_read = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>docs_read = '<STR_LIT>' % (result['<STR_LIT>'],)<EOL><DEDENT>if result['<STR_LIT>'] == <NUM_LIT:1>:<EOL><INDENT>docs_written = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>docs_written = '<STR_LIT>' % (result['<STR_LIT>'],)<EOL><DEDENT>if result['<STR_LIT>'] == <NUM_LIT:1>:<EOL><INDENT>missing_checked = '<STR_LIT>' % (<EOL>result['<STR_LIT>'],)<EOL><DEDENT>else:<EOL><INDENT>missing_checked = '<STR_LIT>' % (<EOL>result['<STR_LIT>'], result['<STR_LIT>'],)<EOL><DEDENT>logging.info('<STR_LIT>' % (docs_read, docs_written))<EOL>logging.info(missing_checked)<EOL>return result<EOL><DEDENT>else:<EOL><INDENT>logger.error('<STR_LIT>' % (<EOL>resp_body['<STR_LIT>'][:<NUM_LIT:6>],))<EOL>result['<STR_LIT>'] = resp_body['<STR_LIT>']<EOL>result['<STR_LIT>'] = resp_body['<STR_LIT>']<EOL>result['<STR_LIT>'] = resp_body['<STR_LIT>']<EOL>raise ReplicationFailure(resp_headers, result)<EOL><DEDENT>

Replicate an existing database to another existing database.


ensure_specifier_exists(target_spec)<EOL>return replicate_existing(source_spec, target_spec)<EOL>

Replicate one existing database to another (optionally existing) DB.


return couchdb.client.Server(<EOL>server_url if server_url else settings._('<STR_LIT>'))<EOL>

Return a CouchDB server instance based on Django project settings.


return get_server(server_url)[db_name]<EOL>

Return a CouchDB database instance, given its name.


def get_doc(doc_id, db_name, server_url='<STR_LIT>', rev=None):

db = get_server(server_url)[db_name]<EOL>if rev:<EOL><INDENT>headers, response = db.resource.get(doc_id, rev=rev)<EOL>return couchdb.client.Document(response)<EOL><DEDENT>return db[doc_id]<EOL>

Return a CouchDB document, given its ID, revision and database name.


server = get_server(server_url)<EOL>if db_name in server:<EOL><INDENT>return server[db_name]<EOL><DEDENT>return server.create(db_name)<EOL>

Return an (optionally existing) CouchDB database instance.


def wrapper(*args, **kwargs):<EOL><INDENT>return list(function(*args, **kwargs))<EOL><DEDENT>wrapper.__name__ = function.__name__<EOL>wrapper.__doc__ = function.__doc__<EOL>return wrapper<EOL>

Wrap a generator function so that it returns a list when called.

For example:

    # Define a generator
    >>> def mygen(n):
    ...     i = 0
    ...     while i < n:
    ...         yield i
    ...         i += 1
    # This is how it might work
    >>> generator = mygen(5)
    >>> generator.next()
    0
    >>> generator.next()
    1
    # Wrap it in generator_to_list, and it will behave differently.
    >>> mygen = generator_to_list(mygen)
    >>> mygen(5)
    [0, 1, 2, 3, 4]


match = re.match(r'<STR_LIT>' + re.escape(os.path.extsep) + r'<STR_LIT>', filename)<EOL>if os.path.exists(filename):<EOL><INDENT>if match:<EOL><INDENT>prefix, number = match.groups()<EOL>number = int(number)<EOL>while os.path.exists(os.path.extsep.join((prefix, str(number)))):<EOL><INDENT>number += <NUM_LIT:1><EOL><DEDENT>return os.path.extsep.join((prefix, str(number)))<EOL><DEDENT><DEDENT>elif match:<EOL><INDENT>return filename<EOL><DEDENT>return logrotate(os.path.extsep.join((filename, '<STR_LIT:0>')))<EOL>

Return the next available filename for a particular filename prefix.

For example:

    >>> import os
    # Make three (empty) files in a directory
    >>> fp0 = open('file.0', 'w')
    >>> fp1 = open('file.1', 'w')
    >>> fp2 = open('file.2', 'w')
    >>> fp0.close(), fp1.close(), fp2.close()
    (None, None, None)
    # Use logrotate to get the next available filename.
    >>> logrotate('file')
    'file.3'
    >>> logrotate('file.2')
    'file.3'
    >>> logrotate('file.1')
    'file.3'

This can be used to get the next available filename for logging, allowing
you to rotate log files, without using Python's ``logging`` module.


module, basename = str(function_name).rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>try:<EOL><INDENT>return getattr(__import__(module, fromlist=[basename]), basename)<EOL><DEDENT>except (ImportError, AttributeError):<EOL><INDENT>raise FunctionNotFound(function_name)<EOL><DEDENT>

Given a Python function name, return the function it refers to.


<EOL>return json_data.replace('<STR_LIT:\n>', '<STR_LIT:U+0020>').replace('<STR_LIT:\r>', '<STR_LIT:U+0020>')<EOL>

Prevent JSON data from taking up multiple lines.


<EOL>return json.dumps({<EOL>'<STR_LIT:error>': type(exc).__name__,<EOL>'<STR_LIT>': str(exc)})<EOL>

Transform a Python exception into a CouchDB JSON error.


self.functions.clear()<EOL>self.function_counter = <NUM_LIT:0><EOL>


function_name = function_name.strip()<EOL>try:<EOL><INDENT>function = get_function(function_name)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self.wfile.write(js_error(exc) + NEWLINE)<EOL>return<EOL><DEDENT>if not getattr(function, '<STR_LIT>', None):<EOL><INDENT>self.functions[function_name] = (self.function_counter, function)<EOL><DEDENT>else:<EOL><INDENT>self.functions[function_name] = (self.function_counter,<EOL>function(self.log))<EOL><DEDENT>self.function_counter += <NUM_LIT:1><EOL>return True<EOL>

Add a function to the function list, in order.


<EOL>for function in sorted(list(self.functions.values()), key=lambda x: x[<NUM_LIT:0>]):<EOL><INDENT>try:<EOL><INDENT>yield [list(function(document))]<EOL><DEDENT>except Exception as exc:<EOL><INDENT>yield []<EOL>self.log(repr(exc))<EOL><DEDENT><DEDENT>

Return the mapping of a document according to the function list.


def handle_reduce(self, reduce_function_names, mapped_docs):

reduce_functions = []<EOL>for reduce_function_name in reduce_function_names:<EOL><INDENT>try:<EOL><INDENT>reduce_function = get_function(reduce_function_name)<EOL>if getattr(reduce_function, '<STR_LIT>', None):<EOL><INDENT>reduce_function = reduce_function(self.log)<EOL><DEDENT>reduce_functions.append(reduce_function)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self.log(repr(exc))<EOL>reduce_functions.append(lambda *args, **kwargs: None)<EOL><DEDENT><DEDENT>keys, values = list(zip(<EOL>(key, value) for ((key, doc_id), value) in mapped_docs))<EOL>results = []<EOL>for reduce_function in reduce_functions:<EOL><INDENT>try:<EOL><INDENT>results.append(reduce_function(keys, values, rereduce=False))<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self.log(repr(exc))<EOL>results.append(None)<EOL><DEDENT><DEDENT>return [True, results]<EOL>

Reduce several mapped documents by several reduction functions.


def handle_rereduce(self, reduce_function_names, values):

<EOL>reduce_functions = []<EOL>for reduce_function_name in reduce_function_names:<EOL><INDENT>try:<EOL><INDENT>reduce_function = get_function(reduce_function_name)<EOL>if getattr(reduce_function, '<STR_LIT>', None):<EOL><INDENT>reduce_function = reduce_function(self.log)<EOL><DEDENT>reduce_functions.append(reduce_function)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self.log(repr(exc))<EOL>reduce_functions.append(lambda *args, **kwargs: None)<EOL><DEDENT><DEDENT>results = []<EOL>for reduce_function in reduce_functions:<EOL><INDENT>try:<EOL><INDENT>results.append(reduce_function(None, values, rereduce=True))<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self.log(repr(exc))<EOL>results.append(None)<EOL><DEDENT><DEDENT>return [True, results]<EOL>

Re-reduce a set of values, with a list of rereduction functions.


def handle_validate(self, function_name, new_doc, old_doc, user_ctx):

try:<EOL><INDENT>function = get_function(function_name)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self.log(repr(exc))<EOL>return False<EOL><DEDENT>try:<EOL><INDENT>return function(new_doc, old_doc, user_ctx)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self.log(repr(exc))<EOL>return repr(exc)<EOL><DEDENT>

Validate...this function is undocumented, but still in CouchDB.


while True:<EOL><INDENT>try:<EOL><INDENT>line = self.rfile.readline()<EOL>try:<EOL><INDENT>cmd = json.loads(line)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>self.wfile.write(repr(exc) + NEWLINE)<EOL>continue<EOL><DEDENT>else:<EOL><INDENT>handler = getattr(self, '<STR_LIT>' + cmd[<NUM_LIT:0>], None)<EOL>if not handler:<EOL><INDENT>self.wfile.write(<EOL>repr(CommandNotFound(cmd[<NUM_LIT:0>])) + NEWLINE)<EOL>continue<EOL><DEDENT>return_value = handler(*cmd[<NUM_LIT:1>:])<EOL>if not return_value:<EOL><INDENT>continue<EOL><DEDENT>self.wfile.write(<EOL>one_lineify(json.dumps(return_value)) + NEWLINE)<EOL><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>self.wfile.write(repr(exc) + NEWLINE)<EOL>continue<EOL><DEDENT><DEDENT>

The main function called to handle a request.


self.wfile.write(json.dumps({'<STR_LIT>': string}) + NEWLINE)<EOL>


def __init__(self, x, y, z, formatter=numpy_formatter):

x = np.asarray(x, dtype=np.float64)<EOL>y = np.asarray(y, dtype=np.float64)<EOL>z = np.ma.asarray(z, dtype=np.float64)<EOL>if x.shape != z.shape:<EOL><INDENT>raise TypeError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>").format(str(x.shape), str(z.shape)))<EOL><DEDENT>if y.shape != z.shape:<EOL><INDENT>raise TypeError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>").format(str(y.shape), str(z.shape)))<EOL><DEDENT>mask = z.mask if z.mask is not np.ma.nomask and z.mask.any() else None<EOL>self._contour_generator = _contour.QuadContourGenerator(<EOL>x, y, z.filled(), mask, True, <NUM_LIT:0>)<EOL>super().__init__(formatter)<EOL>

Initialize a :class:`QuadContourGenerator`, see class docstring.


@classmethod<EOL><INDENT>def from_curvilinear(cls, x, y, z, formatter=numpy_formatter):<DEDENT>

return cls(x, y, z, formatter)<EOL>

Construct a contour generator from a curvilinear grid.

        Note
        ----
        This is an alias for the default constructor.

        Parameters
        ----------
        x : array_like
            x coordinates of each point in `z`.  Must be the same size as `z`.
        y : array_like
            y coordinates of each point in `z`.  Must be the same size as `z`.
        z : array_like
            The 2-dimensional curvilinear grid of data to compute
            contours for.  Masked arrays are supported.
        formatter : callable
            A conversion function to convert from the internal `Matplotlib`_
            contour format to an external format.  See :ref:`formatters` for
            more information.

        Returns
        -------
        : :class:`QuadContourGenerator`
            Initialized contour generator.


@classmethod<EOL><INDENT>def from_rectilinear(cls, x, y, z, formatter=numpy_formatter):<DEDENT>

x = np.asarray(x, dtype=np.float64)<EOL>y = np.asarray(y, dtype=np.float64)<EOL>z = np.ma.asarray(z, dtype=np.float64)<EOL>if x.ndim != <NUM_LIT:1>:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>".format(x.ndim))<EOL><DEDENT>if y.ndim != <NUM_LIT:1>:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>".format(y.ndim))<EOL><DEDENT>if z.ndim != <NUM_LIT:2>:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>".format(z.ndim))<EOL><DEDENT>if x.size != z.shape[<NUM_LIT:1>]:<EOL><INDENT>raise TypeError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>").format(x.size, z.shape[<NUM_LIT:1>]))<EOL><DEDENT>if y.size != z.shape[<NUM_LIT:0>]:<EOL><INDENT>raise TypeError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>").format(y.size, z.shape[<NUM_LIT:0>]))<EOL><DEDENT>y, x = np.meshgrid(y, x, indexing='<STR_LIT>')<EOL>return cls(x, y, z, formatter)<EOL>

Construct a contour generator from a rectilinear grid.

        Parameters
        ----------
        x : array_like
            x coordinates of each column of `z`.  Must be the same length as
            the number of columns in `z`.  (len(x) == z.shape[1])
        y : array_like
            y coordinates of each row of `z`.  Must be the same length as the
            number of columns in `z`.  (len(y) == z.shape[0])
        z : array_like
            The 2-dimensional rectilinear grid of data to compute contours for.
            Masked arrays are supported.
        formatter : callable
            A conversion function to convert from the internal `Matplotlib`_
            contour format to an external format.  See :ref:`formatters` for
            more information.

        Returns
        -------
        : :class:`QuadContourGenerator`
            Initialized contour generator.


@classmethod<EOL><INDENT>def from_uniform(<EOL>cls, z, origin=(<NUM_LIT:0>, <NUM_LIT:0>), step=(<NUM_LIT:1>, <NUM_LIT:1>), formatter=numpy_formatter):<DEDENT>

z = np.ma.asarray(z, dtype=np.float64)<EOL>if z.ndim != <NUM_LIT:2>:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>".format(z.ndim))<EOL><DEDENT>if len(origin) != <NUM_LIT:2>:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>".format(<EOL>len(origin)))<EOL><DEDENT>if len(step) != <NUM_LIT:2>:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>".format(<EOL>len(step)))<EOL><DEDENT>if any(s == <NUM_LIT:0> for s in step):<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>".format(<EOL>str(step)))<EOL><DEDENT>y, x = np.mgrid[<EOL>origin[<NUM_LIT:0>]:(origin[<NUM_LIT:0>]+step[<NUM_LIT:0>]*z.shape[<NUM_LIT:0>]):step[<NUM_LIT:0>],<EOL>origin[<NUM_LIT:1>]:(origin[<NUM_LIT:1>]+step[<NUM_LIT:1>]*z.shape[<NUM_LIT:1>]):step[<NUM_LIT:1>]]<EOL>return cls(x, y, z, formatter)<EOL>

Construct a contour generator from a uniform grid.

        NOTE
        ----
        The default `origin` and `step` values is equivalent to calling
        :meth:`matplotlib.axes.Axes.contour` with only the `z` argument.

        Parameters
        ----------
        z : array_like
            The 2-dimensional uniform grid of data to compute contours for.
            Masked arrays are supported.
        origin : (number.Number, number.Number)
            The (x, y) coordinate of data point `z[0,0]`.
        step :  (number.Number, number.Number)
            The (x, y) distance between data points in `z`.
        formatter : callable
            A conversion function to convert from the internal `Matplotlib`_
            contour format to an external format.  See :ref:`formatters` for
            more information.

        Returns
        -------
        : :class:`QuadContourGenerator`
            Initialized contour generator.


def null_formatter(level, vertices, codes=None):

return level, vertices, codes<EOL>

Null formatter that passes through the raw vertices and codes.


def numpy_formatter(_, vertices, codes=None):

if codes is None:<EOL><INDENT>return vertices<EOL><DEDENT>numpy_vertices = []<EOL>for vertices_, codes_ in zip(vertices, codes):<EOL><INDENT>starts = np.nonzero(codes_ == MPLPATHCODE.MOVETO)[<NUM_LIT:0>]<EOL>stops = np.nonzero(codes_ == MPLPATHCODE.CLOSEPOLY)[<NUM_LIT:0>]<EOL>for start, stop in zip(starts, stops):<EOL><INDENT>numpy_vertices.append(vertices_[start:stop+<NUM_LIT:1>, :])<EOL><DEDENT><DEDENT>return numpy_vertices<EOL>

`NumPy`_ style contour formatter.

    Contours are returned as a list of Nx2 arrays containing the x and y
    vertices of the contour line.

    For filled contours the direction of vertices matters:

    * CCW (ACW): The vertices give the exterior of a contour polygon.
    * CW: The vertices give a hole of a contour polygon.  This hole will
        always be inside the exterior of the last contour exterior.

    .. note:: This is the fastest format.

    .. _NumPy: http://www.numpy.org


def matlab_formatter(level, vertices, codes=None):

vertices = numpy_formatter(level, vertices, codes)<EOL>if codes is not None:<EOL><INDENT>level = level[<NUM_LIT:0>]<EOL><DEDENT>headers = np.vstack((<EOL>[v.shape[<NUM_LIT:0>] for v in vertices],<EOL>[level]*len(vertices))).T<EOL>vertices = np.vstack(<EOL>list(it.__next__() for it in<EOL>itertools.cycle((iter(headers), iter(vertices)))))<EOL>return vertices<EOL>

`MATLAB`_ style contour formatter.

    Contours are returned as a single Nx2, `MATLAB`_ style, contour array.
    There are two types of rows in this format:

    * Header: The first element of a header row is the level of the contour
      (the lower level for filled contours) and the second element is the
      number of vertices (to follow) belonging to this contour line.
    * Vertex: x,y coordinate pairs of the vertex.

    A header row is always followed by the coresponding number of vertices.
    Another header row may follow if there are more contour lines.

    For filled contours the direction of vertices matters:

    * CCW (ACW): The vertices give the exterior of a contour polygon.
    * CW: The vertices give a hole of a contour polygon.  This hole will
        always be inside the exterior of the last contour exterior.

    For further explanation of this format see the `Mathworks documentation
    <https://www.mathworks.com/help/matlab/ref/contour-properties.html#prop_ContourMatrix>`_
    noting that the MATLAB format used in the `contours` package is the
    transpose of that used by `MATLAB`_ (since `MATLAB`_ is column-major
    and `NumPy`_ is row-major by default).

    .. _NumPy: http://www.numpy.org

    .. _MATLAB: https://www.mathworks.com/products/matlab.html


def shapely_formatter(_, vertices, codes=None):

elements = []<EOL>if codes is None:<EOL><INDENT>for vertices_ in vertices:<EOL><INDENT>if np.all(vertices_[<NUM_LIT:0>, :] == vertices_[-<NUM_LIT:1>, :]):<EOL><INDENT>if len(vertices) < <NUM_LIT:3>:<EOL><INDENT>elements.append(Point(vertices_[<NUM_LIT:0>, :]))<EOL><DEDENT>else:<EOL><INDENT>elements.append(LinearRing(vertices_))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>elements.append(LineString(vertices_))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for vertices_, codes_ in zip(vertices, codes):<EOL><INDENT>starts = np.nonzero(codes_ == MPLPATHCODE.MOVETO)[<NUM_LIT:0>]<EOL>stops = np.nonzero(codes_ == MPLPATHCODE.CLOSEPOLY)[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>rings = [LinearRing(vertices_[start:stop+<NUM_LIT:1>, :])<EOL>for start, stop in zip(starts, stops)]<EOL>elements.append(Polygon(rings[<NUM_LIT:0>], rings[<NUM_LIT:1>:]))<EOL><DEDENT>except ValueError as err:<EOL><INDENT>if np.any(stop - start - <NUM_LIT:1> == <NUM_LIT:0>):<EOL><INDENT>if stops[<NUM_LIT:0>] < starts[<NUM_LIT:0>]+<NUM_LIT:2>:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>rings = [<EOL>LinearRing(vertices_[start:stop+<NUM_LIT:1>, :])<EOL>for start, stop in zip(starts, stops)<EOL>if stop >= start+<NUM_LIT:2>]<EOL>elements.append(Polygon(rings[<NUM_LIT:0>], rings[<NUM_LIT:1>:]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise(err)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return elements<EOL>

`Shapely`_ style contour formatter.

    Contours are returned as a list of :class:`shapely.geometry.LineString`,
    :class:`shapely.geometry.LinearRing`, and :class:`shapely.geometry.Point`
    geometry elements.

    Filled contours return a list of :class:`shapely.geometry.Polygon`
    elements instead.

    .. note:: If possible, `Shapely speedups`_ will be enabled.

    .. _Shapely: http://toblerity.org/shapely/manual.html

    .. _Shapely speedups: http://toblerity.org/shapely/manual.html#performance

    See Also
    --------
    `descartes <https://bitbucket.org/sgillies/descartes/>`_ : Use `Shapely`_
    or GeoJSON-like geometric objects as matplotlib paths and patches.


def __init__(self, formatter=numpy_formatter, *args, **kwargs):

<EOL>self.formatter = formatter<EOL>

Initialize a :class:`ContourMixin`, see class docstring.


if not isinstance(level, numbers.Number):<EOL><INDENT>raise TypeError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>").format(type(level)))<EOL><DEDENT>vertices = self._contour_generator.create_contour(level)<EOL>return self.formatter(level, vertices)<EOL>

Get contour lines at the given level.

        Parameters
        ----------
        level : numbers.Number
            The data level to calculate the contour lines for.

        Returns
        -------
        :
            The result of the :attr:`formatter` called on the contour at the
            given `level`.


def filled_contour(self, min=None, max=None):

<EOL>if min is None:<EOL><INDENT>min = np.finfo(np.float64).min<EOL><DEDENT>if max is None:<EOL><INDENT>max = np.finfo(np.float64).max<EOL><DEDENT>vertices, codes = (<EOL>self._contour_generator.create_filled_contour(min, max))<EOL>return self.formatter((min, max), vertices, codes)<EOL>

Get contour polygons between the given levels.

        Parameters
        ----------
        min : numbers.Number or None
            The minimum data level of the contour polygon.  If :obj:`None`,
            ``numpy.finfo(numpy.float64).min`` will be used.
        max : numbers.Number or None
            The maximum data level of the contour polygon.  If :obj:`None`,
            ``numpy.finfo(numpy.float64).max`` will be used.

        Returns
        -------
        :
            The result of the :attr:`formatter` called on the filled contour
            between `min` and `max`.


if seen == None:<EOL><INDENT>seen = []<EOL><DEDENT>if hasattr(obj, "<STR_LIT>"):<EOL><INDENT>state = obj.__getstate__()<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>if state == None:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if isinstance(state,tuple):<EOL><INDENT>if not isinstance(state[<NUM_LIT:0>], dict):<EOL><INDENT>state=state[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>state=state[<NUM_LIT:0>].update(state[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>result = {}<EOL>for i in state:<EOL><INDENT>try:<EOL><INDENT>pickle.dumps(state[i], protocol=<NUM_LIT:2>)<EOL><DEDENT>except pickle.PicklingError as e:<EOL><INDENT>if not state[i] in seen:<EOL><INDENT>seen.append(state[i])<EOL>result[i]=get_pickling_errors(state[i],seen)<EOL><DEDENT><DEDENT><DEDENT>return result<EOL>


if dump is None:<EOL><INDENT>dump = '<STR_LIT>'<EOL><DEDENT>if dump.startswith('<STR_LIT>'):<EOL><INDENT>if dump == '<STR_LIT>':<EOL><INDENT>proto = protocol<EOL><DEDENT>else:<EOL><INDENT>proto = dump.strip('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>proto = int(proto)<EOL>assert proto>=-<NUM_LIT:1><EOL><DEDENT>except:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>code = """<STR_LIT>"""<EOL><DEDENT>rt dill<EOL><INDENT>open('<STR_LIT>', '<STR_LIT:wb>') as fd:<EOL><DEDENT>dill.dump( value, fd, protocol=%i )<EOL>fd.flush()<EOL>os.fsync(fd.fileno())<EOL>proto<EOL>elif dump == '<STR_LIT>':<EOL><INDENT>code = """<STR_LIT>"""<EOL><DEDENT>rt numpy<EOL><INDENT>open('<STR_LIT>', '<STR_LIT:wb>') as fd:<EOL><DEDENT>numpy.save(file=fd, arr=value)<EOL>fd.flush()<EOL>os.fsync(fd.fileno())<EOL>elif dump == '<STR_LIT>':<EOL><INDENT>code =


if pull is None or pull.startswith('<STR_LIT>'):<EOL><INDENT>code = """<STR_LIT>"""<EOL><DEDENT>rt dill<EOL><INDENT>open('<STR_LIT>', '<STR_LIT:rb>') as fd:<EOL><DEDENT>PULLED_DATA = dill.load( fd )<EOL>elif pull == '<STR_LIT>':<EOL><INDENT>code = """<STR_LIT>"""<EOL><DEDENT>rt numpy<EOL><INDENT>open('<STR_LIT>', '<STR_LIT:rb>') as fd:<EOL><DEDENT>PULLED_DATA=numpy.load(file=fd)<EOL>elif pull == '<STR_LIT>':<EOL><INDENT>code =


@wraps(func)<EOL>def wrapper(self, *args, **kwargs):<EOL><INDENT>if self.path is None:<EOL><INDENT>warnings.warn('<STR_LIT>')<EOL>return<EOL><DEDENT>return func(self, *args, **kwargs)<EOL><DEDENT>return wrapper<EOL>

Decorate methods when repository path is required.


def __clean_before_after(self, stateBefore, stateAfter, keepNoneEmptyDirectory=True):

<EOL>errors    = []<EOL>afterDict = {}<EOL>[afterDict.setdefault(list(aitem)[<NUM_LIT:0>],[]).append(aitem) for aitem in stateAfter]<EOL>for bitem in reversed(stateBefore):<EOL><INDENT>relaPath = list(bitem)[<NUM_LIT:0>]<EOL>basename = os.path.basename(relaPath)<EOL>btype    = bitem[relaPath]['<STR_LIT:type>']<EOL>alist    = afterDict.get(relaPath, [])<EOL>aitem    = [a for a in alist if a[relaPath]['<STR_LIT:type>']==btype]<EOL>if len(aitem)><NUM_LIT:1>:<EOL><INDENT>errors.append("<STR_LIT>"%(basename,btype,relaPath))<EOL>continue<EOL><DEDENT>if not len(aitem):<EOL><INDENT>removeDirs  = []<EOL>removeFiles = []<EOL>if btype == '<STR_LIT>':<EOL><INDENT>if not len(relaPath):<EOL><INDENT>errors.append("<STR_LIT>")<EOL>continue<EOL><DEDENT>removeDirs.append(os.path.join(self.__path,relaPath))<EOL>removeFiles.append(os.path.join(self.__path,relaPath,self.__dirInfo))<EOL>removeFiles.append(os.path.join(self.__path,relaPath,self.__dirLock))<EOL><DEDENT>elif btype == '<STR_LIT:file>':<EOL><INDENT>removeFiles.append(os.path.join(self.__path,relaPath))<EOL>removeFiles.append(os.path.join(self.__path,relaPath,self.__fileInfo%basename))<EOL>removeFiles.append(os.path.join(self.__path,relaPath,self.__fileLock%basename))<EOL><DEDENT>else:<EOL><INDENT>removeDirs.append(os.path.join(self.__path,relaPath))<EOL>removeFiles.append(os.path.join(self.__path,relaPath,self.__fileInfo%basename))<EOL><DEDENT>for fpath in removeFiles:<EOL><INDENT>if os.path.isfile(fpath):<EOL><INDENT>try:<EOL><INDENT>os.remove(fpath)<EOL><DEDENT>except Exception as err:<EOL><INDENT>errors.append("<STR_LIT>"%(fpath, str(err)))<EOL><DEDENT><DEDENT><DEDENT>for dpath in removeDirs:<EOL><INDENT>if os.path.isdir(dpath):<EOL><INDENT>if keepNoneEmptyDirectory or not len(os.listdir(dpath)):<EOL><INDENT>try:<EOL><INDENT>shutil.rmtree(dpath)<EOL><DEDENT>except Exception as err:<EOL><INDENT>errors.append("<STR_LIT>"%(fpath, str(err)))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return len(errors)==<NUM_LIT:0>, errors<EOL>

clean repository given before and after states


return self.__repo['<STR_LIT>']<EOL>


return self.__path<EOL>

The repository instance path which points to the directory where
        .pyreprepo is.


return self.__repo['<STR_LIT>']<EOL>

Get repository unique name as generated when repository was created


if self.__path is None:<EOL><INDENT>return <NUM_LIT:0>,<NUM_LIT:0><EOL><DEDENT>nfiles = <NUM_LIT:0><EOL>ndirs  = <NUM_LIT:0><EOL>for fdict in self.get_repository_state():<EOL><INDENT>fdname = list(fdict)[<NUM_LIT:0>]<EOL>if fdname == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>if fdict[fdname].get('<STR_LIT>', False):<EOL><INDENT>nfiles += <NUM_LIT:1><EOL><DEDENT>elif fdict[fdname].get('<STR_LIT>', False):<EOL><INDENT>ndirs += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT><DEDENT>return ndirs,nfiles<EOL>

Get repository descriptive stats

:Returns:
    #. numberOfDirectories (integer): Number of diretories in repository
    #. numberOfFiles (integer): Number of files in repository


self.__path   = None<EOL>self.__repo   = {'<STR_LIT>': str(uuid.uuid1()),<EOL>'<STR_LIT>': time.time(),<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': str(__version__),<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': []}<EOL>


if path.strip() in ('<STR_LIT>','<STR_LIT:.>'):<EOL><INDENT>path = os.getcwd()<EOL><DEDENT>repoPath = os.path.realpath( os.path.expanduser(path) )<EOL>if os.path.isfile( os.path.join(repoPath,self.__repoFile) ):<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>from .OldRepository import Repository<EOL>REP = Repository()<EOL>result = REP.is_repository(repoPath)<EOL><DEDENT>except:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>if result:<EOL><INDENT>warnings.warn("<STR_LIT>")<EOL><DEDENT>return result<EOL><DEDENT><DEDENT>

Check if there is a Repository in path.

:Parameters:
    #. path (string): The real path of the directory where to check if
       there is a repository.

:Returns:
    #. result (boolean): Whether it's a repository or not.


def load_repository(self, path, verbose=True, ntrials=<NUM_LIT:3>):

assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>repo = None<EOL>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>self.__load_repository(path=path, verbose=True)<EOL><DEDENT>except Exception as err1:<EOL><INDENT>try:<EOL><INDENT>from .OldRepository import Repository<EOL>REP = Repository(path)<EOL><DEDENT>except Exception as err2:<EOL><INDENT>error = "<STR_LIT>"%(err1, err2)<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>repo  = REP<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>error = None<EOL>repo  = self<EOL>break<EOL><DEDENT><DEDENT>assert error is None, error<EOL>return repo<EOL>

Load repository from a directory path and update the current instance.
First, new repository still will be loaded. If failed, then old
style repository load will be tried.

:Parameters:
    #. path (string): The path of the directory from where to load
       the repository from. If '.' or an empty string is passed,
       the current working directory will be used.
    #. verbose (boolean): Whether to be verbose about abnormalities
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
     #. repository (pyrep.Repository): returns self repository with loaded data.


def create_repository(self, path, info=None, description=None, replace=True, allowNoneEmpty=True, raiseError=True):

assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(allowNoneEmpty, bool), "<STR_LIT>"<EOL>assert isinstance(replace, bool), "<STR_LIT>"<EOL>assert isinstance(path, basestring), "<STR_LIT>"<EOL>if info is None:<EOL><INDENT>info = '<STR_LIT>'<EOL><DEDENT>try:<EOL><INDENT>pickle.dumps(info)<EOL><DEDENT>except Exception as err:<EOL><INDENT>raise Exception("<STR_LIT>"%str(err))<EOL><DEDENT>if description is None:<EOL><INDENT>description = '<STR_LIT>'<EOL><DEDENT>assert isinstance(description, basestring), "<STR_LIT>"<EOL>if path.strip() in ('<STR_LIT>','<STR_LIT:.>'):<EOL><INDENT>path = os.getcwd()<EOL><DEDENT>realPath = os.path.realpath( os.path.expanduser(path) )<EOL>message = []<EOL>if self.is_repository(realPath):<EOL><INDENT>if not replace:<EOL><INDENT>message.append("<STR_LIT>"%path)<EOL>return False, message<EOL><DEDENT>else:<EOL><INDENT>message.append("<STR_LIT>"%path)<EOL>try:<EOL><INDENT>for _df in os.listdir(realPath):<EOL><INDENT>_p = os.path.join(realPath, _df)<EOL>if os.path.isdir(_p):<EOL><INDENT>shutil.rmtree( _p )<EOL><DEDENT>else:<EOL><INDENT>os.remove(_p)<EOL><DEDENT><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>message.append("<STR_LIT>"%(str(err)))<EOL>return False, '<STR_LIT:\n>'.join(message)<EOL><DEDENT><DEDENT><DEDENT>if not os.path.isdir(realPath):<EOL><INDENT>os.makedirs(realPath)<EOL><DEDENT>elif len(os.listdir(realPath)) and not allowNoneEmpty:<EOL><INDENT>return False, "<STR_LIT>"<EOL><DEDENT>oldRepo = self.__repo<EOL>self.reset()<EOL>self.__path = realPath.rstrip(os.sep)<EOL>self.__repo['<STR_LIT>'] = info<EOL>saved = self.save(description=description)<EOL>if not saved:<EOL><INDENT>self.__repo = oldRepo<EOL>message.append("<STR_LIT>")<EOL>return False, '<STR_LIT:\n>'.join(message)<EOL><DEDENT>return True, '<STR_LIT:\n>'.join(message)<EOL>

create a repository in a directory. This method insures the creation of
the directory in the system if it is missing.\n

**N.B. If replace is True and existing repository is found in path, create_repository erases all existing files and directories in path.**

:Parameters:
    #. path (string): The real absolute path where to create the Repository.
       If '.' or an empty string is passed, the current working directory will be used.
    #. description (None, str): Repository main directory information.
    #. info (None, object): Repository information. It can
       be None or any pickle writable type of data.
    #. replace (boolean): Whether to replace existing repository.
    #. allowNoneEmpty (boolean): Allow creating repository in none-empty
       directory.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.

:Returns:
    #. success (boolean): Whether creating repository was successful
    #. message (None, str): Any returned message.


def remove_repository(self, path=None, removeEmptyDirs=True):

assert isinstance(removeEmptyDirs, bool), "<STR_LIT>"<EOL>if path is not None:<EOL><INDENT>if path != self.__path:<EOL><INDENT>repo = Repository()<EOL>repo.load_repository(path)<EOL><DEDENT>else:<EOL><INDENT>repo = self<EOL><DEDENT><DEDENT>else:<EOL><INDENT>repo = self<EOL><DEDENT>assert repo.path is not None, "<STR_LIT>"<EOL>for fdict in reversed(repo.get_repository_state()):<EOL><INDENT>relaPath   = list(fdict)[<NUM_LIT:0>]<EOL>realPath   = os.path.join(repo.path, relaPath)<EOL>path, name = os.path.split(realPath)<EOL>if fdict[relaPath]['<STR_LIT:type>'] == '<STR_LIT:file>':<EOL><INDENT>if os.path.isfile(realPath):<EOL><INDENT>os.remove(realPath)<EOL><DEDENT>if os.path.isfile(os.path.join(repo.path,path,self.__fileInfo%name)):<EOL><INDENT>os.remove(os.path.join(repo.path,path,self.__fileInfo%name))<EOL><DEDENT>if os.path.isfile(os.path.join(repo.path,path,self.__fileLock%name)):<EOL><INDENT>os.remove(os.path.join(repo.path,path,self.__fileLock%name))<EOL><DEDENT>if os.path.isfile(os.path.join(repo.path,path,self.__fileClass%name)):<EOL><INDENT>os.remove(os.path.join(repo.path,path,self.__fileClass%name))<EOL><DEDENT><DEDENT>elif fdict[relaPath]['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>if os.path.isfile(os.path.join(realPath,self.__dirInfo)):<EOL><INDENT>os.remove(os.path.join(realPath,self.__dirInfo))<EOL><DEDENT>if os.path.isfile(os.path.join(realPath,self.__dirLock)):<EOL><INDENT>os.remove(os.path.join(realPath,self.__dirLock))<EOL><DEDENT>if not len(os.listdir(realPath)) and removeEmptyDirs:<EOL><INDENT>shutil.rmtree( realPath )<EOL><DEDENT><DEDENT><DEDENT>if os.path.isfile(os.path.join(repo.path,self.__repoFile)):<EOL><INDENT>os.remove(os.path.join(repo.path,self.__repoFile))<EOL><DEDENT>if os.path.isfile(os.path.join(repo.path,self.__repoLock)):<EOL><INDENT>os.remove(os.path.join(repo.path,self.__repoLock))<EOL><DEDENT>

Remove all repository from path along with all repository tracked files.

:Parameters:
    #. path (None, string): The path the repository to remove.
    #. removeEmptyDirs (boolean): Whether to remove remaining empty
       directories.


@path_required<EOL><INDENT>def save(self, description=None, raiseError=True, ntrials=<NUM_LIT:3>):<DEDENT>

assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>if description is not None:<EOL><INDENT>assert isinstance(description, basestring), "<STR_LIT>"<EOL><DEDENT>dirInfoPath = os.path.join(self.__path, self.__dirInfo)<EOL>if description is None and not os.path.isfile(dirInfoPath):<EOL><INDENT>description = '<STR_LIT>'<EOL><DEDENT>LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))<EOL>acquired, code = LR.acquire_lock()<EOL>m = "<STR_LIT>"%(code,)<EOL>if not acquired:<EOL><INDENT>assert not raiseError, Exception(m)<EOL>return False, m<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>repoInfoPath = os.path.join(self.__path, self.__repoFile)<EOL>error = None<EOL>self.__save_dirinfo(description=description, dirInfoPath=dirInfoPath)<EOL>if os.path.isfile(repoInfoPath):<EOL><INDENT>with open(repoInfoPath, '<STR_LIT:rb>') as fd:<EOL><INDENT>repo = self.__load_repository_pickle_file(os.path.join(self.__path, self.__repoFile))<EOL>self.__repo['<STR_LIT>'] = repo['<STR_LIT>']<EOL><DEDENT><DEDENT>with open(repoInfoPath, '<STR_LIT:wb>') as fd:<EOL><INDENT>self.__repo["<STR_LIT>"] = time.time()<EOL>pickle.dump( self.__repo,fd, protocol=self._DEFAULT_PICKLE_PROTOCOL )<EOL>fd.flush()<EOL>os.fsync(fd.fileno())<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>error = "<STR_LIT>"%err<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>LR.release_lock()<EOL>assert error is None or not raiseError, error<EOL>return error is None, error<EOL>

Save repository '.pyreprepo' to disk and create (if missing) or
update (if description is not None) '.pyrepdirinfo'.

:Parameters:
    #. description (None, str): Repository main directory information.
       If given will be replaced.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
    #. success (bool): Whether saving was successful.
    #. error (None, string): Fail to save repository message in case
       saving is not successful. If success is True, error will be None.


assert isinstance(path, basestring), "<STR_LIT>"<EOL>name = os.path.basename(path)<EOL>if not len(name):<EOL><INDENT>return False, "<STR_LIT>"<EOL><DEDENT>for em in [self.__repoLock,self.__repoFile,self.__dirInfo,self.__dirLock]:<EOL><INDENT>if name == em:<EOL><INDENT>return False, "<STR_LIT>"%em<EOL><DEDENT><DEDENT>for pm in [self.__fileInfo,self.__fileLock]:<EOL><INDENT>if name == pm or (name.endswith(pm[<NUM_LIT:3>:]) and name.startswith('<STR_LIT:.>')):<EOL><INDENT>return False, "<STR_LIT>"%pm<EOL><DEDENT><DEDENT>return True, None<EOL>

Get whether creating a file or a directory from the basenane of the given
path is allowed

:Parameters:
    #. path (str): The absolute or relative path or simply the file
       or directory name.

:Returns:
    #. allowed (bool): Whether name is allowed.
    #. message (None, str): Reason for the name to be forbidden.


def to_repo_relative_path(self, path, split=False):

path = os.path.normpath(path)<EOL>if path == '<STR_LIT:.>':<EOL><INDENT>path = '<STR_LIT>'<EOL><DEDENT>path = path.split(self.__path)[-<NUM_LIT:1>].strip(os.sep)<EOL>if split:<EOL><INDENT>return path.split(os.sep)<EOL><DEDENT>else:<EOL><INDENT>return path<EOL><DEDENT>

Given a path, return relative path to diretory

:Parameters:
    #. path (str): Path as a string
    #. split (boolean): Whether to split path to its components

:Returns:
    #. relativePath (str, list): Relative path as a string or as a list
       of components if split is True


state = []<EOL>def _walk_dir(relaPath, dirList):<EOL><INDENT>dirDict = {'<STR_LIT:type>':'<STR_LIT>',<EOL>'<STR_LIT>':os.path.isdir(os.path.join(self.__path,relaPath)),<EOL>'<STR_LIT>':os.path.isfile(os.path.join(self.__path,relaPath,self.__dirInfo)),<EOL>}<EOL>state.append({relaPath:dirDict})<EOL>for fname in sorted([f for f in dirList if isinstance(f, basestring)]):<EOL><INDENT>relaFilePath = os.path.join(relaPath,fname)<EOL>realFilePath = os.path.join(self.__path,relaFilePath)<EOL>fileDict = {'<STR_LIT:type>':'<STR_LIT:file>',<EOL>'<STR_LIT>':os.path.isfile(realFilePath),<EOL>'<STR_LIT>':os.path.isfile(os.path.join(self.__path,relaPath,self.__fileInfo%fname)),<EOL>}<EOL>state.append({relaFilePath:fileDict})<EOL><DEDENT>for ddict in sorted([d for d in dirList if isinstance(d, dict)], key=lambda k: list(k)[<NUM_LIT:0>]):<EOL><INDENT>dirname = list(ddict)[<NUM_LIT:0>]<EOL>_walk_dir(relaPath=os.path.join(relaPath,dirname), dirList=ddict[dirname])<EOL><DEDENT><DEDENT>if relaPath is None:<EOL><INDENT>_walk_dir(relaPath='<STR_LIT>', dirList=self.__repo['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>assert isinstance(relaPath, basestring), "<STR_LIT>"<EOL>relaPath = self.to_repo_relative_path(path=relaPath, split=False)<EOL>spath    = relaPath.split(os.sep)<EOL>dirList  = self.__repo['<STR_LIT>']<EOL>while len(spath):<EOL><INDENT>dirname = spath.pop(<NUM_LIT:0>)<EOL>dList   = [d for d in dirList if isinstance(d, dict)]<EOL>if not len(dList):<EOL><INDENT>dirList = None<EOL>break<EOL><DEDENT>cDict = [d for d in dList if dirname in d]<EOL>if not len(cDict):<EOL><INDENT>dirList = None<EOL>break<EOL><DEDENT>dirList = cDict[<NUM_LIT:0>][dirname]<EOL><DEDENT>if dirList is not None:<EOL><INDENT>_walk_dir(relaPath=relaPath, dirList=dirList)<EOL><DEDENT><DEDENT>return state<EOL>

Get a list representation of repository state along with useful
information. List state is ordered relativeley to directories level

:Parameters:
    #. relaPath (None, str): relative directory path from where to
       start. If None all repository representation is returned.

:Returns:
    #. state (list): List representation of the repository.
       List items are all dictionaries. Every dictionary has a single
       key which is the file or the directory name and the value is a
       dictionary of information including:

           * 'type': the type of the tracked whether it's file, dir, or objectdir
           * 'exists': whether file or directory actually exists on disk
           * 'pyrepfileinfo': In case of a file or an objectdir whether .%s_pyrepfileinfo exists
           * 'pyrepdirinfo': In case of a directory whether .pyrepdirinfo exists


return copy.deepcopy(self.__get_repository_directory(relativePath))<EOL>

Get repository directory list copy.

:Parameters:
    #. relativePath (string): The relative to the repository path .

:Returns:
    #. dirList (None, list): List of directories and files in repository
       directory. If directory is not tracked in repository None is
       returned


relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>fileName     = os.path.basename(relativePath)<EOL>isRepoFile,fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)<EOL>if not isRepoFile:<EOL><INDENT>return None, "<STR_LIT>"<EOL><DEDENT>if not infoOnDisk:<EOL><INDENT>return None, "<STR_LIT>"<EOL><DEDENT>fileInfoPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileInfo%fileName)<EOL>try:<EOL><INDENT>with open(fileInfoPath, '<STR_LIT:rb>') as fd:<EOL><INDENT>info = pickle.load(fd)<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>return None, "<STR_LIT>"%str(err)<EOL><DEDENT>return info, '<STR_LIT>'<EOL>

Get file information dict from the repository given its relative path.

:Parameters:
    #. relativePath (string): The relative to the repository path of
       the file.

:Returns:
    #. info (None, dictionary): The file information dictionary.
       If None, it means an error has occurred.
    #. errorMessage (string): The error message if any error occurred.


return self.__get_repository_directory(relativePath) is not None<EOL>

Get whether directory is registered in repository.

:Parameters:
    #. relativePath (string): The relative to the repository path.

:Returns:
    #. result (boolean): Whether directory is tracked and registered.


relativePath  = self.to_repo_relative_path(path=relativePath, split=False)<EOL>if relativePath == '<STR_LIT>':<EOL><INDENT>return False, False, False, False<EOL><DEDENT>relaDir, name = os.path.split(relativePath)<EOL>fileOnDisk    = os.path.isfile(os.path.join(self.__path, relativePath))<EOL>infoOnDisk    = os.path.isfile(os.path.join(self.__path,os.path.dirname(relativePath),self.__fileInfo%name))<EOL>classOnDisk   = os.path.isfile(os.path.join(self.__path,os.path.dirname(relativePath),self.__fileClass%name))<EOL>cDir          = self.__repo['<STR_LIT>']<EOL>if len(relaDir):<EOL><INDENT>for dirname in relaDir.split(os.sep):<EOL><INDENT>dList = [d for d in cDir if isinstance(d, dict)]<EOL>if not len(dList):<EOL><INDENT>cDir = None<EOL>break<EOL><DEDENT>cDict = [d for d in dList if dirname in d]<EOL>if not len(cDict):<EOL><INDENT>cDir = None<EOL>break<EOL><DEDENT>cDir = cDict[<NUM_LIT:0>][dirname]<EOL><DEDENT><DEDENT>if cDir is None:<EOL><INDENT>return False, fileOnDisk, infoOnDisk, classOnDisk<EOL><DEDENT>if str(name) not in [str(i) for i in cDir]:<EOL><INDENT>return False, fileOnDisk, infoOnDisk, classOnDisk<EOL><DEDENT>return True, fileOnDisk, infoOnDisk, classOnDisk<EOL>

Check whether a given relative path is a repository file path

:Parameters:
    #. relativePath (string): File relative path

:Returns:
    #. isRepoFile (boolean): Whether file is a repository file.
    #. isFileOnDisk (boolean): Whether file is found on disk.
    #. isFileInfoOnDisk (boolean): Whether file info is found on disk.
    #. isFileClassOnDisk (boolean): Whether file class is found on disk.


@path_required<EOL><INDENT>def walk_files_path(self, relativePath="<STR_LIT>", fullPath=False, recursive=False):<DEDENT>

assert isinstance(fullPath, bool), "<STR_LIT>"<EOL>assert isinstance(recursive, bool), "<STR_LIT>"<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>dirList      = self.__get_repository_directory(relativePath=relativePath)<EOL>assert dirList is not None, "<STR_LIT>"%relativePath<EOL>def _walk(rpath, dlist,recursive):<EOL><INDENT>for fname in dlist:<EOL><INDENT>if isinstance(fname, basestring):<EOL><INDENT>if fullPath:<EOL><INDENT>yield os.path.join(self.__path, rpath, fname)<EOL><DEDENT>else:<EOL><INDENT>yield os.path.join(rpath, fname)<EOL><DEDENT><DEDENT><DEDENT>if recursive:<EOL><INDENT>for ddict in dlist:<EOL><INDENT>if isinstance(ddict, dict):<EOL><INDENT>dname = list(ddict)[<NUM_LIT:0>]<EOL>for p in _walk(rpath=os.path.join(rpath,dname), dlist=ddict[dname],recursive=recursive):<EOL><INDENT>yield p<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return _walk(rpath=relativePath, dlist=dirList, recursive=recursive)<EOL>

Walk the repository relative path and yield file relative/full path.

:parameters:
    #. relativePath (string): The relative path from which start the walk.
    #. fullPath (boolean): Whether to return full or relative path.
    #. recursive (boolean): Whether walk all directories files recursively


def walk_files_info(self, relativePath="<STR_LIT>", fullPath=False, recursive=False):

assert isinstance(fullPath, bool), "<STR_LIT>"<EOL>assert isinstance(recursive, bool), "<STR_LIT>"<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>for relaPath in self.walk_files_path(relativePath=relativePath, fullPath=False, recursive=recursive):<EOL><INDENT>fpath, fname = os.path.split(relaPath)<EOL>fileInfoPath = os.path.join(self.__path,fpath,self.__fileInfo%fname)<EOL>if os.path.isfile(fileInfoPath):<EOL><INDENT>with open(fileInfoPath, '<STR_LIT:rb>') as fd:<EOL><INDENT>info = pickle.load(fd)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>info = None<EOL><DEDENT>if fullPath:<EOL><INDENT>yield (os.path.join(self.__path, relaPath), info)<EOL><DEDENT>else:<EOL><INDENT>yield (relaPath, info)<EOL><DEDENT><DEDENT>

Walk the repository relative path and yield tuple of two items where
first item is file relative/full path and second item is file info.
If file info is not found on disk, second item will be None.

:parameters:
    #. relativePath (string): The relative path from which start the walk.
    #. fullPath (boolean): Whether to return full or relative path.
    #. recursive (boolean): Whether walk all directories files recursively


def walk_directories_path(self, relativePath="<STR_LIT>", fullPath=False, recursive=False):

assert isinstance(fullPath, bool), "<STR_LIT>"<EOL>assert isinstance(recursive, bool), "<STR_LIT>"<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>dirList      = self.__get_repository_directory(relativePath=relativePath)<EOL>assert dirList is not None, "<STR_LIT>"%relativePath<EOL>def _walk(rpath, dlist,recursive):<EOL><INDENT>for ddict in dlist:<EOL><INDENT>if isinstance(ddict, dict):<EOL><INDENT>dname = list(ddict)[<NUM_LIT:0>]<EOL>if fullPath:<EOL><INDENT>yield os.path.join(self.__path, rpath, dname)<EOL><DEDENT>else:<EOL><INDENT>yield os.path.join(rpath, dname)<EOL><DEDENT><DEDENT><DEDENT>if recursive:<EOL><INDENT>for ddict in dlist:<EOL><INDENT>if isinstance(ddict, dict):<EOL><INDENT>dname = list(ddict)[<NUM_LIT:0>]<EOL>for p in _walk(rpath=os.path.join(rpath,dname), dlist=ddict[dname],recursive=recursive):<EOL><INDENT>yield p<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return _walk(rpath=relativePath, dlist=dirList, recursive=recursive)<EOL>

Walk repository relative path and yield directory relative/full path

:parameters:
    #. relativePath (string): The relative path from which start the walk.
    #. fullPath (boolean): Whether to return full or relative path.
    #. recursive (boolean): Whether walk all directories files recursively.


def walk_directories_info(self, relativePath="<STR_LIT>", fullPath=False, recursive=False):

assert isinstance(fullPath, bool), "<STR_LIT>"<EOL>assert isinstance(recursive, bool), "<STR_LIT>"<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>for dpath in self.walk_directories_path(relativePath=relativePath, fullPath=False, recursive=recursive):<EOL><INDENT>dirInfoPath = os.path.join(self.__path,dpath,self.__dirInfo)<EOL>if os.path.isfile(dirInfoPath):<EOL><INDENT>with open(dirInfoPath, '<STR_LIT:rb>') as fd:<EOL><INDENT>info = pickle.load(fd)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>info = None<EOL><DEDENT>if fullPath:<EOL><INDENT>yield (os.path.join(self.__path, dpath), info)<EOL><DEDENT>else:<EOL><INDENT>yield (dpath, info)<EOL><DEDENT><DEDENT>

Walk the repository relative path and yield tuple of two items where
first item is directory relative/full path and second item is directory
info. If directory file info is not found on disk, second item will be None.

:parameters:
    #. relativePath (string): The relative path from which start the walk.
    #. fullPath (boolean): Whether to return full or relative path.
    #. recursive (boolean): Whether walk all directories files recursively.


@path_required<EOL><INDENT>def create_package(self, path=None, name=None, mode=None):<DEDENT>

<EOL>assert mode in (None, '<STR_LIT:w>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'), '<STR_LIT>'%str(mode)<EOL>if mode is None:<EOL><INDENT>mode = '<STR_LIT>'<EOL><DEDENT>if path is None:<EOL><INDENT>root = os.path.split(self.__path)[<NUM_LIT:0>]<EOL><DEDENT>elif path.strip() in ('<STR_LIT>','<STR_LIT:.>'):<EOL><INDENT>root = os.getcwd()<EOL><DEDENT>else:<EOL><INDENT>root = os.path.realpath( os.path.expanduser(path) )<EOL><DEDENT>assert os.path.isdir(root), '<STR_LIT>'%path<EOL>if name is None:<EOL><INDENT>ext = mode.split("<STR_LIT::>")<EOL>if len(ext) == <NUM_LIT:2>:<EOL><INDENT>if len(ext[<NUM_LIT:1>]):<EOL><INDENT>ext = "<STR_LIT:.>"+ext[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>ext = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>ext = '<STR_LIT>'<EOL><DEDENT>name = os.path.split(self.__path)[<NUM_LIT:1>]+ext<EOL><DEDENT>tarfilePath = os.path.join(root, name)<EOL>try:<EOL><INDENT>tarHandler = tarfile.TarFile.open(tarfilePath, mode=mode)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise Exception("<STR_LIT>"%e)<EOL><DEDENT>for dpath in sorted(list(self.walk_directories_path(recursive=True))):<EOL><INDENT>t = tarfile.TarInfo( dpath )<EOL>t.type = tarfile.DIRTYPE<EOL>tarHandler.addfile(t)<EOL>tarHandler.add(os.path.join(self.__path,dpath,self.__dirInfo), arcname=self.__dirInfo)<EOL><DEDENT>for fpath in self.walk_files_path(recursive=True):<EOL><INDENT>relaPath, fname = os.path.split(fpath)<EOL>tarHandler.add(os.path.join(self.__path,fpath), arcname=fname)<EOL>tarHandler.add(os.path.join(self.__path,relaPath,self.__fileInfo%fname), arcname=self.__fileInfo%fname)<EOL>tarHandler.add(os.path.join(self.__path,relaPath,self.__fileClass%fname), arcname=self.__fileClass%fname)<EOL><DEDENT>tarHandler.add(os.path.join(self.__path,self.__repoFile), arcname="<STR_LIT>")<EOL>tarHandler.close()<EOL>

Create a tar file package of all the repository files and directories.
Only files and directories that are tracked in the repository
are stored in the package tar file.

**N.B. On some systems packaging requires root permissions.**

:Parameters:
    #. path (None, string): The real absolute path where to create the
       package. If None, it will be created in the same directory as
       the repository. If '.' or an empty string is passed, the current
       working directory will be used.
    #. name (None, string): The name to give to the package file
       If None, the package directory name will be used with the
       appropriate extension added.
    #. mode (None, string): The writing mode of the tarfile.
       If None, automatically the best compression mode will be chose.
       Available modes are ('w', 'w:', 'w:gz', 'w:bz2')


@path_required<EOL><INDENT>def add_directory(self, relativePath, description=None, clean=False,<EOL>raiseError=True, ntrials=<NUM_LIT:3>):<DEDENT>

assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(relativePath, basestring), "<STR_LIT>"<EOL>if description is not None:<EOL><INDENT>assert isinstance(description, basestring), "<STR_LIT>"<EOL><DEDENT>assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>path = self.to_repo_relative_path(path=relativePath, split=False)<EOL>if self.is_repository_directory(path):<EOL><INDENT>return True, "<STR_LIT>"<EOL><DEDENT>allowed, reason = self.is_name_allowed(path)<EOL>if not allowed:<EOL><INDENT>if raiseError:<EOL><INDENT>raise Exception(reason)<EOL><DEDENT>return False, reason<EOL><DEDENT>LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))<EOL>acquired, code = LR.acquire_lock()<EOL>if not acquired:<EOL><INDENT>m = "<STR_LIT>"%(code,)<EOL>if raiseError:<EOL><INDENT>raise Exception(m)<EOL><DEDENT>return False,m<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>repo = self.__load_repository_pickle_file(os.path.join(self.__path, self.__repoFile))<EOL>self.__repo['<STR_LIT>'] = repo['<STR_LIT>']<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = str(err)<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>break<EOL><DEDENT><DEDENT>if error is not None:<EOL><INDENT>_ = LR.release_lock()<EOL>assert not raiseError, Exception(error)<EOL>return False, error<EOL><DEDENT>error   = None<EOL>posList = self.__repo['<STR_LIT>']<EOL>dirPath = self.__path<EOL>spath   = path.split(os.sep)<EOL>for idx, name in enumerate(spath):<EOL><INDENT>LD =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(dirPath, self.__dirLock))<EOL>acquired, code = LD.acquire_lock()<EOL>if not acquired:<EOL><INDENT>error = "<STR_LIT>"%(code,dirPath)<EOL>break<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>dirPath = os.path.join(dirPath, name)<EOL>riPath  = os.path.join(dirPath, self.__dirInfo)<EOL>dList   = [d for d in posList if isinstance(d, dict)]<EOL>dList   = [d for d in dList if name in d]<EOL>if not len(dList) and clean and os.path.exists(dirPath):<EOL><INDENT>try:<EOL><INDENT>shutil.rmtree( dirPath, ignore_errors=True )<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = "<STR_LIT>"%(dirPath, err)<EOL>break<EOL><DEDENT><DEDENT>if not os.path.exists(dirPath):<EOL><INDENT>try:<EOL><INDENT>os.mkdir(dirPath)<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = "<STR_LIT>"%(dirPath, err)<EOL>break<EOL><DEDENT><DEDENT>self.__save_dirinfo(description=[None, description][idx==len(spath)-<NUM_LIT:1>],<EOL>dirInfoPath=riPath, create=True)<EOL>if not len(dList):<EOL><INDENT>rsd = {name:[]}<EOL>posList.append(rsd)<EOL>posList = rsd[name]<EOL><DEDENT>else:<EOL><INDENT>assert len(dList) == <NUM_LIT:1>, "<STR_LIT>"<EOL>posList = dList[<NUM_LIT:0>][name]<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>LD.release_lock()<EOL>error = "<STR_LIT>"%(dirPath, str(err))<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>LD.release_lock()<EOL>break<EOL><DEDENT><DEDENT>if error is not None:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if error is None:<EOL><INDENT>try:<EOL><INDENT>_, error = self.__save_repository_pickle_file(lockFirst=False, raiseError=False)<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = str(err)<EOL>pass<EOL><DEDENT><DEDENT>try:<EOL><INDENT>LD.release_lock()<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>LR.release_lock()<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>assert error is None or not raiseError, error<EOL>return error is None, error<EOL>

Add a directory in the repository and creates its attribute in the
Repository with utc timestamp. It insures adding all the missing
directories in the path.

:Parameters:
    #. relativePath (string): The relative to the repository path to
       where directory must be added.
    #. description (None, string): Any random description about the
       added directory.
    #. clean (boolean): Whether to remove existing non repository
       tracked files and folders in all created directory chain tree.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
    #. success (boolean): Whether adding the directory was successful.
    #. message (None, string): Reason why directory was not added or
       random information.


return copy.deepcopy(self.__get_repository_parent_directory(relativePath))<EOL>

Get repository parent directory list copy.

:Parameters:
    #. relativePath (string): The relative to the repository path .

:Returns:
    #. dirList (None, list): List of directories and files in repository
       parent directory. If directory is not tracked in repository
       None is returned


@path_required<EOL><INDENT>def remove_directory(self, relativePath, clean=False, raiseError=True, ntrials=<NUM_LIT:3>):<DEDENT>

assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(clean, bool), "<STR_LIT>"<EOL>assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>parentPath, dirName = os.path.split(relativePath)<EOL>if relativePath == '<STR_LIT>':<EOL><INDENT>return False, "<STR_LIT>"<EOL><DEDENT>if not self.is_repository_directory(relativePath):<EOL><INDENT>return False, "<STR_LIT>"%relativePath<EOL><DEDENT>realPath = os.path.join(self.__path,relativePath)<EOL>if not os.path.isdir(realPath):<EOL><INDENT>error = "<STR_LIT>"<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT>LD =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path,parentPath,self.__dirLock))<EOL>acquired, code = LD.acquire_lock()<EOL>if not acquired:<EOL><INDENT>error = "<STR_LIT>"%(code,realPath)<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT>LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))<EOL>acquired, code = LR.acquire_lock()<EOL>if not acquired:<EOL><INDENT>LD.release_lock()<EOL>m = "<STR_LIT>"%(code,)<EOL>assert raiseError,  Exception(m)<EOL>return False,m<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>error = None<EOL>try:<EOL><INDENT>dirList = self.__get_repository_parent_directory(relativePath=relativePath)<EOL>assert dirList is not None, "<STR_LIT>"%(relativePath,)<EOL>stateBefore = self.get_repository_state(relaPath=parentPath)<EOL>_files = [f for f in dirList if isinstance(f, basestring)]<EOL>_dirs  = [d for d in dirList if isinstance(d, dict)]<EOL>_dirs  = [d for d in dirList if dirName not in d]<EOL>_ = [dirList.pop(<NUM_LIT:0>) for _ in range(len(dirList))]<EOL>dirList.extend(_files)<EOL>dirList.extend(_dirs)<EOL>if clean:<EOL><INDENT>shutil.rmtree(realPath)<EOL><DEDENT>else:<EOL><INDENT>stateAfter = self.get_repository_state(relaPath=parentPath)<EOL>success, errors = self.__clean_before_after(stateBefore=stateBefore, stateAfter=stateAfter, keepNoneEmptyDirectory=True)<EOL>assert success, "<STR_LIT:\n>".join(errors)<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>error = str(err)<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if error is None:<EOL><INDENT>_, error = self.__save_repository_pickle_file(lockFirst=False, raiseError=False)<EOL><DEDENT>LD.release_lock()<EOL>LR.release_lock()<EOL>assert error is None or not raiseError, "<STR_LIT>"%(relativePath, ntrials, error,)<EOL>return error is None, error<EOL>

Remove directory from repository tracking.

:Parameters:
    #. relativePath (string): The relative to the repository path of the
       directory to remove from the repository.
    #. clean (boolean): Whether to os remove directory. If False only
       tracked files will be removed along with left empty directories.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
    #. success (boolean): Whether removing the directory was successful.
    #. reason (None, string): Reason why directory was not removed.


@path_required<EOL><INDENT>def rename_directory(self, relativePath, newName, raiseError=True, ntrials=<NUM_LIT:3>):<DEDENT>

assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>parentPath, dirName = os.path.split(relativePath)<EOL>if relativePath == '<STR_LIT>':<EOL><INDENT>error = "<STR_LIT>"<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT>realPath = os.path.join(self.__path,relativePath)<EOL>newRealPath = os.path.join(os.path.dirname(realPath), newName)<EOL>if os.path.isdir(newRealPath):<EOL><INDENT>error = "<STR_LIT>"%(newRealPath,)<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT>LD =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path,parentPath, self.__dirLock))<EOL>acquired, code = LD.acquire_lock()<EOL>if not acquired:<EOL><INDENT>error = "<STR_LIT>"%(code,dirPath)<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT>error = None<EOL>LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))<EOL>acquired, code = LR.acquire_lock()<EOL>if not acquired:<EOL><INDENT>LD.release_lock()<EOL>m = "<STR_LIT>"%(code,dirPath)<EOL>assert raiseError,  Exception(m)<EOL>return False,m<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>repo = self.__load_repository_pickle_file(os.path.join(self.__path, self.__repoFile))<EOL>self.__repo['<STR_LIT>'] = repo['<STR_LIT>']<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = str(err)<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>break<EOL><DEDENT><DEDENT>if error is not None:<EOL><INDENT>LD.release_lock()<EOL>LR.release_lock()<EOL>assert not raiseError, Exception(error)<EOL>return False, error<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>error = None<EOL>try:<EOL><INDENT>dirList = self.__get_repository_parent_directory(relativePath=relativePath)<EOL>assert dirList is not None, "<STR_LIT>"%(relativePath,)<EOL>_dirDict = [nd for nd in dirList  if isinstance(nd,dict)]<EOL>_dirDict = [nd for nd in _dirDict if dirName in nd]<EOL>assert len(_dirDict) == <NUM_LIT:1>, "<STR_LIT>"<EOL>os.rename(realPath, newRealPath)<EOL>_dirDict[<NUM_LIT:0>][newName] = _dirDict[<NUM_LIT:0>][dirName]<EOL>_dirDict[<NUM_LIT:0>].pop(dirName)<EOL>self.__save_dirinfo(description=None, dirInfoPath=parentPath, create=False)<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = str(err)<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>break<EOL><DEDENT><DEDENT>if error is None:<EOL><INDENT>_, error = self.__save_repository_pickle_file(lockFirst=False, raiseError=False)<EOL><DEDENT>LR.release_lock()<EOL>LD.release_lock()<EOL>assert error is None or not raiseError, "<STR_LIT>"%(relativePath, newName, ntrials, error,)<EOL>return error is None, error<EOL>

Rename a directory in the repository. It insures renaming the directory in the system.

:Parameters:
    #. relativePath (string): The relative to the repository path of
       the directory to be renamed.
    #. newName (string): The new directory name.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
    #. success (boolean): Whether renaming the directory was successful.
    #. message (None, string): Some explanatory message or error reason
       why directory was not renamed.


@path_required<EOL><INDENT>def copy_directory(self, relativePath, newRelativePath,<EOL>overwrite=False, raiseError=True, ntrials=<NUM_LIT:3>):<DEDENT>

<EOL>assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(overwrite, bool), "<STR_LIT>"<EOL>assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>if relativePath == '<STR_LIT>':<EOL><INDENT>m = "<STR_LIT>"<EOL>assert not raiseError, m<EOL>return False, m<EOL><DEDENT>realPath     = os.path.join(self.__path,relativePath)<EOL>parentRealPath, dirName = os.path.split(realPath)<EOL>parentRelativePath = os.path.dirname(relativePath)<EOL>if not self.is_repository_directory(relativePath):<EOL><INDENT>m = "<STR_LIT>"%(relativePath)<EOL>assert not raiseError, m<EOL>return False, m<EOL><DEDENT>newRelativePath = self.to_repo_relative_path(path=newRelativePath, split=False)<EOL>newRealPath     = os.path.join(self.__path,newRelativePath)<EOL>newParentRealPath, newDirName = os.path.split(newRealPath)<EOL>newParentRelativePath = os.path.dirname(newRelativePath)<EOL>if realPath == newRealPath:<EOL><INDENT>m = "<STR_LIT>"<EOL>assert not raiseError, m<EOL>return False, m<EOL><DEDENT>if self.is_repository_directory(newRelativePath):<EOL><INDENT>m = "<STR_LIT>"%(newRelativePath)<EOL>assert not raiseError, m<EOL>return False, m<EOL><DEDENT>if os.path.isdir(newRealPath):<EOL><INDENT>if overwrite:<EOL><INDENT>try:<EOL><INDENT>shutil.rmtree(newRealPath)<EOL><DEDENT>except Exception as err:<EOL><INDENT>assert not raiseError, str(err)<EOL>return False, str(err)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>error = "<STR_LIT>"%(newRealPath,)<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT><DEDENT>try:<EOL><INDENT>success, reason = self.add_directory(newParentRelativePath, raiseError=False, ntrials=ntrials)<EOL><DEDENT>except Exception as err:<EOL><INDENT>reason  = "<STR_LIT>"%(str(err))<EOL>success = False<EOL><DEDENT>if not success:<EOL><INDENT>assert not raiseError, reason<EOL>return False, reason<EOL><DEDENT>LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))<EOL>acquired, code = LR.acquire_lock()<EOL>if not acquired:<EOL><INDENT>m = "<STR_LIT>"%(code,)<EOL>assert raiseError,  Exception(m)<EOL>return False,m<EOL><DEDENT>try:<EOL><INDENT>repo = self.__load_repository_pickle_file(os.path.join(self.__path, self.__repoFile))<EOL>self.__repo['<STR_LIT>'] = repo['<STR_LIT>']<EOL><DEDENT>except Exception as err:<EOL><INDENT>LR.release_lock()<EOL>assert not raiseError, Exception(str(err))<EOL>return False,m<EOL><DEDENT>L0 =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(parentRealPath, self.__dirLock))<EOL>acquired, code = L0.acquire_lock()<EOL>if not acquired:<EOL><INDENT>LR.release_lock()<EOL>error = "<STR_LIT>"%(code,dirPath)<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT>L1 = None<EOL>if parentRealPath != newParentRealPath:<EOL><INDENT>L1 =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(newParentRealPath, self.__dirLock))<EOL>acquired, code = L1.acquire_lock()<EOL>if not acquired:<EOL><INDENT>L0.release_lock()<EOL>LR.release_lock()<EOL>error = "<STR_LIT>"%(code,dirPath)<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT><DEDENT>error = None<EOL>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>assert self.is_repository_directory(relativePath), "<STR_LIT>"%(relativePath)<EOL>assert not self.is_repository_directory(newRelativePath), "<STR_LIT>"%(relativePath)<EOL>dirList = self.__get_repository_parent_directory(relativePath=relativePath)<EOL>assert dirList is not None, "<STR_LIT>"%(relativePath,)<EOL>newDirList = self.__get_repository_parent_directory(relativePath=newRelativePath)<EOL>assert newDirList is not None, "<STR_LIT>"%(newRelativePath,)<EOL>_dirDict = [nd for nd in dirList  if isinstance(nd,dict)]<EOL>_dirDict = [nd for nd in _dirDict if dirName in nd]<EOL>assert len(_dirDict) == <NUM_LIT:1>, "<STR_LIT>"<EOL>_newDirDict = [nd for nd in newDirList  if isinstance(nd,dict)]<EOL>_newDirDict = [nd for nd in _newDirDict if newDirName in nd]<EOL>assert len(_newDirDict) == <NUM_LIT:0>, "<STR_LIT>"<EOL>_newDirDict = copy.deepcopy(_dirDict[<NUM_LIT:0>])<EOL>if dirName != newDirName:<EOL><INDENT>_newDirDict[newDirName] = _newDirDict.pop(dirName)<EOL><DEDENT>copy_tree(realPath, newRealPath)<EOL>newDirList.append(_newDirDict)<EOL>self.__save_dirinfo(description=None, dirInfoPath=newParentRelativePath, create=False)<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = str(err)<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>break<EOL><DEDENT><DEDENT>if error is None:<EOL><INDENT>_, error = self.__save_repository_pickle_file(lockFirst=False, raiseError=False)<EOL><DEDENT>LR.release_lock()<EOL>L0.release_lock()<EOL>if L1 is not None:<EOL><INDENT>L1.release_lock()<EOL><DEDENT>assert error is None or not raiseError, "<STR_LIT>"%(relativePath, newRelativePath, ntrials, error,)<EOL>return error is None, error<EOL>

Copy a directory in the repository. New directory must not exist.

:Parameters:
    #. relativePath (string): The relative to the repository path of
       the directory to be copied.
    #. newRelativePath (string): The new directory relative path.
    #. overwrite (boolean): Whether to overwrite existing but not tracked
       directory in repository.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
    #. success (boolean): Whether renaming the directory was successful.
    #. message (None, string): Some explanatory message or error reason
       why directory was not renamed.


@path_required<EOL><INDENT>def dump_file(self, value, relativePath,<EOL>description=None,<EOL>dump=None, pull=None,<EOL>replace=False, raiseError=True, ntrials=<NUM_LIT:3>):<DEDENT>

<EOL>assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(replace, bool), "<STR_LIT>"<EOL>assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>if description is None:<EOL><INDENT>description = '<STR_LIT>'<EOL><DEDENT>assert isinstance(description, basestring), "<STR_LIT>"<EOL>if pull is None and dump is not None:<EOL><INDENT>if dump.startswith('<STR_LIT>') or dump.startswith('<STR_LIT>') or dump.startswith('<STR_LIT>') or dump =='<STR_LIT>':<EOL><INDENT>pull = dump<EOL><DEDENT><DEDENT>dump = get_dump_method(dump, protocol=self._DEFAULT_PICKLE_PROTOCOL)<EOL>pull = get_pull_method(pull)<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>savePath     = os.path.join(self.__path,relativePath)<EOL>fPath, fName = os.path.split(savePath)<EOL>success, reason = self.is_name_allowed(savePath)<EOL>if not success:<EOL><INDENT>assert not raiseError, reason<EOL>return False, reason<EOL><DEDENT>try:<EOL><INDENT>success, reason = self.add_directory(fPath, raiseError=False, ntrials=ntrials)<EOL><DEDENT>except Exception as err:<EOL><INDENT>reason  = "<STR_LIT>"%(str(err))<EOL>success = False<EOL><DEDENT>if not success:<EOL><INDENT>assert not raiseError, reason<EOL>return False, reason<EOL><DEDENT>LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))<EOL>acquired, code = LR.acquire_lock()<EOL>if not acquired:<EOL><INDENT>m = "<STR_LIT>"%(code,)<EOL>assert raiseError, Exception(m)<EOL>return False,m<EOL><DEDENT>LF =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(fPath,self.__fileLock%fName))<EOL>acquired, code = LF.acquire_lock()<EOL>if not acquired:<EOL><INDENT>LR.release_lock()<EOL>error = "<STR_LIT>"%(code,relativePath)<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>repo = self.__load_repository_pickle_file(os.path.join(self.__path, self.__repoFile))<EOL>self.__repo['<STR_LIT>'] = repo['<STR_LIT>']<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = str(err)<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>break<EOL><DEDENT><DEDENT>if error is not None:<EOL><INDENT>LR.release_lock()<EOL>LF.release_lock()<EOL>assert not raiseError, Exception(error)<EOL>return False, error<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>error = None<EOL>try:<EOL><INDENT>isRepoFile, fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)<EOL>if isRepoFile:<EOL><INDENT>assert replace, "<STR_LIT>"<EOL><DEDENT>fileInfoPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileInfo%fName)<EOL>if isRepoFile and fileOnDisk:<EOL><INDENT>with open(fileInfoPath, '<STR_LIT:rb>') as fd:<EOL><INDENT>info = pickle.load(fd)<EOL><DEDENT>assert info['<STR_LIT>'] == self.__repo['<STR_LIT>'], "<STR_LIT>"<EOL>info['<STR_LIT>'] = time.time()<EOL><DEDENT>else:<EOL><INDENT>info = {'<STR_LIT>':self.__repo['<STR_LIT>']}<EOL>info['<STR_LIT>'] = info['<STR_LIT>'] = time.time()<EOL><DEDENT>info['<STR_LIT>'] = dump<EOL>info['<STR_LIT>'] = pull<EOL>info['<STR_LIT:description>'] = description<EOL>if not isRepoFile:<EOL><INDENT>dirList = self.__get_repository_directory(fPath)<EOL><DEDENT>my_exec( dump.replace("<STR_LIT>", str(savePath)), locals=locals(), globals=globals(), description='<STR_LIT>'  )<EOL>with open(fileInfoPath, '<STR_LIT:wb>') as fd:<EOL><INDENT>pickle.dump( info,fd, protocol=self._DEFAULT_PICKLE_PROTOCOL)<EOL>fd.flush()<EOL>os.fsync(fd.fileno())<EOL><DEDENT>fileClassPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileClass%fName)<EOL>with open(fileClassPath, '<STR_LIT:wb>') as fd:<EOL><INDENT>if value is None:<EOL><INDENT>klass = None<EOL><DEDENT>else:<EOL><INDENT>klass = value.__class__<EOL><DEDENT>pickle.dump(klass , fd, protocol=self._DEFAULT_PICKLE_PROTOCOL )<EOL>fd.flush()<EOL>os.fsync(fd.fileno())<EOL><DEDENT>if not isRepoFile:<EOL><INDENT>dirList.append(fName)<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>error = "<STR_LIT>"%(str(err),)<EOL>try:<EOL><INDENT>if '<STR_LIT>' in dump:<EOL><INDENT>mi = get_pickling_errors(value)<EOL>if mi is not None:<EOL><INDENT>error += '<STR_LIT>'%str(mi)<EOL><DEDENT><DEDENT><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>break<EOL><DEDENT><DEDENT>if error is None:<EOL><INDENT>_, error = self.__save_repository_pickle_file(lockFirst=False, raiseError=False)<EOL><DEDENT>LR.release_lock()<EOL>LF.release_lock()<EOL>assert not raiseError or error is None, "<STR_LIT>"%(relativePath, ntrials, error,)<EOL>return success, error<EOL>

Dump a file using its value to the system and creates its
attribute in the Repository with utc timestamp.

:Parameters:
    #. value (object): The value of a file to dump and add to the
       repository. It is any python object or file.
    #. relativePath (str): The relative to the repository path to where
       to dump the file.
    #. description (None, string): Any description about the file.
    #. dump (None, string): The dumping method.
       If None it will be set automatically to pickle and therefore the
       object must be pickleable. If a string is given, it can be a
       keyword ('json','pickle','dill') or a string compileable code to
       dump the data. The string code must include all the necessary
       imports and a '$FILE_PATH' that replaces the absolute file path
       when the dumping will be performed.\n
       e.g. "import numpy as np; np.savetxt(fname='$FILE_PATH', X=value, fmt='%.6e')"
    #. pull (None, string): The pulling method. If None it will be set
       automatically to pickle and therefore the object must be
       pickleable. If a string is given, it can be a keyword
       ('json','pickle','dill') or a string compileable code to pull
       the data. The string code must include all the necessary imports,
       a '$FILE_PATH' that replaces the absolute file path when the
       dumping will be performed and finally a PULLED_DATA variable.\n
       e.g "import numpy as np; PULLED_DATA=np.loadtxt(fname='$FILE_PATH')"
    #. replace (boolean): Whether to replace any existing file.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
    #. success (boolean): Whether renaming the directory was successful.
    #. message (None, string): Some explanatory message or error reason
       why directory was not dumped.


return self.dump_file(*args, **kwargs)<EOL>